Skip to content

Commit

Permalink
Merge pull request usegalaxy-eu#195 from sanjaysrikakulam/cloud_migra…
Browse files Browse the repository at this point in the history
…tion_15_4_2024

Cloud migration April 2024
  • Loading branch information
sanjaysrikakulam authored May 22, 2024
2 parents 9bcffa5 + c1a0112 commit fac482e
Show file tree
Hide file tree
Showing 35 changed files with 23,246 additions and 29,050 deletions.
38 changes: 8 additions & 30 deletions dns.tf
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ resource "aws_route53_record" "build-usegalaxy" {

## ZFS server #1 (all flash)
resource "aws_route53_record" "ssds1-galaxyproject" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "zfs0f.galaxyproject.eu"
type = "A"
Expand All @@ -169,6 +170,7 @@ resource "aws_route53_record" "ssds1-galaxyproject" {

## ZFS server #2 (spinning disks w/ flash cache)
resource "aws_route53_record" "zfs1-galaxyproject" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "zfs1.galaxyproject.eu"
type = "A"
Expand All @@ -178,6 +180,7 @@ resource "aws_route53_record" "zfs1-galaxyproject" {

## ZFS server #3 (all flash)
resource "aws_route53_record" "zfs2f-galaxyproject" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "zfs2f.galaxyproject.eu"
type = "A"
Expand All @@ -188,6 +191,7 @@ resource "aws_route53_record" "zfs2f-galaxyproject" {

## ZFS server #4 (all flash)
resource "aws_route53_record" "zfs3f-galaxyproject" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "zfs3f.galaxyproject.eu"
type = "A"
Expand All @@ -204,36 +208,6 @@ resource "aws_route53_record" "zfs3f-galaxyproject" {
# records = ["10.5.68.230"]
#}


# VMs
resource "aws_route53_record" "plausible" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "plausible.galaxyproject.eu"
type = "A"
ttl = "600"
records = ["192.52.44.75"]
}

resource "aws_route53_record" "apollo-main" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "apollo.internal.galaxyproject.eu"
type = "A"
ttl = "600"
records = ["10.5.68.7"]
}

resource "aws_route53_record" "ftp" {
allow_overwrite = true
zone_id = var.zone_usegalaxy_eu
name = "ftp.usegalaxy.eu"
type = "A"
ttl = "600"
records = ["132.230.223.213"]
}


## Interactive Tools
## We redirect all subdomains planning for URLs like
## https://727a121642ce1f94-3a20d7fa7b014959af58c7f6a47d1af.interactivetoolentrypoint.interactivetool.{some-subdomain}.usegalaxy.eu/
Expand Down Expand Up @@ -294,6 +268,7 @@ resource "aws_route53_record" "it-subdomain-main" {

# SPF and DMARC records
resource "aws_route53_record" "usegalaxy_eu_dmarc_txt" {
allow_overwrite = true
zone_id = var.zone_usegalaxy_eu
name = "_dmarc.usegalaxy.eu"
type = "TXT"
Expand All @@ -304,6 +279,7 @@ resource "aws_route53_record" "usegalaxy_eu_dmarc_txt" {
}

resource "aws_route53_record" "usegalaxy_eu_spf_txt" {
allow_overwrite = true
zone_id = var.zone_usegalaxy_eu
name = ""
type = "TXT"
Expand All @@ -314,6 +290,7 @@ resource "aws_route53_record" "usegalaxy_eu_spf_txt" {
}

resource "aws_route53_record" "galaxyproject_eu_dmarc_txt" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "_dmarc.galaxyproject.eu"
type = "TXT"
Expand All @@ -324,6 +301,7 @@ resource "aws_route53_record" "galaxyproject_eu_dmarc_txt" {
}

resource "aws_route53_record" "galaxyproject_eu_spf_txt" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = ""
type = "TXT"
Expand Down
26 changes: 26 additions & 0 deletions instance_core_apollo.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
resource "openstack_compute_instance_v2" "apollo-usegalaxy" {
name = "apollo.internal.galaxyproject.eu"
image_name = "apollo_16_04_2024"
flavor_name = "m1.large"
key_pair = "cloud2"
security_groups = ["egress", "public-web2", "public-ssh", "default", "public-ping"]

network {
name = "bioinf"
}

user_data = <<-EOF
#cloud-config
package_update: true
package_upgrade: true
EOF
}

resource "aws_route53_record" "apollo-usegalaxy-internal" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "apollo.internal.galaxyproject.eu"
type = "A"
ttl = "600"
records = ["${openstack_compute_instance_v2.apollo-usegalaxy.access_ip_v4}"]
}
2 changes: 1 addition & 1 deletion instance_core_celery.tf
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ data "openstack_images_image_v2" "celery-image" {
resource "openstack_compute_instance_v2" "celery" {
name = "celery-${count.index}.galaxyproject.eu"
image_id = data.openstack_images_image_v2.celery-image.id
flavor_name = "c1.c36m100"
flavor_name = "m1.xxlarge"
key_pair = "cloud2"
tags = []
security_groups = ["default", "ingress-from-proxy"]
Expand Down
21 changes: 12 additions & 9 deletions instance_core_cvmfs0_eu.tf
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ variable "cvmfs-stratum0-eu-dns" {

resource "openstack_compute_instance_v2" "cvmfs-stratum0-eu" {
name = var.cvmfs-stratum0-eu-dns
image_name = "generic-rockylinux8-v60-j168-5333625af7b2-main"
image_name = "cvmfs-stratum0_22_04_2024"
flavor_name = "m1.small"
key_pair = "cloud2"
security_groups = ["egress", "public-ssh", "public-ping", "public-web2"]
Expand All @@ -22,15 +22,18 @@ resource "openstack_compute_instance_v2" "cvmfs-stratum0-eu" {
EOF
}

resource "openstack_blockstorage_volume_v2" "cvmfs-data-eu" {
name = "cvmfs stratum 0 EU"
description = "spool space for cvmfs"
size = 500
}
# 22.4.2024: This resource creation is commented out because the volume
# from the old cloud is being attached to the instance in the new cloud.
# resource "openstack_blockstorage_volume_v2" "cvmfs-data-eu" {
# name = "cvmfs stratum 0 EU"
# description = "spool space for cvmfs"
# size = 500
# }

resource "openstack_compute_volume_attach_v2" "cvmfs-va-eu" {
instance_id = "${openstack_compute_instance_v2.cvmfs-stratum0-eu.id}"
volume_id = "${openstack_blockstorage_volume_v2.cvmfs-data-eu.id}"
resource "openstack_compute_volume_attach_v2" "cvmfs-stratum0-va-eu" {
instance_id = openstack_compute_instance_v2.cvmfs-stratum0-eu.id
volume_id = "b637697c-5227-4b0c-a300-7afdd2256cc4"
device = "/dev/vdb"
}

resource "aws_route53_record" "cvmfs-stratum0-eu" {
Expand Down
2 changes: 1 addition & 1 deletion instance_core_cvmfs1-ufr0.tf
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ variable "cvmfs1-ufr0-eu-dns" {

resource "openstack_compute_instance_v2" "cvmfs1-ufr0-eu" {
name = var.cvmfs1-ufr0-eu-dns
image_name = "generic-rockylinux8-v60-j168-5333625af7b2-main"
image_name = "cvmfs1-ufr0-internal_22_04_2024"
flavor_name = "m1.small"
key_pair = "cloud2"
security_groups = ["egress", "public-ssh", "public-ping", "public-web2"]
Expand Down
17 changes: 10 additions & 7 deletions instance_core_dokku.tf
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
resource "openstack_compute_instance_v2" "dokku" {
name = "apps.galaxyproject.eu"
image_name = "generic-rockylinux8-v60-j168-5333625af7b2-main"
image_name = "Ubuntu 22.04"
flavor_name = "m1.xlarge"
key_pair = "cloud2"
security_groups = ["egress", "public-ssh", "public-ping", "public-web2"]
Expand All @@ -10,15 +10,18 @@ resource "openstack_compute_instance_v2" "dokku" {
}
}

resource "openstack_blockstorage_volume_v2" "dokku-data" {
name = "stats"
description = "Data volume for dokku"
size = 50
}
# 17.4.2024: This resource creation is commented out because the volume
# from the old cloud is being attached to the instance in the new cloud.
# resource "openstack_blockstorage_volume_v3" "dokku-data" {
# name = "stats"
# description = "Data volume for dokku"
# size = 50
# }

resource "openstack_compute_volume_attach_v2" "dokku-va" {
instance_id = openstack_compute_instance_v2.dokku.id
volume_id = openstack_blockstorage_volume_v2.dokku-data.id
volume_id = "6cbce9f7-1ffd-4848-9ec8-0a2ccfd52225"
device = "/dev/vdb"
}

resource "aws_route53_record" "dokku-dns" {
Expand Down
9 changes: 9 additions & 0 deletions instance_core_incoming.tf
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,12 @@ resource "aws_route53_record" "incoming" {
ttl = "600"
records = ["${openstack_compute_instance_v2.incoming.access_ip_v4}"]
}

resource "aws_route53_record" "ftp" {
allow_overwrite = true
zone_id = var.zone_usegalaxy_eu
name = "ftp.usegalaxy.eu"
type = "A"
ttl = "600"
records = ["${openstack_compute_instance_v2.incoming.access_ip_v4}"]
}
24 changes: 10 additions & 14 deletions instance_core_influxdb.tf
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
resource "openstack_compute_instance_v2" "influxdb-usegalaxy" {
name = "influxdb.galaxyproject.eu"
image_name = "generic-rockylinux8-v60-j168-5333625af7b2-main"
image_name = "influxdb_15_04_2024"
flavor_name = "m1.xlarge"
key_pair = "cloud2"
security_groups = ["egress", "public-ssh", "public-ping", "public-influxdb", "public-web2"]
Expand All @@ -15,27 +15,23 @@ resource "openstack_compute_instance_v2" "influxdb-usegalaxy" {

user_data = <<-EOF
#cloud-config
bootcmd:
- test -z "$(blkid /dev/vdb)" && mkfs -t ext4 /dev/vdb
- mkdir -p /data
mounts:
- ["/dev/vdb", "/data", auto, "defaults,nofail", "0", "2"]
runcmd:
- [ chown, "centos.centos", -R, /data ]
package_update: true
package_upgrade: true
EOF
}

resource "openstack_blockstorage_volume_v2" "influxdb-data" {
name = "influxdb"
description = "Data volume for InfluxDB"
size = 100
}
# 17.4.2024: This resource creation is commented out because the volume
# from the old cloud is being attached to the instance in the new cloud.
# resource "openstack_blockstorage_volume_v3" "influxdb-data" {
# name = "influxdb"
# description = "Data volume for InfluxDB"
# size = 100
# }

resource "openstack_compute_volume_attach_v2" "influxdb-va" {
instance_id = openstack_compute_instance_v2.influxdb-usegalaxy.id
volume_id = openstack_blockstorage_volume_v2.influxdb-data.id
volume_id = "f6e77471-9180-4055-941f-b18fe353571c"
device = "/dev/vdb"
}

resource "aws_route53_record" "influxdb-usegalaxy-internal" {
Expand Down
97 changes: 49 additions & 48 deletions instance_core_jenkins-worker-bronze.tf
Original file line number Diff line number Diff line change
@@ -1,55 +1,56 @@
variable "workers-bronze" {
default = 1
}
# 26.04.2024: Disabled as not needed anymore
# variable "workers-bronze" {
# default = 1
# }

variable "workers-bronze-volume-size" {
default = 200
}
# variable "workers-bronze-volume-size" {
# default = 200
# }

resource "openstack_compute_instance_v2" "jenkins-workers-bronze" {
name = "worker-${count.index}.bronze.build.galaxyproject.eu"
image_name = var.jenkins_image
flavor_name = "m1.xlarge"
key_pair = "jenkins2"
security_groups = ["default"]
count = var.workers-bronze
# resource "openstack_compute_instance_v2" "jenkins-workers-bronze" {
# name = "worker-${count.index}.bronze.build.galaxyproject.eu"
# image_name = var.jenkins_image
# flavor_name = "m1.xlarge"
# key_pair = "jenkins2"
# security_groups = ["default"]
# count = var.workers-bronze

network {
name = "bioinf"
}
# network {
# name = "bioinf"
# }

user_data = <<-EOF
#cloud-config
bootcmd:
- test -z "$(blkid /dev/vdb)" && mkfs -t ext4 -L jenkins /dev/vdb
- mkdir -p /scratch
mounts:
- ["/dev/vdb", "/scratch", auto, "defaults,nofail", "0", "2"]
runcmd:
- [ chown, "centos.centos", -R, /scratch ]
EOF
}
# user_data = <<-EOF
# #cloud-config
# bootcmd:
# - test -z "$(blkid /dev/vdb)" && mkfs -t ext4 -L jenkins /dev/vdb
# - mkdir -p /scratch
# mounts:
# - ["/dev/vdb", "/scratch", auto, "defaults,nofail", "0", "2"]
# runcmd:
# - [ chown, "centos.centos", -R, /scratch ]
# EOF
# }

resource "openstack_blockstorage_volume_v2" "jenkins-workers-bronze-volume" {
name = "jenkins-workers-bronze-volume"
description = "Data volume for Jenkins worker-${count.index}.bronze.build.galaxyproject.eu"
volume_type = "default"
size = var.workers-bronze-volume-size
count = var.workers-bronze
}
# resource "openstack_blockstorage_volume_v2" "jenkins-workers-bronze-volume" {
# name = "jenkins-workers-bronze-volume"
# description = "Data volume for Jenkins worker-${count.index}.bronze.build.galaxyproject.eu"
# volume_type = "default"
# size = var.workers-bronze-volume-size
# count = var.workers-bronze
# }

resource "openstack_compute_volume_attach_v2" "jenkins-workers-bronze-va" {
instance_id = element(openstack_compute_instance_v2.jenkins-workers-bronze.*.id, count.index)
volume_id = element(openstack_blockstorage_volume_v2.jenkins-workers-bronze-volume.*.id, count.index)
count = var.workers-bronze
}
# resource "openstack_compute_volume_attach_v2" "jenkins-workers-bronze-va" {
# instance_id = element(openstack_compute_instance_v2.jenkins-workers-bronze.*.id, count.index)
# volume_id = element(openstack_blockstorage_volume_v2.jenkins-workers-bronze-volume.*.id, count.index)
# count = var.workers-bronze
# }

resource "aws_route53_record" "jenkins-workers-bronze" {
allow_overwrite = true
zone_id = var.zone_galaxyproject_eu
name = "worker-${count.index}.bronze.build.galaxyproject.eu"
type = "A"
ttl = "7200"
records = ["${element(openstack_compute_instance_v2.jenkins-workers-bronze.*.access_ip_v4, count.index)}"]
count = var.workers-bronze
}
# resource "aws_route53_record" "jenkins-workers-bronze" {
# allow_overwrite = true
# zone_id = var.zone_galaxyproject_eu
# name = "worker-${count.index}.bronze.build.galaxyproject.eu"
# type = "A"
# ttl = "7200"
# records = ["${element(openstack_compute_instance_v2.jenkins-workers-bronze.*.access_ip_v4, count.index)}"]
# count = var.workers-bronze
# }
4 changes: 2 additions & 2 deletions instance_core_jenkins-worker-gold.tf
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ resource "openstack_compute_instance_v2" "jenkins-workers-gold" {
EOF
}

resource "openstack_blockstorage_volume_v2" "jenkins-workers-gold-volume" {
resource "openstack_blockstorage_volume_v3" "jenkins-workers-gold-volume" {
name = "jenkins-workers-${count.index}-gold-volume"
description = "Data volume for Jenkins worker-${count.index}.gold.build.galaxyproject.eu"
volume_type = "default"
Expand All @@ -40,7 +40,7 @@ resource "openstack_blockstorage_volume_v2" "jenkins-workers-gold-volume" {

resource "openstack_compute_volume_attach_v2" "jenkins-workers-gold-va" {
instance_id = element(openstack_compute_instance_v2.jenkins-workers-gold.*.id, count.index)
volume_id = element(openstack_blockstorage_volume_v2.jenkins-workers-gold-volume.*.id, count.index)
volume_id = element(openstack_blockstorage_volume_v3.jenkins-workers-gold-volume.*.id, count.index)
count = var.workers-gold
}

Expand Down
Loading

0 comments on commit fac482e

Please sign in to comment.