From 6a45476578e02fe10ba2be216b10b7b54a4fa23a Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sat, 11 Feb 2023 16:28:43 +0000 Subject: [PATCH 01/11] Add an initial host config for marconi and the docker role --- playbook.yml | 5 +++++ roles/docker/tasks/docker-compose.yml | 9 +++++++++ roles/docker/tasks/docker.yml | 27 +++++++++++++++++++++++++++ roles/docker/tasks/main.yml | 5 +++++ 4 files changed, 46 insertions(+) create mode 100644 roles/docker/tasks/docker-compose.yml create mode 100644 roles/docker/tasks/docker.yml create mode 100644 roles/docker/tasks/main.yml diff --git a/playbook.yml b/playbook.yml index ddfbce0..82e1334 100644 --- a/playbook.yml +++ b/playbook.yml @@ -40,3 +40,8 @@ roles: - role: ssh_gateway tags: ssh_gateway +- name: Configure marconi + hosts: MARCONI + roles: + - role: docker + tags: docker diff --git a/roles/docker/tasks/docker-compose.yml b/roles/docker/tasks/docker-compose.yml new file mode 100644 index 0000000..0530010 --- /dev/null +++ b/roles/docker/tasks/docker-compose.yml @@ -0,0 +1,9 @@ +--- +- name: Install python3 pip + apt: + name: python3-pip + state: present + +- name: Install docker-compose + pip: + name: docker-compose diff --git a/roles/docker/tasks/docker.yml b/roles/docker/tasks/docker.yml new file mode 100644 index 0000000..481c002 --- /dev/null +++ b/roles/docker/tasks/docker.yml @@ -0,0 +1,27 @@ +--- +- name: Install docker dependancies + apt: + name: [ + "apt-transport-https", + "ca-certificates", + "curl", + "gnupg-agent", + "software-properties-common" + ] + state: present + +- name: Add docker GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + +- name: Add docker repo + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable + state: present + +- name: Install docker + apt: + update_cache: true + name: docker-ce + state: present diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml new file mode 100644 index 0000000..756edf0 --- /dev/null +++ b/roles/docker/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: Configure docker + import_tasks: docker.yml +- name: Configure docker-compose + import_tasks: docker-compose.yml From 80e02b367dceb198a07421bbfa7e39784bf2cbd7 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 12 Nov 2023 17:36:38 +0000 Subject: [PATCH 02/11] Add container configuration --- README.md | 1 + playbook.yml | 2 ++ roles/suws_webserver/tasks/containers.yml | 22 ++++++++++++++++++++++ roles/suws_webserver/tasks/main.yml | 3 +++ 4 files changed, 28 insertions(+) create mode 100644 roles/suws_webserver/tasks/containers.yml create mode 100644 roles/suws_webserver/tasks/main.yml diff --git a/README.md b/README.md index bc76015..dec2334 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,4 @@ New roles should be developed on branches, and changes rolled out to all servers - `ipxe` - `backed_up` - `ssh_gateway` +- `suws_webserver` diff --git a/playbook.yml b/playbook.yml index 82e1334..cbbb3ab 100644 --- a/playbook.yml +++ b/playbook.yml @@ -45,3 +45,5 @@ roles: - role: docker tags: docker + - role: suws_webserver + tags: suws_webserver diff --git a/roles/suws_webserver/tasks/containers.yml b/roles/suws_webserver/tasks/containers.yml new file mode 100644 index 0000000..b598ea5 --- /dev/null +++ b/roles/suws_webserver/tasks/containers.yml @@ -0,0 +1,22 @@ +--- +- name: Install git + ansible.builtin.apt: + name: git + state: present + +- name: Checkout container configuration + ansible.builtin.git: + repo: "git@github.com:sown/marconi_container_config.git" + dest: "/docker" + version: master + +- name: Setup containers + community.docker.docker_compose: + project_src: "{{ item }}" + with_items: + # setup utility containers first + - "/docker/traefik" + # setup remaining containers + - "/docker/suws_website_default" + - "/docker/suws_wordpress" + - "/docker/suws_wiki" diff --git a/roles/suws_webserver/tasks/main.yml b/roles/suws_webserver/tasks/main.yml new file mode 100644 index 0000000..5758196 --- /dev/null +++ b/roles/suws_webserver/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Configure containers + import_tasks: containers.yml From 6cd70940c9fe9c9912af7cf115e1374789a0d638 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 10 Dec 2023 17:33:11 +0000 Subject: [PATCH 03/11] Cleanup docker role --- roles/docker/tasks/docker.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/roles/docker/tasks/docker.yml b/roles/docker/tasks/docker.yml index 481c002..4f34798 100644 --- a/roles/docker/tasks/docker.yml +++ b/roles/docker/tasks/docker.yml @@ -1,13 +1,12 @@ --- - name: Install docker dependancies apt: - name: [ - "apt-transport-https", - "ca-certificates", - "curl", - "gnupg-agent", - "software-properties-common" - ] + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common state: present - name: Add docker GPG key @@ -19,9 +18,9 @@ apt_repository: repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable state: present + update_cache: true - name: Install docker apt: - update_cache: true name: docker-ce state: present From 9e5c99c57061d589fb0781259e832b190611f492 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 10 Dec 2023 18:17:43 +0000 Subject: [PATCH 04/11] Add containers docker configurations containers-prod replaces marconi for hosting the suws website --- playbook.yml | 15 +++++++++++---- roles/containers_dev/tasks/containers.yml | 19 +++++++++++++++++++ .../tasks/main.yml | 0 .../tasks/containers.yml | 2 +- roles/containers_prod/tasks/main.yml | 3 +++ 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 roles/containers_dev/tasks/containers.yml rename roles/{suws_webserver => containers_dev}/tasks/main.yml (100%) rename roles/{suws_webserver => containers_prod}/tasks/containers.yml (87%) create mode 100644 roles/containers_prod/tasks/main.yml diff --git a/playbook.yml b/playbook.yml index cbbb3ab..8421dca 100644 --- a/playbook.yml +++ b/playbook.yml @@ -40,10 +40,17 @@ roles: - role: ssh_gateway tags: ssh_gateway -- name: Configure marconi - hosts: MARCONI +- name: Configure containers dev + hosts: CONTAINERS-1 roles: - role: docker tags: docker - - role: suws_webserver - tags: suws_webserver + - role: containers_dev + tags: containers_dev +- name: Configure containers prod + hosts: CONTAINERS-2 + roles: + - role: docker + tags: docker + - role: containers_prod + tags: containers_prod diff --git a/roles/containers_dev/tasks/containers.yml b/roles/containers_dev/tasks/containers.yml new file mode 100644 index 0000000..984c500 --- /dev/null +++ b/roles/containers_dev/tasks/containers.yml @@ -0,0 +1,19 @@ +--- +- name: Install git + ansible.builtin.apt: + name: git + state: present + +- name: Checkout container configuration + ansible.builtin.git: + repo: "git@github.com:sown/containers-dev-docker-config.git" + dest: "/docker/managed" + version: master + +- name: Setup containers + community.docker.docker_compose: + project_src: "{{ item }}" + with_items: + # setup utility containers first + - "/docker/managed/traefik" + # setup remaining containers diff --git a/roles/suws_webserver/tasks/main.yml b/roles/containers_dev/tasks/main.yml similarity index 100% rename from roles/suws_webserver/tasks/main.yml rename to roles/containers_dev/tasks/main.yml diff --git a/roles/suws_webserver/tasks/containers.yml b/roles/containers_prod/tasks/containers.yml similarity index 87% rename from roles/suws_webserver/tasks/containers.yml rename to roles/containers_prod/tasks/containers.yml index b598ea5..c021dc4 100644 --- a/roles/suws_webserver/tasks/containers.yml +++ b/roles/containers_prod/tasks/containers.yml @@ -6,7 +6,7 @@ - name: Checkout container configuration ansible.builtin.git: - repo: "git@github.com:sown/marconi_container_config.git" + repo: "git@github.com:sown/containers-prod-docker-config.git" dest: "/docker" version: master diff --git a/roles/containers_prod/tasks/main.yml b/roles/containers_prod/tasks/main.yml new file mode 100644 index 0000000..5758196 --- /dev/null +++ b/roles/containers_prod/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Configure containers + import_tasks: containers.yml From 4dc48eb5655c718ac7003d13adc5d2711c8decb3 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 10 Dec 2023 18:21:45 +0000 Subject: [PATCH 05/11] Update readme with new tags --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dec2334..6b8a4bb 100644 --- a/README.md +++ b/README.md @@ -68,4 +68,6 @@ New roles should be developed on branches, and changes rolled out to all servers - `ipxe` - `backed_up` - `ssh_gateway` -- `suws_webserver` +- `docker` +- `containers_dev` +- `containers_prod` From f568ffe50b6a1ad4d000d40218a4254293a14186 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 17 Dec 2023 22:12:58 +0000 Subject: [PATCH 06/11] Add configuration for secure containers server --- playbook.yml | 7 +++++++ roles/containers_secure/tasks/containers.yml | 19 +++++++++++++++++++ roles/containers_secure/tasks/main.yml | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 roles/containers_secure/tasks/containers.yml create mode 100644 roles/containers_secure/tasks/main.yml diff --git a/playbook.yml b/playbook.yml index 8421dca..f050cb8 100644 --- a/playbook.yml +++ b/playbook.yml @@ -54,3 +54,10 @@ tags: docker - role: containers_prod tags: containers_prod +- name: Configure containers secure + hosts: CONTAINERS-3 + roles: + - role: docker + tags: docker + - role: containers_secure + tags: containers_secure diff --git a/roles/containers_secure/tasks/containers.yml b/roles/containers_secure/tasks/containers.yml new file mode 100644 index 0000000..c79738b --- /dev/null +++ b/roles/containers_secure/tasks/containers.yml @@ -0,0 +1,19 @@ +--- +- name: Install git + ansible.builtin.apt: + name: git + state: present + +- name: Checkout container configuration + ansible.builtin.git: + repo: "git@github.com:sown/containers-secure-docker-config.git" + dest: "/docker" + version: master + +- name: Setup containers + docker_compose: + project_src: "{{ item }}" + with_items: + # setup utility containers first + - "/docker/traefik" + # setup remaining containers diff --git a/roles/containers_secure/tasks/main.yml b/roles/containers_secure/tasks/main.yml new file mode 100644 index 0000000..5758196 --- /dev/null +++ b/roles/containers_secure/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Configure containers + import_tasks: containers.yml From bd272503b8790885feb16c1e69d379cb7a443d54 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 17 Dec 2023 22:16:35 +0000 Subject: [PATCH 07/11] Fix issues with docker compose version compatibility --- roles/docker/tasks/docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/docker/tasks/docker-compose.yml b/roles/docker/tasks/docker-compose.yml index 0530010..5bf7ca3 100644 --- a/roles/docker/tasks/docker-compose.yml +++ b/roles/docker/tasks/docker-compose.yml @@ -4,6 +4,13 @@ name: python3-pip state: present +# Install docker-compose as required by ansible +# Ansible requires docker-compose version 1 insalled by python +# dockers python library v 7 breaks compatibility with docker-compse with no intention to fix either version +# so lock the version of docker as well - name: Install docker-compose pip: - name: docker-compose + name: + - docker==6.1.3 + - docker-compose + state: present From 2ff278dee562a5cfe655861ad3a182f449b0225f Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 11 Feb 2024 19:27:15 +0000 Subject: [PATCH 08/11] use FQDN to fix linter error --- roles/containers_secure/tasks/containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/containers_secure/tasks/containers.yml b/roles/containers_secure/tasks/containers.yml index c79738b..75090bf 100644 --- a/roles/containers_secure/tasks/containers.yml +++ b/roles/containers_secure/tasks/containers.yml @@ -11,7 +11,7 @@ version: master - name: Setup containers - docker_compose: + community.docker.docker_compos: project_src: "{{ item }}" with_items: # setup utility containers first From 030972fb896dda3011379a01d98371a7e136c28c Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 11 Feb 2024 22:58:19 +0000 Subject: [PATCH 09/11] Add container role to hosts and add missing containers --- hosts.yml | 1 + roles/containers_prod/tasks/containers.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hosts.yml b/hosts.yml index dcd8808..feac01e 100644 --- a/hosts.yml +++ b/hosts.yml @@ -12,3 +12,4 @@ query_filters: - role: server - role: ssh-gateway - role: lxd-host + - role: container-host diff --git a/roles/containers_prod/tasks/containers.yml b/roles/containers_prod/tasks/containers.yml index c021dc4..19f96ad 100644 --- a/roles/containers_prod/tasks/containers.yml +++ b/roles/containers_prod/tasks/containers.yml @@ -20,3 +20,5 @@ - "/docker/suws_website_default" - "/docker/suws_wordpress" - "/docker/suws_wiki" + - "/docker/prometheus" + - "/docker/grafana" From f0b3ad7200d70579c5ef6a5f5922c33922912e78 Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Sun, 11 Feb 2024 23:05:03 +0000 Subject: [PATCH 10/11] Update to use docker compose v2 This also updates ansible to the latest version to get this feature Also remove docker-compose v1 instalation --- requirements.txt | 77 ++++++++++++-------- roles/containers_dev/tasks/containers.yml | 2 +- roles/containers_prod/tasks/containers.yml | 2 +- roles/containers_secure/tasks/containers.yml | 2 +- roles/docker/tasks/docker-compose.yml | 16 ---- roles/docker/tasks/main.yml | 2 - roles/monitored/vars/main.yml | 4 +- 7 files changed, 50 insertions(+), 55 deletions(-) delete mode 100644 roles/docker/tasks/docker-compose.yml diff --git a/requirements.txt b/requirements.txt index 4165225..f527c15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,89 +1,99 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile # -ansible==8.0.0 +ansible==9.2.0 # via -r requirements.in -ansible-compat==4.1.2 +ansible-compat==4.1.11 # via ansible-lint -ansible-core==2.15.0 +ansible-core==2.16.3 # via # ansible # ansible-compat # ansible-lint -ansible-lint==6.17.0 +ansible-lint==24.2.0 # via -r requirements.in -attrs==23.1.0 - # via jsonschema -black==23.3.0 +attrs==23.2.0 + # via + # jsonschema + # referencing +black==24.1.1 # via ansible-lint -bracex==2.3.post1 +bracex==2.4 # via wcmatch -build==0.10.0 +build==1.0.3 # via pip-tools -cffi==1.15.1 +cffi==1.16.0 # via cryptography -click==8.1.3 +click==8.1.7 # via # black # pip-tools -cryptography==41.0.1 +cryptography==42.0.2 # via ansible-core -filelock==3.12.1 +filelock==3.13.1 # via ansible-lint -jinja2==3.1.2 +jinja2==3.1.3 # via ansible-core -jsonschema==4.17.3 +jsonschema==4.21.1 # via # ansible-compat # ansible-lint +jsonschema-specifications==2023.12.1 + # via jsonschema markdown-it-py==3.0.0 # via rich -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 mdurl==0.1.2 # via markdown-it-py mypy-extensions==1.0.0 # via black -packaging==23.1 +packaging==23.2 # via # ansible-compat # ansible-core # ansible-lint # black # build -pathspec==0.11.1 +pathspec==0.12.1 # via # ansible-lint # black # yamllint -pip-tools==6.13.0 +pip-tools==7.3.0 # via -r requirements.in -platformdirs==3.5.3 +platformdirs==4.2.0 # via black pycparser==2.21 # via cffi -pygments==2.15.1 +pygments==2.17.2 # via rich pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyyaml==6.0 +pyyaml==6.0.1 # via # ansible-compat # ansible-core # ansible-lint # yamllint +referencing==0.33.0 + # via + # jsonschema + # jsonschema-specifications resolvelib==1.0.1 # via ansible-core -rich==13.4.2 +rich==13.7.0 # via ansible-lint -ruamel-yaml==0.17.31 +rpds-py==0.17.1 + # via + # jsonschema + # referencing +ruamel-yaml==0.18.6 # via ansible-lint -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via ruamel-yaml subprocess-tee==0.4.1 # via @@ -93,12 +103,15 @@ tomli==2.0.1 # via # black # build + # pip-tools # pyproject-hooks -wcmatch==8.4.1 +typing-extensions==4.9.0 + # via black +wcmatch==8.5 # via ansible-lint -wheel==0.40.0 +wheel==0.42.0 # via pip-tools -yamllint==1.32.0 +yamllint==1.34.0 # via # -r requirements.in # ansible-lint diff --git a/roles/containers_dev/tasks/containers.yml b/roles/containers_dev/tasks/containers.yml index 984c500..184b9a4 100644 --- a/roles/containers_dev/tasks/containers.yml +++ b/roles/containers_dev/tasks/containers.yml @@ -11,7 +11,7 @@ version: master - name: Setup containers - community.docker.docker_compose: + community.docker.docker_compose_v2: project_src: "{{ item }}" with_items: # setup utility containers first diff --git a/roles/containers_prod/tasks/containers.yml b/roles/containers_prod/tasks/containers.yml index 19f96ad..7b94f32 100644 --- a/roles/containers_prod/tasks/containers.yml +++ b/roles/containers_prod/tasks/containers.yml @@ -11,7 +11,7 @@ version: master - name: Setup containers - community.docker.docker_compose: + community.docker.docker_compose_v2: project_src: "{{ item }}" with_items: # setup utility containers first diff --git a/roles/containers_secure/tasks/containers.yml b/roles/containers_secure/tasks/containers.yml index 75090bf..1704543 100644 --- a/roles/containers_secure/tasks/containers.yml +++ b/roles/containers_secure/tasks/containers.yml @@ -11,7 +11,7 @@ version: master - name: Setup containers - community.docker.docker_compos: + community.docker.docker_compose_v2: project_src: "{{ item }}" with_items: # setup utility containers first diff --git a/roles/docker/tasks/docker-compose.yml b/roles/docker/tasks/docker-compose.yml deleted file mode 100644 index 5bf7ca3..0000000 --- a/roles/docker/tasks/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Install python3 pip - apt: - name: python3-pip - state: present - -# Install docker-compose as required by ansible -# Ansible requires docker-compose version 1 insalled by python -# dockers python library v 7 breaks compatibility with docker-compse with no intention to fix either version -# so lock the version of docker as well -- name: Install docker-compose - pip: - name: - - docker==6.1.3 - - docker-compose - state: present diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 756edf0..9e4dc0c 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,5 +1,3 @@ --- - name: Configure docker import_tasks: docker.yml -- name: Configure docker-compose - import_tasks: docker-compose.yml diff --git a/roles/monitored/vars/main.yml b/roles/monitored/vars/main.yml index f6d6c03..e140ff4 100644 --- a/roles/monitored/vars/main.yml +++ b/roles/monitored/vars/main.yml @@ -1,4 +1,4 @@ -nrpe_allowed_ips: +nrpe_allowed_ips: # noqa var-naming[no-role-prefix] - 127.0.0.1 - ::1 - 10.5.0.243 # monitor @@ -7,7 +7,7 @@ nrpe_allowed_ips: - 10.5.0.215 # monitor2 - 152.78.103.187 # monitor2 - 2001:630:d0:f700::215 # monitor2 -nrpe_checks: +nrpe_checks: # noqa var-naming[no-role-prefix] users: check: "/usr/lib/nagios/plugins/check_users" arguments: "-w 10 -c 20" From b7b0613dc82a7bbed2ec1b29516e1b9beefdbdad Mon Sep 17 00:00:00 2001 From: Tyler Ward Date: Tue, 5 Mar 2024 22:17:50 +0000 Subject: [PATCH 11/11] Update python to 3.10 --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index be3e668..9d265f4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: "3.10" - uses: actions/cache@v2 with: path: venv/