Upgrade Postgres 11 -> 15

Dockerfile

@@ -13,11 +13,12 @@ FROM python:3.10-slim-bullseye as base
 #   postgresql-client -- for running database commands
 # We need to recreate the /usr/share/man/man{1..8} directories first because
 # they were clobbered by a parent image.
+ENV POSTGRESQL_CLIENT_VERSION="15"
 RUN set -ex \
     && RUN_DEPS=" \
     libpcre3 \
     mime-support \
-    postgresql-client-12 \
+    postgresql-client-${POSTGRESQL_CLIENT_VERSION} \
     vim \
     " \
     && seq 1 8 | xargs -I{} mkdir -p /usr/share/man/man{} \
@@ -142,7 +143,7 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
     mime-support \
     nodejs \
     openssh-client \
-    postgresql-client-14 \
+    postgresql-client-${POSTGRESQL_CLIENT_VERSION} \
     sudo \
     vim \
     zlib1g-dev \

README.md

@@ -11,7 +11,7 @@ To begin you should have the following applications installed on your local deve
 - [pip](http://www.pip-installer.org/) >= 20
 - [virtualenv](http://www.virtualenv.org/) >= 1.10
 - [virtualenvwrapper](http://pypi.python.org/pypi/virtualenvwrapper) >= 3.0
-- Postgres >= 11
+- Postgres >= 15
 - git >= 2.26
 
 

deploy/deploy-services.yaml

@@ -0,0 +1,29 @@
+- name: Deploy project-related K8s services
+  hosts: k8s
+  gather_facts: no
+  tasks:
+    - name: Create/update pgBouncer SelfSigned Certificate
+      kubernetes.core.k8s:
+        host: "{{ k8s_auth_host }}"
+        ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
+        state: present
+        wait: yes
+        validate:
+          fail_on_error: yes
+          strict: yes
+        definition:
+          apiVersion: cert-manager.io/v1
+          kind: Certificate
+          metadata:
+            name: "{{ k8s_pgbouncer_selfsigned_certificate_name }}"
+            namespace: "{{ k8s_namespace }}"
+          spec:
+            commonName: "{{ k8s_pgbouncer_selfsigned_certificate_name }}"
+            secretName: "{{ k8s_pgbouncer_selfsigned_certificate_name }}"
+            privateKey:
+              algorithm: ECDSA
+              size: 256
+            issuerRef:
+              name: "{{ k8s_pgbouncer_selfsigned_certificate_cluster_issuer }}"
+              kind: ClusterIssuer
+              group: cert-manager.io

deploy/group_vars/all.yml

@@ -39,6 +39,9 @@ admin_database_password: !vault |
 # caktus.aws-web-stacks: Ansible role to automate AWS CloudFormation stack
 #                        provisioning with aws-web-stacks.
 # ----------------------------------------------------------------------------
+# Parameters
+cloudformation_stack_database_engine_version: "15.4"
+cloudformation_stack_database_parameter_group_family: postgres15
 
 cloudformation_stack:
   profile: "{{ aws_profile }}"
@@ -62,8 +65,8 @@ cloudformation_stack:
     AssetsBucketAccessControl: Private
     AssetsUseCloudFront: "false"
     DatabaseClass: db.t3.small
-    DatabaseEngineVersion: "11"
-    DatabaseParameterGroupFamily: postgres11
+    DatabaseEngineVersion: "{{ cloudformation_stack_database_engine_version }}"
+    DatabaseParameterGroupFamily: "{{ cloudformation_stack_database_parameter_group_family }}"
     DatabaseName: "{{ app_name }}"
     DatabaseUser: "{{ app_name }}_admin"
     DatabasePassword: "{{ admin_database_password }}"
@@ -101,6 +104,12 @@ k8s_auth_host: "{{ ClusterEndpoint }}"
 k8s_auth_ssl_ca_cert: "k8s_auth_ssl_ca_cert.txt"
 k8s_memcached_enabled: true
 
+# Use a SelfSigned issuer to create a Certificate to enable TLS mode to use for
+# connections from clients (client_tls_sslmode)
+# https://cert-manager.io/docs/configuration/selfsigned/
+k8s_pgbouncer_selfsigned_certificate_cluster_issuer: cluster-selfsigned-ca-issuer
+k8s_pgbouncer_selfsigned_certificate_name: pgbouncer-client-tls
+
 # App pod configuration:
 k8s_container_name: app
 k8s_container_port: 8000
@@ -136,6 +145,7 @@ env_default_file_storage: "{{ app_name }}.storages.MediaBoto3Storage"
 env_media_storage_bucket_name: "{{ app_name }}-{{ env_name }}-philly-private-assets"
 env_aws_default_acl: ""
 env_media_location: media/
+env_database_ssl_require: true
 
 # S3 bucket configuration:
 k8s_s3_cluster_name: "{{ cluster_name }}"
@@ -180,6 +190,10 @@ azure_client_secret: !vault |
   6339636266626239303165666261353362626564363636346665
 
 k8s_environment_variables:
+  CLIENT_TLS_SSLMODE: require
+  CLIENT_TLS_CERT_FILE: /etc/pgbouncer/ssl/tls.crt
+  CLIENT_TLS_KEY_FILE: /etc/pgbouncer/ssl/tls.key
+  SERVER_TLS_SSLMODE: require
   DATABASE_URL: "{{ env_database_url }}"
   DJANGO_SETTINGS_MODULE: "{{ env_django_settings }}"
   DJANGO_DEBUG: "False"
@@ -220,3 +234,7 @@ k8s_descheduler_release_values:
       # During upgrades or reboots, don't pre-emptively drain a node.
       RemovePodsViolatingNodeTaints:
         enabled: false
+
+
+# Must be PostgreSQL v12 for DB restore
+k8s_hosting_services_image_tag: 0.5.0-postgres15

deploy/host_vars/production.yml

@@ -94,6 +94,7 @@ k8s_hosting_services_project_name: "{{ app_name }}"
 k8s_hosting_services_healthcheck_url: https://hc-ping.com/7d334762-8f35-453e-85b7-55e0b3147723
 k8s_hosting_services_database_url: "{{ env_database_url }}"
 k8s_hosting_services_backup_base_bucket: hip-production-philly-backups
+k8s_hosting_services_image_tag: 0.5.0-postgres15
 k8s_hosting_services_aws_region: us-east-1
 k8s_hosting_services_aws_access_key: AKIAQ4VHAAV5UM2FBDUB
 k8s_hosting_services_aws_secret_access_key: !vault |

deploy/stack/eks-no-nat.yml

@@ -799,6 +799,9 @@ Parameters:
       - postgres10
       - postgres11
       - postgres12
+      - postgres13
+      - postgres14
+      - postgres15
       - sqlserver-ee-11.0
       - sqlserver-ee-12.0
       - sqlserver-ee-13.0
@@ -1085,6 +1088,7 @@ Resources:
     DeletionPolicy: Retain
     Properties:
       AccessControl: !Ref 'AssetsBucketAccessControl'
+      AllowMajorVersionUpgrade: 'true'
       BucketEncryption: !If
         - UseAES256EncryptionCond
         - ServerSideEncryptionConfiguration:

deploy/templates/selfsigned-clusterissuer.yaml

@@ -0,0 +1,31 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: cluster-selfsigned-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: cluster-selfsigned-ca
+  secretName: cluster-selfsigned-root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cluster-selfsigned-ca-issuer
+spec:
+  ca:
+    secretName: cluster-selfsigned-root-secret

docker-compose.yml

@@ -12,7 +12,7 @@ services:
     environment:
       POSTGRES_HOST_AUTH_METHOD: trust
       POSTGRES_DB: hip
-    image: postgres:12-alpine
+    image: postgres:15-alpine
     ports:
       - "5433:5432"
     volumes:
@@ -39,7 +39,7 @@ services:
       args:
         USER_UID: ${USER_UID:-1000}
         USER_GID: ${USER_GID:-1000}
-    command: ["sleep", "infinity"]
+    command: [ "sleep", "infinity" ]
     links:
       - db:db
     ports:

requirements/base/base.in

@@ -27,7 +27,7 @@ pdfid
 social-auth-app-django
 
 # Postgres
-psycopg2-binary
+psycopg2-binary==2.9.9
 
 
 # Wagtail

requirements/base/base.txt

@@ -122,7 +122,7 @@ phonenumberslite==8.12.18
     # via -r requirements/base/base.in
 pillow==9.3.0
     # via wagtail
-psycopg2-binary==2.8.6
+psycopg2-binary==2.9.9
     # via -r requirements/base/base.in
 pycparser==2.20
     # via cffi

requirements/dev/dev.txt

@@ -17,7 +17,10 @@ anyascii==0.1.7
 anyio==2.1.0
     # via jupyter-server
 appnope==0.1.2
-    # via -r requirements/dev/dev.in
+    # via
+    #   -r requirements/dev/dev.in
+    #   ipykernel
+    #   ipython
 argon2-cffi==20.1.0
     # via notebook
 asgiref==3.3.4