Merge pull request #67 from lsst-dm/tickets/DM-48635

DM-48635 Add tooling for local testing of consdb

.github/workflows/build.yaml

@@ -32,13 +32,13 @@ jobs:
         with:
           image: ${{ github.repository }}-hinfo
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          dockerfile: Dockerfile.hinfo
+          dockerfile: docker/Dockerfile.hinfo
 
       - name: Build pqserver
         uses: lsst-sqre/build-and-push-to-ghcr@v1
         with:
           image: ${{ github.repository }}-pq
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          dockerfile: Dockerfile.pqserver
+          dockerfile: docker/Dockerfile.pqserver
           build-args: |
             GITHUB_TAG=${{ env.GITHUB_TAG }}

.github/workflows/pytest.yaml

@@ -18,7 +18,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Build docker image
-      run: docker build -f Dockerfile.pytest -t pytest_image .
+      run: docker build -f docker/Dockerfile.pytest -t pytest_image .
 
     - name: Create directory for coverage report
       run: mkdir -p ${{ github.workspace }}/pytest_reports

.gitignore

@@ -33,3 +33,9 @@ htmlcov
 .tox
 .venv
 .vscode
+
+# docker build products
+.env
+docker/pgadmin-data
+docker/pgadmin-cfg
+docker/pgdata

docker/Dockerfile.alembic

@@ -0,0 +1,27 @@
+FROM python:3.12-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV CONSDB_URL=""
+ENV SDM_SCHEMAS_DIR="/sdm_schemas"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git postgresql-client && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/lsst/sdm_schemas.git
+
+# Create a working directory
+WORKDIR /app
+
+# Add python dependencies
+RUN pip install \
+    lsst-felis \
+    testing.postgresql \
+    alembic sqlalchemy \
+    pyyaml \
+    black \
+    psycopg2-binary
+
+COPY . /app/
+CMD [ "/bin/sh", "-c" ]

Dockerfile.hinfo → docker/Dockerfile.hinfo

@@ -1,4 +1,4 @@
-ARG OBS_LSST_VERSION=w_2024_21
+ARG OBS_LSST_VERSION=w_2025_04
 FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION}
 USER lsst
 

docker/docker-compose.yml

@@ -0,0 +1,286 @@
+# This file is part of consdb.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -------------------------------------------------------------------------
+#
+# This file defines a multi-container Docker application with the following
+# services:
+# - A PostgreSQL database server (db).
+# - A pgAdmin web interface for managing the database (pgadmin).
+# - A pqserver application (app) and a reference pqserver application (refapp)
+#   that connect to the database.
+#
+# You can use the app on port 8888 to test changes to pqserver, and compare
+# with a reference copy (such as a checkout from the main branch) on port 8889
+# to observe differences in behavior.
+#
+# Usage:
+# 1. (optional) Get some header service YAML files and place them in some
+#    directories. Set up environment variables (perhaps in a .env file)
+#    to point to these directories.
+# 2. Run `docker-compose up --build -d` to start all services.
+# 3. Wait for the alembic migrations to update the database and the YAML
+#    files to be ingested. This may take a while if you have a lot of
+#    YAML files.
+# 4. Access services via the following URLs:
+#    - pgAdmin: http://localhost:8080 (default credentials: [email protected]
+#      password consdb).
+#    - app: http://localhost:8888.
+#    - refapp: http://localhost:8889.
+# 5. Use `docker-compose down` to stop all services (data is persisted in
+#    volumes).
+#
+# If you're looking for some header service YAML files, try:
+#     /sdf/data/rubin/offline/s3-tmp-backup/
+#
+# Environment Variables:
+# - REFERENCE_PQSERV: Defines the directory context for building the "refapp"
+#   service.
+#   By default, it points to "../../consdb", probably giving you two identical
+#   copies of pqserver, but can be overridden to use a different location.
+# - ALEMBIC_UPDATE: If 1 (the default), then Alembic will be used to update
+#   the schema in the database to the head revision.
+# - LATISS_YAML: A directory that contains sample YAML files generated by the
+#   AT header service.
+#   If defined, the contents of this directory will be loaded into the
+#   database.
+# - COMCAM_YAML: A directory that contains sample YAML files generated by
+# the CC header service.
+#
+# These variables can be stored in an .env file in the same directory as
+# docker-compose.yml. You can also use this file for sensitive information
+# such as passwords. Some of the security settings you can change are listed
+# below. Remember to change the passwords to something unique to your system.
+#
+# POSTGRES_PASSWORD=pgpassword
+# [email protected]
+# PGADMIN_DEFAULT_PASSWORD=consdb
+# PGADMIN_CONFIG_SERVER_MODE=True
+# PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=True
+#
+# Do not commit the .env file to version control. If no file is provided, the
+# containers will use the above passwords by default. Note that these settings
+# affect the environment only when run for the first time; if you want to
+# change any of these settings you'll need to do so manually or delete
+# your docker volumes and start over.
+#
+# Ensure Docker and Docker Compose are installed before using this file.
+---
+services:
+  pgadmin-init:
+    # This service prepares configuration files for pgadmin4. It stays
+    # alive when it's done so that pgadmin4 can depend_on the health check.
+    image: busybox:1.37-uclibc
+    command:
+      - /bin/sh
+      - -c
+      - |
+        echo '{
+          "Servers": {
+            "1": {
+              "Name": "consdb",
+              "Group": "Servers",
+              "Host": "db",
+              "Port": 5432,
+              "MaintenanceDB": "postgres",
+              "Username": "postgres",
+              "PassFile": "/pgpass",
+              "SSLMode": "prefer",
+              "SavePassword": true
+            }
+          }
+        }' > /home/servers.json && chmod 666 /home/servers.json
+        echo '*:*:*:postgres:${POSTGRES_PASSWORD:-pgpassword}' > /home/pgpass
+        chmod 600 /home/pgpass
+        echo "Config files written" > /dev/stderr
+        while (true); do sleep 3600; done
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /home/servers.json && test -f /home/pgpass"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    volumes:
+      - ./pgadmin-cfg:/home/
+
+  db:
+    # The PostgreSQL database service
+    image: postgres:17.2-alpine3.21
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-pgpassword}
+    volumes:
+      - ./pgdata:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 10s
+      retries: 5
+
+  pgadmin:
+    # This is a pgadmin4 instance for convenient inspection of the database.
+    user: "0"
+    image: dpage/pgadmin4:8.14
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:[email protected]}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-consdb}
+      PGADMIN_CONFIG_SERVER_MODE: ${PGADMIN_CONFIG_SERVER_MODE:-False}
+      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED:
+        ${PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED:-False}
+    volumes:
+      - ./pgadmin-data:/var/lib/pgadmin
+      - ./pgadmin-cfg/servers.json:/pgadmin4/servers.json
+      - ./pgadmin-cfg/pgpass:/root/.pgpass
+    ports:
+      - 8080:80
+    depends_on:
+      pgadmin-init:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+
+
+  alembic:
+    # This service exists to run alembic on the database, if needed. If not
+    # needed, you can set the ALEMBIC_UPDATE environment variable to 0. This
+    # service updates the latiss, lsstcomcam, and lsstcomcamsim databases to
+    # the head revision.
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.alembic
+    environment:
+      ALEMBIC_UPDATE: ${ALEMBIC_UPDATE:-1}
+      CONSDB_URL:
+        postgresql://postgres:${POSTGRES_PASSWORD:-pgpassword}@db:5432/postgres
+      PGPASSWORD: ${POSTGRES_PASSWORD:-pgpassword}
+    depends_on:
+      db:
+        condition: service_healthy
+    command:
+      - /bin/sh
+      - -c
+      - |
+        echo Alembic update: $$ALEMBIC_UPDATE > /dev/stderr
+        if [ "$$ALEMBIC_UPDATE" = "1" ]
+        then
+            psql -h db -U postgres -d postgres \
+                -c "CREATE SCHEMA IF NOT EXISTS \"cdb\";"
+            for instrument in latiss lsstcomcam lsstcomcamsim
+            do
+                echo Alembic update for $$instrument > /dev/stderr
+                psql -h db -U postgres -d postgres \
+                    -c "CREATE SCHEMA IF NOT EXISTS \"cdb_$$instrument\";"
+                alembic -n $$instrument upgrade head >&2
+            done
+        fi
+        echo Alembic finished > /dev/stderr
+        touch finished.txt
+        while (true); do sleep 3600; done
+    healthcheck:
+      test: ["CMD-SHELL", "test -f finished.txt"]
+      interval: 30s
+      timeout: 10s
+      retries: 100
+
+
+  hinfo_latiss:
+    # This service loads header files from LATISS into the database.
+    # To use it, set the environment variable LATISS_YAML to a directory
+    # tree containing YAML files to be loaded into the database by hinfo.
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.hinfo
+    environment:
+      - POSTGRES_URL=postgresql://postgres:${POSTGRES_PASSWORD:-pgpassword}@db:5432/postgres
+      - INSTRUMENT=LATISS
+      - LATISS_YAML=${LATISS_YAML}
+    volumes:
+      - ${LATISS_YAML:-/dev/null}:/yaml
+    depends_on:
+      alembic:
+        condition: service_healthy
+    entrypoint:
+      - bash
+      - -c
+      - |
+        if [ ! -z "$$LATISS_YAML" ]
+        then
+            source /opt/lsst/software/stack/loadLSST.bash
+            setup obs_lsst
+            echo hinfo $$LATISS_YAML > /dev/stderr
+            python -m lsst.consdb.hinfo /yaml >& 2
+        fi
+
+  hinfo_comcam:
+    # This service loads header files from LATISS into the database.
+    # To use it, set the environment variable LATISS_YAML to a directory
+    # tree containing YAML files to be loaded into the database by hinfo.
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.hinfo
+    environment:
+      - POSTGRES_URL=postgresql://postgres:${POSTGRES_PASSWORD:-pgpassword}@db:5432/postgres
+      - INSTRUMENT=LSSTComCam
+      - COMCAM_YAML=${COMCAM_YAML}
+    volumes:
+      - ${COMCAM_YAML:-/dev/null}:/yaml
+    depends_on:
+      alembic:
+        condition: service_healthy
+    entrypoint:
+      - bash
+      - -c
+      - |
+        if [ ! -z "$$COMCAM_YAML" ]
+        then
+            source /opt/lsst/software/stack/loadLSST.bash
+            setup obs_lsst
+            echo hinfo $$COMCAM_YAML > /dev/stderr
+            python -m lsst.consdb.hinfo /yaml >& 2
+        fi
+
+  app:
+    # The main instance of the pqserver app
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.pqserver
+    ports:
+      - 8888:8080
+    environment:
+      - POSTGRES_URL=postgresql://postgres:${POSTGRES_PASSWORD:-pgpassword}@db:5432/postgres
+    depends_on:
+      alembic:
+        condition: service_healthy
+
+  refapp:
+    # A reference instance of the pqserver app for comparison purposes
+    build:
+      context: ${REFERENCE_PQSERV:-../../consdb}
+      dockerfile: docker/Dockerfile.pqserver
+    ports:
+      - 8889:8080
+    environment:
+      - POSTGRES_URL=postgresql://postgres:${POSTGRES_PASSWORD:-pgpassword}@db:5432/postgres
+    depends_on:
+      alembic:
+        condition: service_healthy

python/lsst/consdb/summarize_efd.py

@@ -20,7 +20,7 @@ class Summary:
 
 # TODO add all summarizing functions
 def gen_mean(
-    config: dict[str, Any]
+    config: dict[str, Any],
 ) -> Callable[[pandas.DataFrame, astropy.time.Time, astropy.time.Time], Summary]:
     def do(series: pandas.DataFrame, start: astropy.time.Time, end: astropy.time.Time) -> Summary:
         return Summary()