Feat identifier cols

.github/workflows/ci.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
       - name: Run pre-commit
         uses: pre-commit/[email protected]
       - name: Lint code base
@@ -29,6 +29,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           FILTER_REGEX_EXCLUDE: "gradlew"
           VALIDATE_JAVA: false
+          VALIDATE_SQLFLUFF: false
       - name: Run Trivy vulnerability scanner in repo mode
         uses: aquasecurity/trivy-action@master
         with:
@@ -39,7 +40,7 @@ jobs:
           output: "trivy-results.sarif"
           severity: "CRITICAL"
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: "trivy-results.sarif"
   build:
@@ -48,7 +49,7 @@ jobs:
     needs: lint
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v3
@@ -75,7 +76,7 @@ jobs:
           username: "robot$miracum-etl+github-actions"
           password: ${{ secrets.MIRACUM_HARBOR_ETL_TOKEN }}
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -93,7 +94,7 @@ jobs:
           IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
@@ -145,11 +146,11 @@ jobs:
     if: ${{ github.event_name != 'pull_request' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Setup Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: 14
       - name: Install semantic release

Dockerfile

@@ -1,4 +1,4 @@
-FROM gradle:7.3-jdk17 AS build
+FROM gradle:7.5.1-jdk17 AS build
 WORKDIR /home/gradle/src
 ENV GRADLE_USER_HOME /gradle
 
@@ -24,4 +24,4 @@ USER 65532
 ARG VERSION=0.0.0
 ENV APP_VERSION=${VERSION} \
     SPRING_PROFILES_ACTIVE="prod"
-ENTRYPOINT ["java", "-XX:MaxRAMPercentage=85", "org.springframework.boot.loader.JarLauncher"]
+ENTRYPOINT ["java", "-XX:MaxRAMPercentage=50", "org.springframework.boot.loader.JarLauncher"]

README.md

@@ -96,9 +96,11 @@ partitioning to split the stored resources by type. Run the following **before**
 FHIR-Gateway to create the `resources` table with partitions for the most common resource types:
 
 ```postgresql
-CREATE TABLE resources
+CREATE TABLE IF NOT EXISTS resources
 (
-    id              serial,
+    id              bigserial PRIMARY KEY,
+    patient_id      varchar(64) NOT NULL,
+    encounter_id    varchar(64) NOT NULL,
     fhir_id         varchar(64) NOT NULL,
     type            varchar(64) NOT NULL,
     data            jsonb       NOT NULL,
@@ -118,9 +120,12 @@ CREATE TABLE resources_medication_administration PARTITION OF resources FOR VALU
 CREATE TABLE resources_procedure PARTITION OF resources FOR VALUES IN ('Procedure');
 CREATE TABLE resources_others PARTITION OF resources DEFAULT;
 
-CREATE INDEX resource_id_idx ON resources (id);
-CREATE INDEX resource_type_idx ON resources (type);
-CREATE INDEX last_updated_at_idx ON resources (last_updated_at DESC);
+CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS resource_id_idx ON resources (id);
+CREATE INDEX IF NOT EXISTS resource_type_idx ON resources (type);
+CREATE INDEX IF NOT EXISTS last_updated_at_idx ON resources (last_updated_at DESC);
+
+CREATE INDEX IF NOT EXISTS resource_pat_id_idx ON resources (patient_id);
+CREATE INDEX IF NOT EXISTS resource_enc_id_idx ON resources (encounter_id);
 ```
 
 Be sure to set `SPRING_SQL_INIT_MODE=never` before starting the FHIR GW.

build.gradle

@@ -1,9 +1,9 @@
 plugins {
-    id "org.springframework.boot" version "2.5.6"
-    id "io.spring.dependency-management" version "1.0.11.RELEASE"
+    id "org.springframework.boot" version "2.7.3"
+    id "io.spring.dependency-management" version "1.0.13.RELEASE"
     id "java"
     id "jacoco"
-    id "com.diffplug.spotless" version "6.0.4"
+    id "com.diffplug.spotless" version "6.9.1"
 }
 
 group = "org.miracum.etl"
@@ -22,10 +22,7 @@ configurations {
 }
 
 ext {
-    set("springCloudVersion", "2020.0.4")
-    // Mitigation for CVE-2021-44228
-    // https://spring.io/blog/2021/12/10/log4j2-vulnerability-and-spring-boot
-    set("log4j2.version", "2.16.0")
+    set("springCloudVersion", "2021.0.3")
 }
 
 repositories {
@@ -45,26 +42,26 @@ dependencies {
 
     implementation "org.miracum:kafka-fhir-serializer:1.0.2"
 
-    implementation "io.micrometer:micrometer-registry-prometheus:1.8.1"
-    implementation "io.micrometer:micrometer-core:1.8.1"
+    implementation "io.micrometer:micrometer-registry-prometheus:1.9.3"
+    implementation "io.micrometer:micrometer-core:1.9.3"
 
-    implementation "net.logstash.logback:logstash-logback-encoder:7.0.1"
+    implementation "net.logstash.logback:logstash-logback-encoder:7.2"
 
-    implementation "ca.uhn.hapi.fhir:hapi-fhir-base:5.6.1"
-    implementation "ca.uhn.hapi.fhir:hapi-fhir-client:5.6.1"
-    implementation "ca.uhn.hapi.fhir:org.hl7.fhir.r4:5.6.15"
-    implementation "ca.uhn.hapi.fhir:hapi-fhir-structures-r4:5.6.1"
-    implementation "ca.uhn.hapi.fhir:hapi-fhir-client-okhttp:5.6.1"
+    implementation "ca.uhn.hapi.fhir:hapi-fhir-base:6.1.0"
+    implementation "ca.uhn.hapi.fhir:hapi-fhir-client:6.1.0"
+    implementation "ca.uhn.hapi.fhir:org.hl7.fhir.r4:5.6.54"
+    implementation "ca.uhn.hapi.fhir:hapi-fhir-structures-r4:5.7.9"
+    implementation "ca.uhn.hapi.fhir:hapi-fhir-client-okhttp:5.7.9"
 
     implementation "io.opentracing.contrib:opentracing-spring-web-starter:4.1.0"
     implementation "io.opentracing.contrib:opentracing-spring-jaeger-web-starter:3.3.1"
     implementation "io.opentracing.contrib:opentracing-spring-jaeger-cloud-starter:3.3.1"
     implementation "io.opentracing.contrib:opentracing-okhttp3:3.0.0"
-    implementation "io.jaegertracing:jaeger-client:1.6.0"
+    implementation "io.jaegertracing:jaeger-client:1.8.1"
 
     developmentOnly "org.springframework.boot:spring-boot-devtools"
-    runtimeOnly "org.postgresql:postgresql:42.3.1"
-    runtimeOnly "com.h2database:h2:2.0.202"
+    runtimeOnly "org.postgresql:postgresql:42.4.2"
+    runtimeOnly "com.h2database:h2:2.1.214"
     annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
     testImplementation "org.springframework.boot:spring-boot-starter-test"
 }

deploy/docker-compose.dev.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   mock-data-loader:
-    image: confluentinc/cp-kafkacat:7.0.1
+    image: confluentinc/cp-kafkacat:7.1.3
     entrypoint: [ "/bin/bash", "-c" ]
     command: >
       "while true; do
@@ -17,7 +17,7 @@ services:
       - kafka1
 
   kafka1:
-    image: docker.vectorized.io/vectorized/redpanda:v21.11.2
+    image: docker.vectorized.io/vectorized/redpanda:v22.2.1
     container_name: kafka1
     command:
       - redpanda
@@ -39,7 +39,7 @@ services:
       - 9644:9644
 
   akhq:
-    image: tchiotludo/akhq:0.19.0
+    image: tchiotludo/akhq:0.22.0
     environment:
       AKHQ_CONFIGURATION: |
         akhq:

deploy/docker-compose.gw-deps.yml

@@ -2,29 +2,30 @@ version: "3.9"
 
 services:
   jaeger:
-    image: jaegertracing/all-in-one:1.29
+    image: jaegertracing/all-in-one:1.37
 
   fhir-pseudonymizer:
     image: harbor.miracum.org/miracum-etl/fhir-pseudonymizer:v1.6.0
     environment:
       JAEGER__HOST: jaeger
       JAEGER__PORT: 6831
       GPAS__URL: ${GPAS_URL:-http://gpas:8080/gpas/gpasService}
+      GPAS__VERSION: ${GPAS_VERSION:-1.11.0}
     volumes:
       - ./anonymization.yaml:/etc/anonymization.yaml:ro
     depends_on:
       - jaeger
       - gpas
 
   loinc-converter:
-    image: harbor.miracum.org/miracum-etl/loinc-conversion:v1.13.1
+    image: harbor.miracum.org/miracum-etl/loinc-conversion:v1.13.3
     environment:
       JAEGER_AGENT_HOST: jaeger
     depends_on:
       - jaeger
 
   fhir-db:
-    image: postgres:14.1
+    image: postgres:14.5
     environment:
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: fhir
@@ -44,7 +45,7 @@ services:
     image: docker.io/hapiproject/hapi:v5.6.0-distroless
 
   gpasinit-patient:
-    image: curlimages/curl:7.80.0
+    image: curlimages/curl:7.84.0
     command: |
       -X POST
       -H 'Content-Type:application/xml'
@@ -73,7 +74,7 @@ services:
       - gpas
 
   gpasinit-fall:
-    image: curlimages/curl:7.80.0
+    image: curlimages/curl:7.84.0
     command: |
       -X POST
       -H 'Content-Type:application/xml'

deploy/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   gateway:
-    image: ${FHIR_GATEWAY_IMAGE_NAME:-ghcr.io/miracum/fhir-gateway:v3.10.2}
+    image: ${FHIR_GATEWAY_IMAGE_NAME:-ghcr.io/miracum/fhir-gateway:v3.10.3}
     restart: on-failure
     environment:
       SPRING_DATASOURCE_URL: ${SPRING_DATASOURCE_URL:-jdbc:postgresql://fhir-db:5432/fhir}

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -205,6 +205,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,7 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

src/main/java/org/miracum/etl/fhirgateway/stores/PostgresFhirResourceRepository.java

@@ -83,6 +83,18 @@ private int insertResources(Bundle bundle) {
             .map(
                 resource ->
                     new Object[] {
+                      // TODO add encounter id / patient id / encounter_start date here
+                      // this would depend also on the resources, e.g.
+                      // type = "Patient": Encounter-ID: NULL, Patient-ID: Patient.id
+                      // type = "Encounter": Encounter-ID: Encounter.id, Patient.id:
+                      // REPLACE(jsonb_path_query(Encounter.DATA, '$.subject') ->> 'reference',
+                      // 'Patient/', '')
+                      // type = "Condition": Encounter-ID: REPLACE(Condition.DATA -> 'encounter' ->>
+                      // 'reference', 'Encounter/', '')
+                      // type = "Procedure": Encounter-ID: REPLACE(Procedure.DATA -> 'encounter' ->>
+                      // 'reference', 'Encounter/', '')
+                      // type = "Observation": Encounter-ID: REPLACE(Observation.DATA -> 'encounter'
+                      // ->> 'reference', 'Encounter/', '')
                       resource.getIdElement().getIdPart(),
                       resource.fhirType(),
                       fhirParser.encodeResourceToString(resource)

src/main/resources/schema.sql

@@ -1,15 +1,30 @@
 CREATE TABLE IF NOT EXISTS resources
 (
     id              bigserial PRIMARY KEY,
+    patient_id      varchar(64) NOT NULL,
+    encounter_id    varchar(64) NOT NULL,
     fhir_id         varchar(64) NOT NULL,
     type            varchar(64) NOT NULL,
     data            jsonb       NOT NULL,
     created_at      timestamp   NOT NULL DEFAULT NOW(),
     last_updated_at timestamp   NOT NULL DEFAULT NOW(),
     is_deleted      boolean     NOT NULL DEFAULT FALSE,
     CONSTRAINT fhir_id_unique UNIQUE (fhir_id, type)
-);
+) PARTITION BY LIST (type);
+
+CREATE TABLE resources_patient PARTITION OF resources FOR VALUES IN ('Patient');
+CREATE TABLE resources_encounter PARTITION OF resources FOR VALUES IN ('Encounter');
+CREATE TABLE resources_condition PARTITION OF resources FOR VALUES IN ('Condition');
+CREATE TABLE resources_observation PARTITION OF resources FOR VALUES IN ('Observation');
+CREATE TABLE resources_medication PARTITION OF resources FOR VALUES IN ('Medication');
+CREATE TABLE resources_medication_statement PARTITION OF resources FOR VALUES IN ('MedicationStatement');
+CREATE TABLE resources_medication_administration PARTITION OF resources FOR VALUES IN ('MedicationAdministration');
+CREATE TABLE resources_procedure PARTITION OF resources FOR VALUES IN ('Procedure');
+CREATE TABLE resources_others PARTITION OF resources DEFAULT;
 
 CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS resource_id_idx ON resources (id);
 CREATE INDEX IF NOT EXISTS resource_type_idx ON resources (type);
 CREATE INDEX IF NOT EXISTS last_updated_at_idx ON resources (last_updated_at DESC);
+
+CREATE INDEX IF NOT EXISTS resource_pat_id_idx ON resources (patient_id);
+CREATE INDEX IF NOT EXISTS resource_enc_id_idx ON resources (encounter_id);

src/test/resources/application-test.yml

@@ -4,8 +4,8 @@ spring:
     url: jdbc:h2:mem:db;DB_CLOSE_DELAY=-1
     username: sa
     password: sa
-    # disabled initialization for now as H2 won't work with the JSONB data type
-    initialization-mode: never
+  # disabled initialization for now as H2 won't work with the JSONB data type
+  sql.init.mode: never
 services:
   gpas:
     enabled: false