Adopt latest changes from optout and core

IABTechLab · Dec 11, 2023 · a67f9d4 · a67f9d4
1 parent 508f09f
commit a67f9d4
Show file tree

Hide file tree

Showing 10 changed files with 137 additions and 12 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -82,8 +82,8 @@ jobs:
       - name: Bring up docker compose
         id: docker-compose
         env:
-          CORE_ROOT: '../'
-          OPTOUT_ROOT: '../'
+          CORE_ROOT: '..'
+          OPTOUT_ROOT: '..'
           CORE_VERSION: ${{ inputs.core_image_version }}
           OPTOUT_VERSION: ${{ inputs.optout_image_version }}
           OPERATOR_VERSION: ${{ inputs.operator_image_version }}

diff --git a/conf/core/default-config.json b/conf/core/default-config.json
@@ -0,0 +1,20 @@
+{
+    "pre_signed_url_expiry": 1800,
+    "core_s3_bucket": null,
+    "aws_region": null,
+    "aws_access_key_id": null,
+    "aws_secret_access_key": null,
+    "sites_metadata_path": null,
+    "clients_metadata_path": null,
+    "client_side_keypairs_metadata_path": null,
+    "services_metadata_path": null,
+    "service_links_metadata_path": null,
+    "keys_metadata_path": null,
+    "salts_metadata_path": null,
+    "operators_metadata_path": null,
+    "enclaves_metadata_path": null,
+    "partners_metadata_path": null,
+    "att_token_enc_key": null,
+    "att_token_enc_salt": null,
+    "enforceJwt": false
+}
diff --git a/conf/core/local-e2e-docker-config.json b/conf/core/local-e2e-docker-config.json
@@ -3,7 +3,6 @@
   "storage_mock": false,
   "aws_s3_endpoint": "http://localstack:5001",
   "core_s3_bucket": "test-core-bucket",
-  "pre_signed_url_expiry": 1800,
   "aws_region": "us-east-2",
   "aws_access_key_id": "no access key needed for test",
   "aws_secret_access_key": "no secret key needed for test",

diff --git a/conf/core/logback.loki-e2e.xml b/conf/core/logback.loki-e2e.xml
@@ -0,0 +1,28 @@
+<configuration>
+    <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
+        <http>
+            <url>http://loki:3100/loki/api/v1/push</url>
+        </http>
+        <format>
+            <label>
+                <pattern>app=uid2-core,host=${HOSTNAME},portoffset=${port_offset:-0},level=%level</pattern>
+            </label>
+            <message>
+                <pattern>l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex</pattern>
+            </message>
+            <sortByTime>true</sortByTime>
+        </format>
+    </appender>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="DEBUG">
+        <appender-ref ref="LOKI" />
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>
diff --git a/conf/optout/default-config.json b/conf/optout/default-config.json
@@ -0,0 +1,34 @@
+{
+  "service_verbose": true,
+  "service_instances": 4,
+  "core_attest_url": null,
+  "core_api_token": null,
+  "att_token_enc_key": null,
+  "att_token_enc_salt": null,
+  "optout_internal_api_token": null,
+  "optout_observe_only": false,
+  "optout_s3_path_compat": false,
+  "optout_partner_endpoint_mock": false,
+  "optout_add_entry_timeout_ms": 1000,
+  "optout_data_dir": "/opt/uid2/optout/",
+  "optout_replica_uris": null,
+  "optout_producer_replica_id": -1,
+  "optout_producer_replica_id_offset": 0,
+  "optout_producer_max_replicas": 3,
+  "optout_delta_rotate_interval": 300,
+  "optout_delta_backtrack_in_days": 1,
+  "optout_partition_interval": 86400,
+  "optout_max_partitions": 90,
+  "optout_producer_buffer_size": 65536,
+  "optout_sender_replica_id": 0,
+  "optout_delete_expired": true,
+  "optout_s3_bucket": null,
+  "optout_s3_folder": null,
+  "cloud_download_threads": 8,
+  "cloud_upload_threads": 2,
+  "cloud_refresh_interval": 60,
+  "clients_metadata_path": null,
+  "operators_metadata_path": null,
+  "partners_metadata_path": null,
+  "partners_config_path": "partners/config.json"
+}
diff --git a/conf/optout/local-e2e-docker-config.json b/conf/optout/local-e2e-docker-config.json
@@ -1,7 +1,6 @@
 {
   "service_instances": 1,
   "storage_mock": false,
-  "enforce_https": false,
   "aws_s3_endpoint": "http://localstack:5001",
   "optout_s3_bucket": "test-optout-bucket",
   "pre_signed_url_expiry": 1800,
@@ -10,12 +9,9 @@
   "aws_secret_access_key": "no secret key needed for test",
   "att_token_enc_key": "<key-for-attestation-token>",
   "att_token_enc_salt": "<salt-for-attestation-token>",
-  "optout_producer_max_replicas": 1,
-  "optout_s3_folder": "optout-v2/",
+  "optout_s3_folder": "optout/",
   "optout_internal_api_token": "test-optout-internal-key",
-  "optout_replica_uris": "http://localhost:8081/optout/write,http://localhost:8081/optout/write,http://localhost:8081/optout/write",
-  "optout_producer_replica_id": 0,
-  "optout_delta_rotate_interval": 60,
+  "optout_replica_uris": null,
   "partners_metadata_path": "http://core:8088/partners/refresh",
   "operators_metadata_path": "http://core:8088/operators/refresh",
   "core_attest_url": "http://core:8088/attest",
@@ -25,4 +21,4 @@
   "enforceJwt": false,
   "core_public_url": "http://localhost:8088",
   "optout_url": "http://localhost:8090"
-}
+}
diff --git a/conf/optout/logback.loki.xml b/conf/optout/logback.loki.xml
@@ -0,0 +1,28 @@
+<configuration>
+    <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
+        <http>
+            <url>http://loki:3100/loki/api/v1/push</url>
+        </http>
+        <format>
+            <label>
+                <pattern>app=uid2-optout,host=${HOSTNAME},portoffset=${port_offset:-0},level=%level</pattern>
+            </label>
+            <message>
+                <pattern>l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex</pattern>
+            </message>
+            <sortByTime>true</sortByTime>
+        </format>
+    </appender>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="DEBUG">
+        <appender-ref ref="LOKI" />
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>
diff --git a/conf/optout/run_tool_local_e2e.sh b/conf/optout/run_tool_local_e2e.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+java -Djava.security.egd=file:/dev/./urandom \
+    -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
+    -Dlogback.configurationFile=./conf/logback.xml \
+    -cp uid2-optout.jar \
+    com.uid2.optout.tool.OptOutLogTool \
+    $*
diff --git a/e2e/docker-build-public.sh b/e2e/docker-build-public.sh
@@ -16,8 +16,15 @@ if [ -z "$OPTOUT_ROOT" ]; then
   exit 1
 fi
 
-mkdir -p "$CORE_CONFIG_FILE_DIR" && cp "$CORE_ROOT/conf/core/local-e2e-docker-config.json" "$CORE_CONFIG_FILE_DIR"
-mkdir -p "$OPTOUT_CONFIG_FILE_DIR" && cp "$OPTOUT_ROOT/conf/optout/local-e2e-docker-config.json" "$OPTOUT_CONFIG_FILE_DIR"
+mkdir -p "$CORE_CONFIG_FILE_DIR"
+cp "$CORE_ROOT/conf/core/default-config.json" "$CORE_CONFIG_FILE_DIR"
+cp "$CORE_ROOT/conf/core/local-e2e-docker-config.json" "$CORE_CONFIG_FILE_DIR"
+cp "$CORE_ROOT/conf/core/logback.loki-e2e.xml" "$CORE_CONFIG_FILE_DIR"
+mkdir -p "$OPTOUT_CONFIG_FILE_DIR"
+cp "$OPTOUT_ROOT/conf/optout/default-config.json" "$OPTOUT_CONFIG_FILE_DIR"
+cp "$OPTOUT_ROOT/conf/optout/local-e2e-docker-config.json" "$OPTOUT_CONFIG_FILE_DIR"
+cp "$OPTOUT_ROOT/conf/optout/logback.loki.xml" "$OPTOUT_CONFIG_FILE_DIR"
+cp "$OPTOUT_ROOT/conf/optout/run_tool_local_e2e.sh" "$OPTOUT_CONFIG_FILE_DIR"
 
 
 CORE_CONFIG_FILE="$ROOT/docker/uid2-core/conf/local-e2e-docker-config.json"

diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
@@ -26,7 +26,9 @@ services:
     ports:
       - "127.0.0.1:8088:8088"
     volumes:
+      - ./docker/uid2-core/conf/default-config.json:/app/conf/default-config.json
       - ./docker/uid2-core/conf/local-e2e-docker-config.json:/app/conf/local-config.json
+      - ./docker/uid2-core/conf/logback.loki-e2e.xml:/app/conf/logback.xml
     depends_on:
       localstack:
         condition: service_healthy
@@ -43,7 +45,10 @@ services:
       - "127.0.0.1:8090:8081"
       - "127.0.0.1:5090:5005"
     volumes:
+      - ./docker/uid2-optout/conf/run_tool_local_e2e.sh:/app/run_tool.sh
+      - ./docker/uid2-optout/conf/default-config.json:/app/conf/default-config.json
       - ./docker/uid2-optout/conf/local-e2e-docker-config.json:/app/conf/local-config.json
+      - ./docker/uid2-optout/conf/logback.loki.xml:/app/conf/logback.xml
       - ./docker/uid2-optout/mount/:/opt/uid2/optout/
     depends_on:
       core: