feat: update orioledb base image and s3 config script (#818)

ansible/files/postgresql_config/supautils.conf.j2

@@ -1,7 +1,7 @@
 # full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_prewarm, pg_surgery, pg_visibility, pgstattuple
 # omitted because deprecated:                intagg, xml2
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, orioledb'
 supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'postgres'

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.0.143"
+postgres-version = "15.1.0.144"

docker/orioledb/Dockerfile

@@ -41,7 +41,7 @@ ARG wal_g_release=2.0.1
 ####################
 # Install postgres
 ####################
-FROM orioledb/orioledb:beta3-s3-pg${postgresql_major}-ubuntu as base
+FROM orioledb/orioledb:latest-pg${postgresql_major}-ubuntu as base
 # Redeclare args for use in subsequent stages
 ARG TARGETARCH
 ARG postgresql_major
@@ -77,10 +77,10 @@ ARG CACHE_EPOCH
 FROM builder as rust-toolchain
 ENV PATH=/root/.cargo/bin:$PATH
 RUN apt-get update && apt-get install -y --no-install-recommends curl pkg-config && \
-  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \
-  rustup --version && \
-  rustc --version && \
-  cargo --version 
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \
+    rustup --version && \
+    rustc --version && \
+    cargo --version
 
 RUN cargo install cargo-pgrx --version 0.10.2 --locked
 RUN cargo pgrx init --pg${postgresql_major} $(which pg_config)
@@ -321,27 +321,11 @@ RUN tar -xvf /tmp/timescaledb.tar.gz -C /tmp && \
 # Build from source
 WORKDIR /tmp/timescaledb-${timescaledb_release}/build
 RUN cmake -DAPACHE_ONLY=1 ..
-# error: too few arguments to function ‘table_multi_insert’
+# error: too few arguments to function ‘table_tuple_update’
+# error: too few arguments to function ‘table_tuple_delete’
 RUN sed -i \
-    -e "358s|^|        bool *insert_indexes = NULL;|g" \
-    -e "364s|);|, insert_indexes);|g" \
-    -e "930s|^|                bool *insert_indexes = NULL;|g" \
-    -e "1115s|);|, insert_indexes);|g" \
-    -e "1134s|);|, insert_indexes);|g" \
-    ../src/copy.c
-# error: static declaration of ‘replace_nestloop_params’ follows non-static declaration
-RUN sed -i \
-    -e "48s|static ||g" \
-    -e "943s|static ||g" \
-    ../src/import/planner.c
-# error: too few arguments to function ‘table_tuple_insert’
-RUN sed -i \
-    -e "1672s|*/|*/ bool *insert_indexes = NULL;|g" \
-    -e "1673s|);|, insert_indexes);|g" \
-    -e "1971s|*/|*/ bool *insert_indexes = NULL;|g" \
-    -e "1981s|);|, insert_indexes);|g" \
-    -e "2559s|}|} bool *insert_indexes = NULL;|g" \
-    -e "2567s|);|, insert_indexes);|g" \
+    -e "1981s|);|, NULL);|g" \
+    -e "2567s|);|, NULL);|g" \
     ../src/nodes/hypertable_modify.c
 RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccache \
     make -j$(nproc)
@@ -562,7 +546,7 @@ FROM rust-toolchain as pg_graphql-source
 ARG pg_graphql_release
 ARG pg_graphql_release_checksum
 ADD --checksum=${pg_graphql_release_checksum} \
-  "https://github.com/supabase/pg_graphql/archive/refs/tags/v${pg_graphql_release}.tar.gz" \
+    "https://github.com/supabase/pg_graphql/archive/refs/tags/v${pg_graphql_release}.tar.gz" \
     /tmp/pg_graphql.tar.gz
 RUN tar -xvf /tmp/pg_graphql.tar.gz -C /tmp && \
     rm -rf /tmp/pg_graphql.tar.gz
@@ -642,7 +626,7 @@ FROM rust-toolchain as pg_jsonschema-source
 ARG pg_jsonschema_release
 ARG pg_jsonschema_release_checksum
 ADD --checksum=${pg_jsonschema_release_checksum} \
-  "https://github.com/supabase/pg_jsonschema/archive/refs/tags/v${pg_jsonschema_release}.tar.gz" \
+    "https://github.com/supabase/pg_jsonschema/archive/refs/tags/v${pg_jsonschema_release}.tar.gz" \
     /tmp/pg_jsonschema.tar.gz
 RUN tar -xvf /tmp/pg_jsonschema.tar.gz -C /tmp && \
     rm -rf /tmp/pg_jsonschema.tar.gz
@@ -768,7 +752,7 @@ FROM rust-toolchain as wrappers-source
 ARG wrappers_release
 ARG wrappers_release_checksum
 ADD --checksum=${wrappers_release_checksum} \
-  "https://github.com/supabase/wrappers/archive/refs/tags/v${wrappers_release}.tar.gz" \
+    "https://github.com/supabase/wrappers/archive/refs/tags/v${wrappers_release}.tar.gz" \
     /tmp/wrappers.tar.gz
 RUN tar -xvf /tmp/wrappers.tar.gz -C /tmp && \
     rm -rf /tmp/wrappers.tar.gz
@@ -1023,6 +1007,7 @@ COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_
 RUN sed -i \
     -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
     -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
+    -e "s|shared_preload_libraries = '\(.*\)'|shared_preload_libraries = '\1, orioledb'|" \
     -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
     -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
     echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
@@ -1044,6 +1029,10 @@ RUN sed -i \
     -e "s|PGHOST= PGHOSTADDR=|PGHOST=\$POSTGRES_HOST|g" \
     /usr/local/bin/docker-entrypoint.sh
 
+# TODO: support s3 credentials once upstream is tested
+# COPY docker/orioledb/entrypoint.sh /
+# ENTRYPOINT ["/entrypoint.sh"]
+
 HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost
 STOPSIGNAL SIGINT
 EXPOSE 5432

docker/orioledb/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -eou pipefail
+
+PG_CONF=/etc/postgresql/postgresql.conf
+
+function configure_orioledb {
+  echo "Configuring OrioleDB..."
+
+  sed -i \
+    -e "s|#max_worker_processes = .*|max_worker_processes = 50 # should fit orioledb.s3_num_workers as long as other workers|" \
+    -e "s|#log_min_messages = .*|log_min_messages = debug1 # will log all S3 requests|" \
+    -e "s|#archive_mode = off\(.*\)|archive_mode = on\1|" \
+    "$PG_CONF"
+
+  echo "
+archive_library = 'orioledb'
+orioledb.main_buffers = 1GB
+orioledb.undo_buffers = 256MB
+orioledb.s3_num_workers = 20 # should be enough for comfortable work
+orioledb.s3_mode = true
+orioledb.s3_host = '$S3_HOST' # replace with your bucket URL, accelerated buckets are recommended
+orioledb.s3_region = '$S3_REGION' # replace with your S3 region
+orioledb.s3_accesskey = '$S3_ACCESS_KEY' # replace with your S3 key
+orioledb.s3_secretkey = '$S3_SECRET_KEY' # replace with your S3 secret key
+" >> "$PG_CONF"
+}
+
+if ! grep -q orioledb "$PG_CONF"; then
+  configure_orioledb
+fi
+
+docker-entrypoint.sh "$@"