fix: Load httpfs & Store DuckDB extensions in PGDATA (#200)

* try to load httpfs

* Make read-only

* Fix

* try this

* Set extension path to PGDATA dir

* rm chmod and codecov

* also rm above

* cargo fmt

---------

Co-authored-by: Ming Ying <[email protected]>

.github/workflows/test-pg_analytics.yml

@@ -149,11 +149,6 @@ jobs:
           LLVM_PROFILE_FILE: target/coverage/pg_analytics-%p-%m.profraw
           RUST_BACKTRACE: full
         run: |
-          echo ""
-          echo "Enabling code coverage..."
-          echo -e "\n# Enable code coverage on Linux only, for CI builds\n[target.'cfg(target_os=\"linux\")']\nrustflags = [\"-Cinstrument-coverage\"]" >> .cargo/config.toml
-          mkdir -p target/coverage target/coverage-report
-
           echo ""
           echo "Running Rust tests..."
           export DATABASE_URL=postgresql://localhost:288${{ matrix.pg_version }}/postgres
@@ -267,11 +262,6 @@ jobs:
           LLVM_PROFILE_FILE: target/coverage/pg_analytics-%p-%m.profraw
           RUST_BACKTRACE: full
         run: |
-          echo ""
-          echo "Enabling code coverage..."
-          echo -e "\n# Enable code coverage on Linux only, for CI builds\n[target.'cfg(target_os=\"linux\")']\nrustflags = [\"-Cinstrument-coverage\"]" >> .cargo/config.toml
-          mkdir -p target/coverage target/coverage-report
-
           echo ""
           echo "Running Rust tests..."
           export DATABASE_URL=postgresql://localhost:288${{ matrix.pg_version }}/postgres

src/duckdb/connection.rs

@@ -255,6 +255,16 @@ pub fn set_search_path(search_path: Vec<String>) -> Result<()> {
     Ok(())
 }
 
+pub fn set_duckdb_extension_directory(extension_directory_path: &str) -> Result<()> {
+    // Set duckdb extension directory
+    execute(
+        format!("SET extension_directory = '{extension_directory_path}'").as_str(),
+        [],
+    )?;
+
+    Ok(())
+}
+
 pub fn execute_explain(query: &str) -> Result<String> {
     let conn = unsafe { &*get_global_connection().get() };
     let mut stmt = conn.prepare(query)?;
@@ -272,3 +282,11 @@ pub fn execute_explain(query: &str) -> Result<String> {
 
     Ok(rows.join(""))
 }
+
+pub fn install_httpfs() -> Result<()> {
+    if !check_extension_loaded("httpfs")? {
+        execute("INSTALL httpfs", [])?;
+        execute("LOAD httpfs", [])?;
+    }
+    Ok(())
+}

src/fdw/base.rs

@@ -19,13 +19,16 @@ use anyhow::{anyhow, bail, Result};
 use duckdb::arrow::array::RecordBatch;
 use pgrx::*;
 use std::collections::HashMap;
+use std::ffi::CStr;
 use strum::IntoEnumIterator;
 use supabase_wrappers::prelude::*;
 use thiserror::Error;
 
 use super::handler::FdwHandler;
 use crate::duckdb::connection;
+use crate::fdw::base::connection::set_duckdb_extension_directory;
 use crate::schema::cell::*;
+
 #[cfg(debug_assertions)]
 use crate::DEBUG_GUCS;
 
@@ -223,6 +226,18 @@ pub fn register_duckdb_view(
         connection::create_secret(DEFAULT_SECRET, user_mapping_options)?;
     }
 
+    // In CloudNativePG, the filesystem is read-only. To work around this, we force
+    // DuckDB to install its extensions in the PGDATA directory, which is writable.
+    let data_dir = unsafe {
+        CStr::from_ptr(pgrx::pg_sys::DataDir)
+            .to_str()
+            .map_err(|e| anyhow::anyhow!("Failed to convert DataDir to &str: {}", e))?
+    };
+    let _ = set_duckdb_extension_directory(data_dir);
+
+    // duckdb-rs stopped bundling in httpfs, so we need to load it ourselves
+    connection::install_httpfs()?;
+
     if !connection::view_exists(table_name, schema_name)? {
         // Initialize DuckDB view
         connection::execute(