Merge pull request #15 from bmsuisse/dev

Polars 1.0

README.md

@@ -49,6 +49,13 @@ In case there is an unsupported DeltaLake Feature, this will just throw `DeltaPr
 
 For now, only az:// Url's for Azure are tested and supported in DuckDB. For polars it's a lot easier, since polars just uses `object_store` create, so it should just work.
 
+The package does some work to make DuckDB's "Azure Storage Options" work in Polars, to be able to use the same options.
+
+This means you can:
+
+- pass an absolute DuckDB-style Path to Polars, meaning something like `abfss://⟨my_storage_account⟩.dfs.core.windows.net/⟨my_filesystem⟩/⟨path⟩`
+- pass "chain" as option, which will act like [DuckDB's Credential Chain](https://duckdb.org/docs/extensions/azure.html#credential_chain-provider). This requires `azure-identity` Package
+
 ## Looking for something different? :)
 
 We also have the following projects around deltalake:

deltalake2db/duckdb.py

@@ -190,6 +190,17 @@ def apply_storage_options(
 TYPE AZURE,
 CONNECTION_STRING '{conn_str}',
 ACCOUNT_NAME '{account_name}'
+);"""
+            )
+        elif account_name is not None and "client_id" in storage_options:
+            con.execute(
+                f"""CREATE SECRET {secret_name} (
+TYPE AZURE,
+PROVIDER SERVICE_PRINCIPAL,
+TENANT_ID  '{storage_options.get("tenant_id")}',
+CLIENT_ID   '{storage_options.get("client_id")}',
+CLIENT_SECRET    '{storage_options.get("client_secret")}',
+ACCOUNT_NAME '{account_name}'
 );"""
             )
         elif account_name is not None:
@@ -217,7 +228,7 @@ def apply_storage_options(
 
 
 type_map = {
-    "byte": ex.DataType.Type.UTINYINT,
+    "byte": ex.DataType.Type.TINYINT,
     "int": ex.DataType.Type.INT,
     "integer": ex.DataType.Type.INT,
     "long": ex.DataType.Type.BIGINT,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deltalake2db"
-version = "0.4.1"
+version = "0.4.2"
 description = ""
 authors = ["Adrian Ehrsam <[email protected]>"]
 license = "MIT"
@@ -14,14 +14,14 @@ azure-identity = { version = "^1.16.0", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pyright = "^1.1.352"
-polars = { version = ">=1.0.0-rc.1", allow-prereleases = true }
+polars = "^1.0.0"
 duckdb = "^1.0.0"
 ruff = "^0.4.3"
 
 [tool.poetry.group.test.dependencies]
 pytest-cov = "^4.1.0"
 pytest = "^8.1.0"
-polars = { version = ">=1.0.0-beta.1", allow-prereleases = true }
+polars = "^1.0.0"
 duckdb = "^1.0.0"
 docker = "^7.0.0"
 azure-storage-blob = "^12.19.1"