Merge branch 'main' into map_nulls

.env

@@ -0,0 +1 @@
+FLINK_VERSION=1.18.1

.github/workflows/ibis-backends.yml

@@ -185,7 +185,7 @@ jobs:
             extras:
               - flink
             additional_deps:
-              - apache-flink
+              - "'apache-flink < 1.19.0'"
             services:
               - flink
         include:
@@ -198,7 +198,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - apache-flink
+                - "'apache-flink < 1.19.0'"
               services:
                 - flink
         exclude:
@@ -314,7 +314,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - apache-flink
+                - "'apache-flink < 1.19.0'"
               services:
                 - flink
           - os: ubuntu-latest
@@ -326,7 +326,7 @@ jobs:
               extras:
                 - flink
               additional_deps:
-                - apache-flink
+                - "'apache-flink < 1.19.0'"
               services:
                 - flink
           - os: windows-latest

.gitignore

@@ -40,7 +40,6 @@ dist
 coverage.xml
 
 # Environments
-.env
 .venv
 env/
 venv/

ci/schema/bigquery.sql

@@ -0,0 +1,164 @@
+CREATE SCHEMA IF NOT EXISTS ibis_gbq_testing;
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.struct (
+    abc STRUCT<a FLOAT64, b STRING, c INT64>
+);
+
+INSERT INTO ibis_gbq_testing.struct VALUES
+    (STRUCT(1.0, 'banana', 2)),
+    (STRUCT(2.0, 'apple', 3)),
+    (STRUCT(3.0, 'orange', 4)),
+    (STRUCT(NULL, 'banana', 2)),
+    (STRUCT(2.0, NULL, 3)),
+    (NULL),
+    (STRUCT(3.0, 'orange', NULL));
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.array_types (
+    x ARRAY<INT64>,
+    y ARRAY<STRING>,
+    z ARRAY<FLOAT64>,
+    grouper STRING,
+    scalar_column FLOAT64,
+);
+
+INSERT INTO ibis_gbq_testing.array_types VALUES
+    ([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0),
+    ([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0),
+    ([6], ['f'], [6.0], 'a', 3.0),
+    ([1], ['a'], [], 'b', 4.0),
+    ([2, 3], ['b', 'c'], NULL, 'b', 5.0),
+    ([4, 5], ['d', 'e'], [4.0, 5.0], 'c', 6.0);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.win (
+    g STRING,
+    x INT64,
+    y INT64
+);
+
+INSERT INTO ibis_gbq_testing.win VALUES
+    ('a', 0, 3),
+    ('a', 1, 2),
+    ('a', 2, 0),
+    ('a', 3, 1),
+    ('a', 4, 1);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.topk (
+    x INT64
+);
+
+INSERT INTO ibis_gbq_testing.topk VALUES (1), (1), (NULL);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.numeric_table (
+    string_col STRING,
+    numeric_col NUMERIC
+);
+
+INSERT INTO ibis_gbq_testing.numeric_table VALUES
+    ('1st value', 0.999999999),
+    ('2nd value', 0.000000002);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.json_t (
+    js JSON
+);
+
+INSERT INTO ibis_gbq_testing.json_t VALUES
+    (JSON '{"a": [1,2,3,4], "b": 1}'),
+    (JSON '{"a":null,"b":2}'),
+    (JSON '{"a":"foo", "c":null}'),
+    (JSON 'null'),
+    (JSON '[42,47,55]'),
+    (JSON '[]');
+
+
+LOAD DATA OVERWRITE ibis_gbq_testing.functional_alltypes (
+    id INT64,
+    bool_col BOOLEAN,
+    tinyint_col INT64,
+    smallint_col INT64,
+    int_col INT64,
+    bigint_col INT64,
+    float_col FLOAT64,
+    double_col FLOAT64,
+    date_string_col STRING,
+    string_col STRING,
+    timestamp_col DATETIME,
+    year INT64,
+    month INT64
+)
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/functional_alltypes.parquet']
+);
+
+LOAD DATA OVERWRITE ibis_gbq_testing.awards_players
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/awards_players.parquet']
+);
+
+LOAD DATA OVERWRITE ibis_gbq_testing.batting
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/batting.parquet']
+);
+
+LOAD DATA OVERWRITE ibis_gbq_testing.diamonds
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/diamonds.parquet']
+);
+
+LOAD DATA OVERWRITE ibis_gbq_testing.astronauts
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/astronauts.parquet']
+);
+
+LOAD DATA OVERWRITE ibis_gbq_testing.functional_alltypes_parted (
+    id INT64,
+    bool_col BOOLEAN,
+    tinyint_col INT64,
+    smallint_col INT64,
+    int_col INT64,
+    bigint_col INT64,
+    float_col FLOAT64,
+    double_col FLOAT64,
+    date_string_col STRING,
+    string_col STRING,
+    timestamp_col DATETIME,
+    year INT64,
+    month INT64
+)
+PARTITION BY _PARTITIONDATE
+FROM FILES (
+    format = 'PARQUET',
+    uris = ['gs://ibis-ci-data/functional_alltypes.parquet']
+);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.timestamp_column_parted (
+    my_timestamp_parted_col TIMESTAMP,
+    string_col STRING,
+    int_col INT64
+)
+PARTITION BY DATE(my_timestamp_parted_col);
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.date_column_parted (
+    my_date_parted_col DATE,
+    string_col STRING,
+    int_col INT64
+)
+PARTITION BY my_date_parted_col;
+
+CREATE OR REPLACE TABLE ibis_gbq_testing.struct_table (
+    array_of_structs_col ARRAY<STRUCT<int_field INTEGER, string_field STRING>>,
+    nested_struct_col STRUCT<sub_struct STRUCT<timestamp_col TIMESTAMP>>,
+    struct_col STRUCT<string_field STRING>
+);
+
+INSERT INTO ibis_gbq_testing.struct_table VALUES
+    ([(12345, 'abcdefg'), (NULL, NULL)],
+     STRUCT(STRUCT(NULL)),
+     STRUCT(NULL)),
+    ([(12345, 'abcdefg'), (NULL, 'hijklmnop')],
+     STRUCT(STRUCT('2017-10-20 16:37:50.000000')),
+     STRUCT('a'));

ci/schema/clickhouse.sql

@@ -94,3 +94,6 @@ INSERT INTO ibis_testing.win VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+CREATE OR REPLACE TABLE ibis_testing.topk (x Nullable(Int64)) ENGINE = Memory;
+INSERT INTO ibis_testing.topk VALUES (1), (1), (NULL);

ci/schema/duckdb.sql

@@ -51,3 +51,7 @@ CREATE OR REPLACE TABLE map (idx BIGINT, kv MAP(STRING, BIGINT));
 INSERT INTO map VALUES
     (1, MAP(['a', 'b', 'c'], [1, 2, 3])),
     (2, MAP(['d', 'e', 'f'], [4, 5, 6]));
+
+
+CREATE OR REPLACE TABLE topk (x BIGINT);
+INSERT INTO topk VALUES (1), (1), (NULL);

ci/schema/exasol.sql

@@ -87,3 +87,6 @@ INSERT INTO "win" VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+CREATE OR REPLACE TABLE EXASOL."topk" ("x" BIGINT);
+INSERT INTO "topk" VALUES (1), (1), (NULL);

ci/schema/mssql.sql

@@ -131,3 +131,8 @@ INSERT INTO win VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+DROP TABLE IF EXISTS topk;
+
+CREATE TABLE topk (x BIGINT);
+INSERT INTO topk VALUES (1), (1), (NULL);

ci/schema/mysql.sql

@@ -119,3 +119,8 @@ INSERT INTO win VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+DROP TABLE IF EXISTS topk CASCADE;
+
+CREATE TABLE topk (x BIGINT);
+INSERT INTO topk VALUES (1), (1), (NULL);

ci/schema/oracle.sql

@@ -116,4 +116,9 @@ INSERT INTO "win" VALUES
     ('a', 3, 1),
     ('a', 4, 1);
 
+DROP TABLE IF EXISTS "topk";
+
+CREATE TABLE "topk" ("x" NUMBER(18));
+INSERT INTO "topk" VALUES (1), (1), (NULL);
+
 COMMIT;

ci/schema/postgres.sql

@@ -289,3 +289,8 @@ CREATE TABLE map (idx BIGINT, kv HSTORE);
 INSERT INTO map VALUES
     (1, 'a=>1,b=>2,c=>3'),
     (2, 'd=>4,e=>5,c=>6');
+
+DROP TABLE IF EXISTS topk;
+
+CREATE TABLE topk (x BIGINT);
+INSERT INTO topk VALUES (1), (1), (NULL);

ci/schema/risingwave.sql

@@ -175,3 +175,8 @@ INSERT INTO "win" VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+DROP TABLE IF EXISTS "topk";
+
+CREATE TABLE "topk" ("x" BIGINT);
+INSERT INTO "topk" VALUES (1), (1), (NULL);

ci/schema/snowflake.sql

@@ -140,3 +140,6 @@ INSERT INTO "win" VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+CREATE OR REPLACE TABLE "topk" ("x" BIGINT);
+INSERT INTO "topk" VALUES (1), (1), (NULL);

ci/schema/sqlite.sql

@@ -119,3 +119,7 @@ INSERT INTO win VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+DROP TABLE IF EXISTS topk;
+CREATE TABLE "topk" ("x" BIGINT);
+INSERT INTO "topk" VALUES (1), (1), (NULL);

ci/schema/trino.sql

@@ -178,3 +178,7 @@ INSERT INTO win VALUES
     ('a', 2, 0),
     ('a', 3, 1),
     ('a', 4, 1);
+
+DROP TABLE IF EXISTS topk;
+CREATE TABLE topk (x BIGINT);
+INSERT INTO topk VALUES (1), (1), (NULL);

compose.yaml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:23.12.4.15-alpine
+    image: clickhouse/clickhouse-server:23.12.5.81-alpine
     ports:
       - 8123:8123 # http port
       - 9000:9000 # native protocol port
@@ -94,7 +94,7 @@ services:
       - trino
 
   minio:
-    image: bitnami/minio:2024.3.10
+    image: bitnami/minio:2024.3.15
     environment:
       MINIO_ROOT_USER: accesskey
       MINIO_ROOT_PASSWORD: secretkey
@@ -156,7 +156,7 @@ services:
       test:
         - CMD-SHELL
         - trino --output-format null --execute 'show schemas in hive; show schemas in memory'
-    image: trinodb/trino:440
+    image: trinodb/trino:442
     ports:
       - 8080:8080
     networks:
@@ -379,7 +379,7 @@ services:
       - exasol:/data
 
   flink-jobmanager:
-    image: flink:1.18.1
+    image: flink:${FLINK_VERSION}
     environment:
       FLINK_PROPERTIES: |
         jobmanager.rpc.address: flink-jobmanager
@@ -390,8 +390,11 @@ services:
       - flink
 
   flink:
-    build: ./docker/flink
-    image: ibis-flink
+    image: ibis-flink-${FLINK_VERSION}
+    build:
+      context: ./docker/flink
+      args:
+        FLINK_VERSION: ${FLINK_VERSION}
     environment:
       FLINK_PROPERTIES: |
         jobmanager.rpc.address: flink-jobmanager

docker/flink/Dockerfile

@@ -1,7 +1,9 @@
-FROM flink:1.18.1
+ARG FLINK_VERSION=1.18.1
+FROM flink:${FLINK_VERSION}
 
 # ibis-flink requires PyFlink dependency
-RUN wget -nv -P $FLINK_HOME/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-python/1.18.1/flink-python-1.18.1.jar
+ARG FLINK_VERSION=1.18.1
+RUN wget -nv -P $FLINK_HOME/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-python/${FLINK_VERSION}/flink-python-${FLINK_VERSION}.jar
 
 # install python3 and pip3
 RUN apt-get update -y && \
@@ -12,4 +14,4 @@ RUN ln -s /usr/bin/python3 /usr/bin/python
 # install PyFlink
 ARG BUILDARCH
 ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-${BUILDARCH:-amd64}
-RUN pip3 install apache-flink==1.18.1
+RUN pip3 install apache-flink==${FLINK_VERSION}

docker/trino/catalog/hive.properties

@@ -1,7 +1,5 @@
 connector.name=hive
 
-hive.allow-drop-table=true
-hive.allow-rename-table=true
 hive.ignore-absent-partitions=true
 hive.metastore.thrift.delete-files-on-drop=true
 hive.metastore.uri=thrift://hive-metastore:9083

docs/_quarto.yml

@@ -449,6 +449,10 @@ quartodoc:
               package: ibis
               dynamic: true
               signature_name: full
+            - name: today
+              package: ibis
+              dynamic: true
+              signature_name: full
             - name: date
               package: ibis
               dynamic: true