From 51504a80eb486dac1d4f33be9a5660184b66f56b Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 24 Jan 2024 14:53:26 -0500 Subject: [PATCH] materialize-bigquery: validate bignumeric columns as integer or number We used to create number columns as "bignumeric" (see https://github.com/estuary/connectors/pull/859.), and now create strings formatted as integer columns as "bignumeric". Allowing "bignumeric" columns to be either integer or number compatible allows both new and old materializations to work. --- materialize-bigquery/sqlgen.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/materialize-bigquery/sqlgen.go b/materialize-bigquery/sqlgen.go index e832693881..fd814cde8c 100644 --- a/materialize-bigquery/sqlgen.go +++ b/materialize-bigquery/sqlgen.go @@ -87,7 +87,7 @@ var bqDialect = func() sql.Dialect { sql.ColValidation{Types: []string{"int64"}, Validate: sql.IntegerCompatible}, sql.ColValidation{Types: []string{"float64"}, Validate: sql.NumberCompatible}, sql.ColValidation{Types: []string{"json"}, Validate: sql.MultipleCompatible}, - sql.ColValidation{Types: []string{"bignumeric"}, Validate: sql.IntegerCompatible}, + sql.ColValidation{Types: []string{"bignumeric"}, Validate: bignumericCompatible}, sql.ColValidation{Types: []string{"date"}, Validate: sql.DateCompatible}, sql.ColValidation{Types: []string{"timestamp"}, Validate: sql.DateTimeCompatible}, ) @@ -118,6 +118,13 @@ var bqDialect = func() sql.Dialect { } }() +// bignumericCompatible allows either integers or numbers, since we used to create number columns as +// "bignumeric" (now they are float64), and currently create strings formatted as integers as +// "bignumeric". This is needed for compatibility for older materializations. +func bignumericCompatible(p pf.Projection) bool { + return sql.IntegerCompatible(p) || sql.NumberCompatible(p) +} + // stringCompatible allow strings of any format, arrays, or objects to be materialized. func stringCompatible(p pf.Projection) bool { if sql.StringCompatible(p) {