From 93c257dd2ffdad35437ebab80227567d2b7bed32 Mon Sep 17 00:00:00 2001 From: tserakhau Date: Fri, 24 Jan 2025 17:47:45 +0300 Subject: [PATCH] Use altered table schema for tableCols map Fix: https://github.com/doublecloud/transfer/issues/173 --- Pull Request resolved: https://github.com/doublecloud/transfer/pull/178 Co-authored-by: tserakhau commit_hash:4a9b80641f7596bba9ca3c144756d62d8775d287 --- .mapping.json | 3 ++ pkg/providers/mysql/schema.go | 2 +- tests/e2e/mysql2ch/snapshot_nofk/ch.sql | 1 + .../mysql2ch/snapshot_nofk/check_db_test.go | 43 +++++++++++++++++++ .../e2e/mysql2ch/snapshot_nofk/dump/dump.sql | 16 +++++++ 5 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/mysql2ch/snapshot_nofk/ch.sql create mode 100644 tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go create mode 100644 tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql diff --git a/.mapping.json b/.mapping.json index b36e88b3..c8f42a7e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2536,6 +2536,9 @@ "tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go", "tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql", "tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql", + "tests/e2e/mysql2ch/snapshot_nofk/ch.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/ch.sql", + "tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go", + "tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql", "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted", "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0", "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1", diff --git a/pkg/providers/mysql/schema.go b/pkg/providers/mysql/schema.go index 02b16139..8252b75e 100644 --- a/pkg/providers/mysql/schema.go +++ b/pkg/providers/mysql/schema.go @@ -179,7 +179,7 @@ func LoadSchema(tx queryExecutor, useFakePrimaryKey bool, includeViews bool) (ab tableSchema[i].FakeKey = true } } - tableCols[tID] = makeTableSchema(currSchema, uniq(keys)) + tableCols[tID] = tableSchema } dbSchema := make(abstract.DBSchema) for tableID, columns := range enrichExpressions(tx, tableCols) { diff --git a/tests/e2e/mysql2ch/snapshot_nofk/ch.sql b/tests/e2e/mysql2ch/snapshot_nofk/ch.sql new file mode 100644 index 00000000..9bcf3484 --- /dev/null +++ b/tests/e2e/mysql2ch/snapshot_nofk/ch.sql @@ -0,0 +1 @@ +CREATE DATABASE source; diff --git a/tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go b/tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go new file mode 100644 index 00000000..f63a095b --- /dev/null +++ b/tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go @@ -0,0 +1,43 @@ +package snapshotnofk + +import ( + "testing" + + "github.com/doublecloud/transfer/pkg/abstract" + chrecipe "github.com/doublecloud/transfer/pkg/providers/clickhouse/recipe" + "github.com/doublecloud/transfer/tests/e2e/mysql2ch" + "github.com/doublecloud/transfer/tests/e2e/pg2ch" + "github.com/doublecloud/transfer/tests/helpers" + "github.com/stretchr/testify/require" +) + +func TestSnapshot(t *testing.T) { + source := helpers.RecipeMysqlSource() + target := chrecipe.MustTarget(chrecipe.WithInitFile("ch.sql"), chrecipe.WithDatabase("source")) + + defer func() { + require.NoError(t, helpers.CheckConnections( + helpers.LabeledPort{Label: "MySQL source", Port: source.Port}, + helpers.LabeledPort{Label: "CH target", Port: target.NativePort}, + )) + }() + + t.Run("fake_keys", func(t *testing.T) { + source.UseFakePrimaryKey = true + transfer := helpers.MakeTransfer(helpers.TransferID, source, target, abstract.TransferTypeSnapshotAndIncrement) + _, err := helpers.ActivateErr(transfer) + require.NoError(t, err) + require.NoError(t, helpers.CompareStorages( + t, + source, + target, + helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(mysql2ch.MySQLBytesToStringOptionalComparator), + )) + }) + t.Run("no_fake_keys", func(t *testing.T) { + source.UseFakePrimaryKey = false + transfer := helpers.MakeTransfer(helpers.TransferID, source, target, abstract.TransferTypeSnapshotAndIncrement) + _, err := helpers.ActivateErr(transfer) + require.Error(t, err) + }) +} diff --git a/tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql b/tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql new file mode 100644 index 00000000..607d928a --- /dev/null +++ b/tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql @@ -0,0 +1,16 @@ +-- Create table without a primary key +CREATE TABLE no_pk ( + id INT NOT NULL, + name VARCHAR(100) NOT NULL, + age INT NOT NULL, + city VARCHAR(100) NOT NULL, + email VARCHAR(100) NOT NULL +); + +-- Insert 5 unique rows +INSERT INTO no_pk (id, name, age, city, email) VALUES +(1, 'Alice', 30, 'New York', 'alice@example.com'), +(2, 'Bob', 25, 'Los Angeles', 'bob@example.com'), +(3, 'Charlie', 35, 'Chicago', 'charlie@example.com'), +(4, 'Diana', 28, 'San Francisco', 'diana@example.com'), +(5, 'Eve', 40, 'Miami', 'eve@example.com');