diff --git a/.changeset/quick-sheep-buy.md b/.changeset/quick-sheep-buy.md new file mode 100644 index 000000000..e313ca642 --- /dev/null +++ b/.changeset/quick-sheep-buy.md @@ -0,0 +1,5 @@ +--- +'@xata.io/cli': patch +--- + +add ability to import new data to existing tables diff --git a/packages/cli/src/commands/import/csv.test.ts b/packages/cli/src/commands/import/csv.test.ts index f13f8da8e..9d6f68101 100644 --- a/packages/cli/src/commands/import/csv.test.ts +++ b/packages/cli/src/commands/import/csv.test.ts @@ -1,5 +1,68 @@ import { describe, expect, test } from 'vitest'; import { splitCommas } from './csv'; +import { compareSchemas } from '../../utils/compareSchema'; +import { Schemas } from '@xata.io/client'; + +const defaultTableInfo = { + name: 'one', + xataCompatible: true, + comment: '', + primaryKey: ['id'], + uniqueConstraints: {}, + checkConstraints: {}, + foreignKeys: {}, + indexes: {}, + oid: '' +}; +const sharedIdCol = { + id: { + name: 'id', + type: 'integer', + comment: '', + nullable: false, + unique: true, + default: null + } +}; +const sourceSchemaDefault: Schemas.BranchSchema = { + name: 'main', + tables: { + one: { + ...defaultTableInfo, + columns: { + ...sharedIdCol, + colToDelete: { + name: 'colToDelete', + type: 'text', + comment: '', + nullable: false, + unique: false, + default: null + } + } + } + } +}; + +const targetSchemaDefault: Schemas.BranchSchema = { + name: 'main', + tables: { + one: { + ...defaultTableInfo, + columns: { + ...sharedIdCol, + newCol: { + name: 'newCol', + type: 'text', + comment: '', + nullable: false, + unique: false, + default: null + } + } + } + } +}; describe('splitCommas', () => { test('returns [] for falsy values', () => { @@ -13,3 +76,127 @@ describe('splitCommas', () => { expect(splitCommas('a,b,c')).toEqual(['a', 'b', 'c']); }); }); + +describe('compare schemas', () => { + test('returns an empty array for identical schemas', () => { + const { edits } = compareSchemas({ source: sourceSchemaDefault, target: sourceSchemaDefault }); + expect(edits).toEqual([]); + }); + test('ignores internal columns on source', () => { + const { edits } = compareSchemas({ + source: { + ...sourceSchemaDefault, + tables: { + ...sourceSchemaDefault.tables, + one: { + ...sourceSchemaDefault.tables.one, + columns: { + ...sourceSchemaDefault.tables.one.columns, + xata_id: { + name: 'xata_id', + type: 'text', + comment: '', + nullable: false, + unique: false, + default: null + } + } + } + } + }, + target: targetSchemaDefault + }); + expect(edits).toMatchInlineSnapshot(compareSnapshot); + }); + test('ignores internal columns on target', () => { + const { edits } = compareSchemas({ + source: sourceSchemaDefault, + target: { + ...targetSchemaDefault, + tables: { + ...targetSchemaDefault.tables, + one: { + ...targetSchemaDefault.tables.one, + columns: { + ...targetSchemaDefault.tables.one.columns, + xata_id: { + name: 'xata_id', + type: 'text', + comment: '', + nullable: false, + unique: false, + default: null + } + } + } + } + } + }); + expect(edits).toMatchInlineSnapshot(compareSnapshot); + }); + test('returns an array with create table if table does not already exist', () => { + const { edits } = compareSchemas({ source: {}, target: targetSchemaDefault }); + expect(edits).toMatchInlineSnapshot(` + [ + { + "create_table": { + "columns": [ + { + "comment": "", + "default": undefined, + "name": "id", + "nullable": false, + "references": undefined, + "type": "integer", + "unique": true, + }, + { + "comment": "", + "default": undefined, + "name": "newCol", + "nullable": false, + "references": undefined, + "type": "text", + "unique": false, + }, + ], + "comment": "", + "name": "one", + }, + }, + ] + `); + }); + test('returns an array with add_column for new columns', () => { + const { edits } = compareSchemas({ source: sourceSchemaDefault, target: targetSchemaDefault }); + expect(edits).toMatchInlineSnapshot(compareSnapshot); + }); + test('returns an array with drop_column for deleted columns', () => { + const { edits } = compareSchemas({ source: sourceSchemaDefault, target: targetSchemaDefault }); + expect(edits).toMatchInlineSnapshot(compareSnapshot); + }); +}); + +const compareSnapshot = ` +[ + { + "add_column": { + "column": { + "comment": "", + "default": undefined, + "name": "newCol", + "nullable": false, + "references": undefined, + "type": "text", + "unique": false, + }, + "table": "one", + }, + }, + { + "drop_column": { + "column": "colToDelete", + "table": "one", + }, + }, +]`; diff --git a/packages/cli/src/commands/import/csv.ts b/packages/cli/src/commands/import/csv.ts index 5415ff2e1..74088d58d 100644 --- a/packages/cli/src/commands/import/csv.ts +++ b/packages/cli/src/commands/import/csv.ts @@ -8,11 +8,9 @@ import { enumFlag } from '../../utils/oclif.js'; import { getBranchDetailsWithPgRoll, isBranchPgRollEnabled, - waitForMigrationToFinish, - xataColumnTypeToPgRollComment + waitForMigrationToFinish } from '../../migrations/pgroll.js'; -import { compareSchemas } from '../../utils/compareSchema.js'; -import { keyBy } from 'lodash'; +import { compareSchemas, inferOldSchemaToNew } from '../../utils/compareSchema.js'; const ERROR_CONSOLE_LOG_LIMIT = 200; const ERROR_LOG_FILE = 'errors.log'; @@ -30,7 +28,7 @@ const bufferEncodings: BufferEncoding[] = [ 'hex' ]; -const INTERNAL_COLUMNS_PGROLL = ['xata_id', 'xata_createdat', 'xata_updatedat', 'xata_version']; +export const INTERNAL_COLUMNS_PGROLL = ['xata_id', 'xata_createdat', 'xata_updatedat', 'xata_version']; export default class ImportCSV extends BaseCommand { static description = 'Import a CSV file'; @@ -246,6 +244,7 @@ export default class ImportCSV extends BaseCommand { const xata = await this.getXataClient(); const { workspace, region, database, branch } = await this.parseDatabase(); const { schema: existingSchema } = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch }); + const newSchema = { tables: [ ...existingSchema.tables.filter((t) => t.name !== table), @@ -254,33 +253,12 @@ export default class ImportCSV extends BaseCommand { }; if (this.#pgrollEnabled) { - const { edits } = compareSchemas( - {}, - { - tables: { - [table]: { - name: table, - xataCompatible: false, - columns: keyBy( - columns - .filter((c) => !INTERNAL_COLUMNS_PGROLL.includes(c.name as any)) - .map((c) => { - return { - name: c.name, - type: c.type, - nullable: c.notNull !== false, - default: c.defaultValue ?? null, - unique: c.unique, - comment: xataColumnTypeToPgRollComment(c) - }; - }), - 'name' - ) - } - } - } - ); - + const sourceSchema = inferOldSchemaToNew({ schema: existingSchema, branchName: branch }); + const targetSchema = inferOldSchemaToNew({ schema: newSchema, branchName: branch }); + const { edits } = compareSchemas({ + source: sourceSchema, + target: targetSchema + }); if (edits.length > 0) { const destructiveOperations = edits .map((op) => { @@ -324,6 +302,7 @@ export default class ImportCSV extends BaseCommand { pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, body: { operations: edits, adaptTables: true } }); + await waitForMigrationToFinish(xata.api, workspace, region, database, branch, jobID); } } else { diff --git a/packages/cli/src/commands/schema/upload.ts b/packages/cli/src/commands/schema/upload.ts index e91e83487..005f0fc62 100644 --- a/packages/cli/src/commands/schema/upload.ts +++ b/packages/cli/src/commands/schema/upload.ts @@ -1,7 +1,13 @@ import { Args, Flags } from '@oclif/core'; import { readFile } from 'fs/promises'; import { BaseCommand } from '../../base.js'; -import { getBranchDetailsWithPgRoll } from '../../migrations/pgroll.js'; +import { + getBranchDetailsWithPgRoll, + isBranchPgRollEnabled, + xataToPgroll, + waitForMigrationToFinish +} from '../../migrations/pgroll.js'; +import { compareSchemas } from '../../utils/compareSchema.js'; export default class UploadSchema extends BaseCommand { static description = 'Apply a schema to the current database from file'; @@ -33,45 +39,69 @@ export default class UploadSchema extends BaseCommand { const xata = await this.getXataClient(); - if (flags['create-only']) { - const { schema } = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch }); - if (schema.tables.length > 0) { - this.info( - 'Schema already exists. `xata schema upload --init` will only initialize the schema if it does not already exist.' - ); - return; - } + const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch }); + if (flags['create-only'] && details.schema.tables.length > 0) { + this.info( + 'Schema already exists. `xata schema upload --init` will only initialize the schema if it does not already exist.' + ); + return; } const schema = JSON.parse(await readFile(args.file, 'utf8')); - if (!Array.isArray(schema.tables)) { - this.error('Schema file does not contain a "tables" property'); - } - const { edits } = await xata.api.migrations.compareBranchWithUserSchema({ - pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, - body: { schema } - }); + if (isBranchPgRollEnabled(details)) { + const { edits } = compareSchemas(xataToPgroll(details), xataToPgroll(schema)); - if (edits.operations.length === 0) { - this.info('Schema is up to date'); - return; - } + if (edits.length === 0) { + this.info('Schema is up to date'); + return; + } + + this.log(JSON.stringify(edits, null, 2)); + + const { confirm } = await this.prompt({ + type: 'confirm', + name: 'confirm', + message: `Do you want to apply the above migration into the ${branch} branch?`, + initial: true + }); + if (!confirm) return this.exit(1); - this.printMigration({ edits }); - this.log(); - - const { confirm } = await this.prompt({ - type: 'confirm', - name: 'confirm', - message: `Do you want to apply the above migration into the ${branch} branch?`, - initial: true - }); - if (!confirm) return this.exit(1); - - await xata.api.migrations.applyBranchSchemaEdit({ - pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, - body: { edits } - }); + const { jobID } = await xata.api.migrations.applyMigration({ + pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, + body: { operations: edits, adaptTables: true } + }); + await waitForMigrationToFinish(xata.api, workspace, region, database, branch, jobID); + } else { + if (!Array.isArray(schema.tables)) { + this.error('Schema file does not contain a "tables" property'); + } + + const { edits } = await xata.api.migrations.compareBranchWithUserSchema({ + pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, + body: { schema } + }); + + if (edits.operations.length === 0) { + this.info('Schema is up to date'); + return; + } + + this.printMigration({ edits }); + this.log(); + + const { confirm } = await this.prompt({ + type: 'confirm', + name: 'confirm', + message: `Do you want to apply the above migration into the ${branch} branch?`, + initial: true + }); + if (!confirm) return this.exit(1); + + await xata.api.migrations.applyBranchSchemaEdit({ + pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, + body: { edits } + }); + } } } diff --git a/packages/cli/src/migrations/pgroll.ts b/packages/cli/src/migrations/pgroll.ts index 97f6e4b3e..cacdb2d4d 100644 --- a/packages/cli/src/migrations/pgroll.ts +++ b/packages/cli/src/migrations/pgroll.ts @@ -121,42 +121,100 @@ export async function getBranchDetailsWithPgRoll( return { ...details, - branchName: branch, - createdAt: new Date().toISOString(), - databaseName: database, - id: pgroll.schema.name, // Not really - lastMigrationID: '', // Not really - version: 1, - metadata: {}, - schema: { - tables: Object.entries(pgroll.schema.tables ?? []).map(([name, table]: any) => ({ - name, + ...pgrollToXata({ schema: pgroll.schema, branchName: branch, databaseName: database }) + } as any; + } + + return details; +} + +export const xataToPgroll = (branch: Schemas.DBBranch): Schemas.BranchSchema => { + const tables: Schemas.BranchSchema['tables'] = Object.fromEntries( + branch.schema.tables.map( + (table: any) => + ({ + name: table.name, checkConstraints: table.checkConstraints, foreignKeys: table.foreignKeys, primaryKey: table.primaryKey, + xataCompatible: true, + indexes: {}, uniqueConstraints: table.uniqueConstraints, - columns: Object.values(table.columns ?? {}) - .filter((column: any) => !['_id', '_createdat', '_updatedat', '_version'].includes(column.name)) - .map((column: any) => ({ - name: column.name, - type: getPgRollLink(table, column) ? 'link' : pgRollToXataColumnType(column.type, column.comment), - link: getPgRollLink(table, column) ? { table: getPgRollLink(table, column).referencedTable } : undefined, - file: - pgRollToXataColumnType(column.type) === 'file' || pgRollToXataColumnType(column.type) === 'file[]' - ? { defaultPublicAccess: false } - : undefined, - notNull: column.nullable === false, - unique: column.unique === true, - defaultValue: column.default, - comment: column.comment - })) - })) - } as any - }; - } + comment: '', + columns: Object.fromEntries( + table.columns.map( + (column: Schemas.DBBranch['schema']['tables'][number]['columns'][number]) => + ({ + name: column.name, + type: xataColumnTypeToPgRoll(column.type), + file: + column.type === 'file' + ? { defaultPublicAccess: column?.file?.defaultPublicAccess } + : column.type === 'file[]' + ? { defaultPublicAccess: column['file[]']?.defaultPublicAccess } + : undefined, + link: column.type === 'link' ? { referencedTable: column?.link?.table } : undefined, + nullable: Boolean(column.notNull), + unique: column.unique, + default: column.defaultValue, + comment: xataColumnTypeToPgRollComment(column), + vector: column.type === 'vector' ? { dimension: column.vector?.dimension } : undefined + } as Schemas.BranchSchema['tables'][number]['columns'][number]) + ) + ) + } as any) + ) + ); - return details; -} + return { + name: branch.branchName, + tables + }; +}; + +const pgrollToXata = ({ + schema, + branchName, + databaseName +}: { + schema: Schemas.BranchSchema; + branchName: string; + databaseName: string; +}) => { + return { + branchName, + createdAt: new Date().toISOString(), + databaseName, + id: schema.name, // Not really + lastMigrationID: '', // Not really + version: 1, + metadata: {}, + schema: { + tables: Object.entries(schema.tables ?? []).map(([name, table]: any) => ({ + name, + checkConstraints: table.checkConstraints, + foreignKeys: table.foreignKeys, + primaryKey: table.primaryKey, + uniqueConstraints: table.uniqueConstraints, + columns: Object.values(table.columns ?? {}) + .filter((column: any) => !['_id', '_createdat', '_updatedat', '_version'].includes(column.name)) + .map((column: any) => ({ + name: column.name, + type: getPgRollLink(table, column) ? 'link' : pgRollToXataColumnType(column.type, column.comment), + link: getPgRollLink(table, column) ? { table: getPgRollLink(table, column).referencedTable } : undefined, + file: + pgRollToXataColumnType(column.type) === 'file' || pgRollToXataColumnType(column.type) === 'file[]' + ? { defaultPublicAccess: false } + : undefined, + notNull: column.nullable === false, + unique: column.unique === true, + defaultValue: column.default, + comment: column.comment + })) + })) + } as any + }; +}; export const isColumnTypeUnsupported = (type: string) => { switch (type) { diff --git a/packages/cli/src/utils/compareSchema.ts b/packages/cli/src/utils/compareSchema.ts index 3bcdcabfa..c77ecf764 100644 --- a/packages/cli/src/utils/compareSchema.ts +++ b/packages/cli/src/utils/compareSchema.ts @@ -2,11 +2,15 @@ import { PgRollOperation } from '@xata.io/pgroll'; import { PartialDeep } from 'type-fest'; import { Schemas } from '@xata.io/client'; import { generateLinkReference, tableNameFromLinkComment, xataColumnTypeToPgRoll } from '../migrations/pgroll.js'; +import { INTERNAL_COLUMNS_PGROLL } from '../commands/import/csv.js'; -export function compareSchemas( - source: PartialDeep, - target: PartialDeep -): { edits: PgRollOperation[] } { +export function compareSchemas({ + source, + target +}: { + source: PartialDeep; + target: PartialDeep; +}): { edits: PgRollOperation[] } { const edits: PgRollOperation[] = []; // Compare tables @@ -17,8 +21,12 @@ export function compareSchemas( // Compare columns for (const table of sourceTables) { - const sourceColumns = Object.keys(source.tables?.[table]?.columns ?? {}); - const targetColumns = Object.keys(target.tables?.[table]?.columns ?? {}); + const sourceColumns = Object.keys(source.tables?.[table]?.columns ?? {}).filter( + (c) => !INTERNAL_COLUMNS_PGROLL.includes(c) + ); + const targetColumns = Object.keys(target.tables?.[table]?.columns ?? {}).filter( + (c) => !INTERNAL_COLUMNS_PGROLL.includes(c) + ); const newColumns = targetColumns.filter((column) => !sourceColumns.includes(column)); const deletedColumns = sourceColumns.filter((column) => !targetColumns.includes(column)); @@ -51,45 +59,6 @@ export function compareSchemas( for (const column of deletedColumns) { edits.push({ drop_column: { table, column } }); } - - // Compare column properties - for (const column of targetColumns) { - const sourceProps = source.tables?.[table]?.columns?.[column] ?? {}; - const targetProps = target.tables?.[table]?.columns?.[column] ?? {}; - - if (sourceProps.type !== targetProps.type) { - edits.push({ - alter_column: { - table, - column, - type: targetProps.type, - references: - targetProps?.type === 'link' && targetProps?.name - ? generateLinkReference({ - column: targetProps.name, - table: tableNameFromLinkComment(targetProps?.comment ?? '') ?? '' - }) - : undefined - } - }); - } - - if (sourceProps.nullable !== targetProps.nullable) { - edits.push({ alter_column: { table, column, nullable: targetProps.nullable } }); - } - - if (sourceProps.unique !== targetProps.unique) { - edits.push({ - alter_column: { - table, - column, - unique: { - name: `${table}_${column}_unique` - } - } - }); - } - } } // Delete tables @@ -104,26 +73,118 @@ export function compareSchemas( create_table: { name: table, comment: props.comment, - columns: Object.entries(props.columns ?? {}).map(([name, column]) => { - return { - name, - type: xataColumnTypeToPgRoll(column?.type as any), - comment: column?.comment, - nullable: !(column?.nullable === false), - unique: column?.unique, - default: column?.default ?? undefined, - references: - column?.type === 'link' && column?.name - ? generateLinkReference({ - column: column?.name, - table: tableNameFromLinkComment(column?.comment ?? '') ?? '' - }) - : undefined - }; - }) + columns: Object.entries(props.columns ?? {}) + .filter(([name, _]) => !INTERNAL_COLUMNS_PGROLL.includes(name)) + .map(([name, column]) => { + return { + name, + type: xataColumnTypeToPgRoll(column?.type as any), + comment: column?.comment, + nullable: !(column?.nullable === false), + unique: column?.unique, + default: column?.default ?? undefined, + references: + column?.type === 'link' && column?.name + ? generateLinkReference({ + column: column?.name, + table: tableNameFromLinkComment(column?.comment ?? '') ?? '' + }) + : undefined + }; + }) } }); } - return { edits }; } + +export const inferOldSchemaToNew = ( + oldSchema: Pick +): Schemas.BranchSchema => { + const schema: Schemas.BranchSchema = { + name: oldSchema.branchName, + tables: Object.fromEntries( + oldSchema.schema.tables.map((table) => [ + table.name, + { + name: table.name, + xataCompatible: true, + comment: '', + primaryKey: 'id', + uniqueConstraints: [], + checkConstraints: [], + foreignKeys: [], + columns: Object.fromEntries( + table.columns.map((column) => [ + column.name, + { + name: column.name, + type: oldColumnTypeToNew(column), + comment: generateCommentFromOldColumn(column), + nullable: !(column.notNull === true), + unique: column.unique === true, + defaultValue: column.defaultValue + } + ]) + ) + } + ]) as any + ) + }; + + return schema; +}; + +const oldColumnTypeToNew = (oldColumn: Schemas.Column) => { + // These types will be limited to the original deprecated Xata types + switch (oldColumn.type) { + case 'bool': + return 'boolean'; + case 'datetime': + return 'timestamptz'; + case 'vector': + return 'vector'; + case 'json': + return 'jsonb'; + case 'file': + return 'xata_file'; + case 'file[]': + return 'xata_file_array'; + case 'int': + return 'integer'; + case 'float': + return 'real'; + case 'multiple': + return 'text[]'; + case 'text': + case 'string': + case 'email': + return 'text'; + case 'link': + return 'link'; + default: + return 'text'; + } +}; + +const generateCommentFromOldColumn = (oldColumn: Schemas.Column) => { + switch (oldColumn.type) { + case 'vector': + return JSON.stringify({ 'xata.search.dimension': oldColumn.vector?.dimension }); + case 'file': + return JSON.stringify({ 'xata.file.dpa': oldColumn?.file?.defaultPublicAccess }); + case 'file[]': + return JSON.stringify({ 'xata.file.dpa': oldColumn?.['file[]']?.defaultPublicAccess }); + case 'link': + return oldColumn.link?.table ? generateLinkComment(oldColumn.link?.table) : ''; + case 'string': + case 'email': + return JSON.stringify({ 'xata.type': oldColumn.type }); + default: + return ''; + } +}; + +const generateLinkComment = (tableName: string) => { + return JSON.stringify({ 'xata.link': tableName }); +};