From 3bbe06a55becfb23a6bf914ad12ae1763a04f900 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 26 Aug 2021 16:29:28 +0200 Subject: [PATCH] recreate table WIP --- .../__tests__/alter-processor.spec.js | 3 +- packages/tools/src/SqlDumper.ts | 79 +++++++++++++++- packages/tools/src/alterPlan.ts | 92 +++++++++++++++++-- .../src/database-info-alter-processor.ts | 4 + packages/types/alter-processor.d.ts | 1 + packages/types/dialect.d.ts | 12 ++- .../src/frontend/driver.js | 13 ++- .../src/frontend/drivers.js | 9 ++ .../src/frontend/drivers.js | 10 ++ .../src/frontend/driver.js | 11 ++- 10 files changed, 222 insertions(+), 12 deletions(-) diff --git a/integration-tests/__tests__/alter-processor.spec.js b/integration-tests/__tests__/alter-processor.spec.js index a586562d..db01ec4c 100644 --- a/integration-tests/__tests__/alter-processor.spec.js +++ b/integration-tests/__tests__/alter-processor.spec.js @@ -55,7 +55,8 @@ async function testTableDiff(conn, driver, mangle) { // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real)); } -const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq']; +// const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq']; +const TESTED_COLUMNS = ['col_pk']; // const TESTED_COLUMNS = ['col_idx']; // const TESTED_COLUMNS = ['col_fk']; // const TESTED_COLUMNS = ['col_std']; diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index d0e1ff7e..7bccec77 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -20,6 +20,7 @@ import { import _isString from 'lodash/isString'; import _isNumber from 'lodash/isNumber'; import _isDate from 'lodash/isDate'; +import uuidv1 from 'uuid/v1'; export class SqlDumper implements AlterProcessor { s = ''; @@ -372,6 +373,25 @@ export class SqlDumper implements AlterProcessor { break; } } + createConstraint(cnt: ConstraintInfo) { + switch (cnt.constraintType) { + case 'primaryKey': + this.createPrimaryKey(cnt as PrimaryKeyInfo); + break; + case 'foreignKey': + this.createForeignKey(cnt as ForeignKeyInfo); + break; + case 'unique': + this.createUnique(cnt as UniqueInfo); + break; + case 'check': + this.createCheck(cnt as CheckInfo); + break; + case 'index': + this.createIndex(cnt as IndexInfo); + break; + } + } dropForeignKey(fk: ForeignKeyInfo) { if (this.dialect.explicitDropConstraint) { this.putCmd('^alter ^table %f ^drop ^foreign ^key %i', fk, fk.constraintName); @@ -463,7 +483,7 @@ export class SqlDumper implements AlterProcessor { changeColumn(oldcol: ColumnInfo, newcol: ColumnInfo, constraints: ConstraintInfo[]) {} - dropTable(obj: TableInfo, { testIfExists = false }) { + dropTable(obj: TableInfo, { testIfExists = false } = {}) { this.putCmd('^drop ^table %f', obj); } @@ -489,4 +509,61 @@ export class SqlDumper implements AlterProcessor { truncateTable(name: NamedObjectInfo) { this.putCmd('^delete ^from %f', name); } + + dropConstraints(table: TableInfo, dropReferences = false) { + if (dropReferences && this.dialect.dropForeignKey) { + table.dependencies.forEach(cnt => this.dropConstraint(cnt)); + } + if (this.dialect.dropIndex) { + table.indexes.forEach(cnt => this.dropIndex(cnt)); + } + if (this.dialect.dropForeignKey) { + table.foreignKeys.forEach(cnt => this.dropForeignKey(cnt)); + } + if (this.dialect.dropPrimaryKey && table.primaryKey) { + this.dropPrimaryKey(table.primaryKey); + } + } + + recreateTable(oldTable: TableInfo, newTable: TableInfo) { + if (oldTable.pairingId != newTable.pairingId) { + throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId'); + } + const tmpTable = `temp_${uuidv1()}`; + + const columnPairs = oldTable.columns + .map(oldcol => ({ + oldcol, + newcol: newTable.columns.find(x => x.pairingId == oldcol.pairingId), + })) + .filter(x => x.newcol); + + this.dropConstraints(oldTable, true); + this.renameTable(oldTable, tmpTable); + + this.createTable(newTable); + + const autoinc = newTable.columns.find(x => x.autoIncrement); + if (autoinc) { + this.allowIdentityInsert(newTable, true); + } + + this.putCmd( + '^insert ^into %f (%,i) select %,s ^from %f', + newTable, + columnPairs.map(x => x.newcol.columnName), + columnPairs.map(x => x.oldcol.columnName), + { ...oldTable, pureName: tmpTable } + ); + + if (autoinc) { + this.allowIdentityInsert(newTable, false); + } + + if (this.dialect.dropForeignKey) { + newTable.dependencies.forEach(cnt => this.createConstraint(cnt)); + } + + this.dropTable({ ...oldTable, pureName: tmpTable }); + } } diff --git a/packages/tools/src/alterPlan.ts b/packages/tools/src/alterPlan.ts index ad7b407e..56277bca 100644 --- a/packages/tools/src/alterPlan.ts +++ b/packages/tools/src/alterPlan.ts @@ -8,6 +8,8 @@ import { SqlDialect, TableInfo, } from '../../types'; +import { DatabaseInfoAlterProcessor } from './database-info-alter-processor'; +import { DatabaseAnalyser } from './DatabaseAnalyser'; interface AlterOperation_CreateTable { operationType: 'createTable'; @@ -68,6 +70,11 @@ interface AlterOperation_RenameConstraint { object: ConstraintInfo; newName: string; } +interface AlterOperation_RecreateTable { + operationType: 'recreateTable'; + table: TableInfo; + operations: AlterOperation[]; +} type AlterOperation = | AlterOperation_CreateColumn @@ -80,7 +87,8 @@ type AlterOperation = | AlterOperation_DropTable | AlterOperation_RenameTable | AlterOperation_RenameColumn - | AlterOperation_RenameConstraint; + | AlterOperation_RenameConstraint + | AlterOperation_RecreateTable; export class AlterPlan { public operations: AlterOperation[] = []; @@ -168,6 +176,14 @@ export class AlterPlan { }); } + recreateTable(table: TableInfo, operations: AlterOperation[]) { + this.operations.push({ + operationType: 'recreateTable', + table, + operations, + }); + } + run(processor: AlterProcessor) { for (const op of this.operations) { runAlterOperation(op, processor); @@ -180,15 +196,15 @@ export class AlterPlan { const table = this.db.tables.find( x => x.pureName == op.oldObject.pureName && x.schemaName == op.oldObject.schemaName ); - const deletedFks = this.dialect.dropColumnDependencies?.includes('foreignKey') + const deletedFks = this.dialect.dropColumnDependencies?.includes('dependencies') ? table.dependencies.filter(fk => fk.columns.find(col => col.refColumnName == op.oldObject.columnName)) : []; const deletedConstraints = _.compact([ - table.primaryKey, - ...table.foreignKeys, - ...table.indexes, - ...table.uniques, + this.dialect.dropColumnDependencies?.includes('primaryKey') ? table.primaryKey : null, + ...(this.dialect.dropColumnDependencies?.includes('foreignKeys') ? table.foreignKeys : []), + ...(this.dialect.dropColumnDependencies?.includes('indexes') ? table.indexes : []), + ...(this.dialect.dropColumnDependencies?.includes('uniques') ? table.uniques : []), ]).filter(cnt => cnt.columns.find(col => col.columnName == op.oldObject.columnName)); const res: AlterOperation[] = [ @@ -209,8 +225,63 @@ export class AlterPlan { return _.flatten(lists); } + _transformToImplementedOps(): AlterOperation[] { + const lists = this.operations.map(op => { + return ( + this._testTableRecreate(op, 'createColumn', this.dialect.createColumn, 'newObject') || + this._testTableRecreate(op, 'dropColumn', this.dialect.dropColumn, 'oldObject') || + this._testTableRecreate(op, 'createConstraint', obj => this._canCreateConstraint(obj), 'newObject') || + this._testTableRecreate(op, 'dropConstraint', obj => this._canDropConstraint(obj), 'oldObject') || [op] + ); + }); + + return _.flatten(lists); + } + + _canCreateConstraint(cnt: ConstraintInfo) { + if (cnt.constraintType == 'primaryKey') return this.dialect.createPrimaryKey; + if (cnt.constraintType == 'foreignKey') return this.dialect.createForeignKey; + if (cnt.constraintType == 'index') return this.dialect.createIndex; + return null; + } + + _canDropConstraint(cnt: ConstraintInfo) { + if (cnt.constraintType == 'primaryKey') return this.dialect.dropPrimaryKey; + if (cnt.constraintType == 'foreignKey') return this.dialect.dropForeignKey; + if (cnt.constraintType == 'index') return this.dialect.dropIndex; + return null; + } + + _testTableRecreate( + op: AlterOperation, + operationType: string, + isAllowed: boolean | Function, + objectField: string + ): AlterOperation[] | null { + if (op.operationType == operationType) { + if (_.isFunction(isAllowed)) { + if (!isAllowed(op[objectField])) return null; + } else { + if (!isAllowed) return null; + } + + const table = this.db.tables.find( + x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName + ); + return [ + { + operationType: 'recreateTable', + table, + operations: [op], + }, + ]; + } + return null; + } + transformPlan() { this.operations = this._addLogicalDependencies(); + this.operations = this._transformToImplementedOps(); } } @@ -246,5 +317,14 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor) case 'renameConstraint': processor.renameConstraint(op.object, op.newName); break; + case 'recreateTable': + { + const newTable = _.cloneDeep(op.table); + const newDb = DatabaseAnalyser.createEmptyStructure(); + newDb.tables.push(newTable); + op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb))); + processor.recreateTable(op.table, newTable); + } + break; } } diff --git a/packages/tools/src/database-info-alter-processor.ts b/packages/tools/src/database-info-alter-processor.ts index e8c304dc..829db4e7 100644 --- a/packages/tools/src/database-info-alter-processor.ts +++ b/packages/tools/src/database-info-alter-processor.ts @@ -66,4 +66,8 @@ export class DatabaseInfoAlterProcessor { } renameConstraint(constraint: ConstraintInfo, newName: string) {} + + recreateTable(oldTable: TableInfo, newTable: TableInfo) { + throw new Error('recreateTable not implemented for DatabaseInfoAlterProcessor'); + } } diff --git a/packages/types/alter-processor.d.ts b/packages/types/alter-processor.d.ts index 849fbe0a..c905b31c 100644 --- a/packages/types/alter-processor.d.ts +++ b/packages/types/alter-processor.d.ts @@ -12,4 +12,5 @@ export interface AlterProcessor { renameTable(table: TableInfo, newName: string); renameColumn(column: ColumnInfo, newName: string); renameConstraint(constraint: ConstraintInfo, newName: string); + recreateTable(oldTable: TableInfo, newTable: TableInfo); } diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index aa098530..32c9d782 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -10,7 +10,17 @@ export interface SqlDialect { explicitDropConstraint?: boolean; anonymousPrimaryKey?: boolean; enableConstraintsPerTable?: boolean; + nosql?: boolean; // mongo + dropColumnDependencies?: string[]; changeColumnDependencies?: string[]; - nosql?: boolean; // mongo + + createColumn?: boolean; + dropColumn?: boolean; + createIndex?: boolean; + dropIndex?: boolean; + createForeignKey?: boolean; + dropForeignKey?: boolean; + createPrimaryKey?: boolean; + dropPrimaryKey?: boolean; } diff --git a/plugins/dbgate-plugin-mssql/src/frontend/driver.js b/plugins/dbgate-plugin-mssql/src/frontend/driver.js index e27421fb..d7d0dc5a 100644 --- a/plugins/dbgate-plugin-mssql/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mssql/src/frontend/driver.js @@ -12,12 +12,21 @@ const dialect = { fallbackDataType: 'nvarchar(max)', explicitDropConstraint: false, enableConstraintsPerTable: true, - dropColumnDependencies: ['default', 'foreignKey', 'index'], - changeColumnDependencies: ['index'], + dropColumnDependencies: ['default', 'dependencies', 'indexes', 'primaryKey'], + changeColumnDependencies: ['indexes'], anonymousPrimaryKey: false, quoteIdentifier(s) { return `[${s}]`; }, + + createColumn: true, + dropColumn: true, + createIndex: true, + dropIndex: true, + createForeignKey: true, + dropForeignKey: true, + createPrimaryKey: true, + dropPrimaryKey: true, }; /** @type {import('dbgate-types').EngineDriver} */ diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 1e4c337e..36f50d87 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -13,6 +13,15 @@ const dialect = { quoteIdentifier(s) { return '`' + s + '`'; }, + + createColumn: true, + dropColumn: true, + createIndex: true, + dropIndex: true, + createForeignKey: true, + dropForeignKey: true, + createPrimaryKey: true, + dropPrimaryKey: true, }; const mysqlDriverBase = { diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index 91de9dd5..2fcc3630 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -35,6 +35,15 @@ const postgresDriver = { dialect: { ...dialect, materializedViews: true, + + createColumn: true, + dropColumn: true, + createIndex: true, + dropIndex: true, + createForeignKey: true, + dropForeignKey: true, + createPrimaryKey: true, + dropPrimaryKey: true, }, }; @@ -47,6 +56,7 @@ const cockroachDriver = { dialect: { ...dialect, materializedViews: true, + dropColumnDependencies: ['primaryKey'], }, }; diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js index 55eef977..d1f0fee2 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js @@ -17,10 +17,19 @@ const dialect = { explicitDropConstraint: true, stringEscapeChar: "'", fallbackDataType: 'nvarchar(max)', - dropColumnDependencies: ['index'], + dropColumnDependencies: ['index', 'primaryKey'], quoteIdentifier(s) { return `[${s}]`; }, + + createColumn: true, + dropColumn: true, + createIndex: true, + dropIndex: true, + createForeignKey: false, + dropForeignKey: false, + createPrimaryKey: false, + dropPrimaryKey: false, }; /** @type {import('dbgate-types').EngineDriver} */