sorting key support, clickhouse recreate table support

packages/tools/src/SqlDumper.ts

@@ -622,10 +622,8 @@ export class SqlDumper implements AlterProcessor {
     if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) {
       throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId');
     }
-    const tmpTable = `temp_${uuidv1()}`;
 
-    // console.log('oldTable', oldTable);
+    const tmpTable = `temp_${uuidv1()}`;
-    // console.log('newTable', newTable);
 
     const columnPairs = oldTable.columns
       .map(oldcol => ({
@@ -634,33 +632,49 @@ export class SqlDumper implements AlterProcessor {
       }))
       .filter(x => x.newcol);
 
-    this.dropConstraints(oldTable, true);
+    if (this.driver.supportsTransactions) {
-    this.renameTable(oldTable, tmpTable);
+      this.dropConstraints(oldTable, true);
+      this.renameTable(oldTable, tmpTable);
 
-    this.createTable(newTable);
+      this.createTable(newTable);
 
-    const autoinc = newTable.columns.find(x => x.autoIncrement);
+      const autoinc = newTable.columns.find(x => x.autoIncrement);
-    if (autoinc) {
+      if (autoinc) {
-      this.allowIdentityInsert(newTable, true);
+        this.allowIdentityInsert(newTable, true);
+      }
+
+      this.putCmd(
+        '^insert ^into %f (%,i) select %,s ^from %f',
+        newTable,
+        columnPairs.map(x => x.newcol.columnName),
+        columnPairs.map(x => x.oldcol.columnName),
+        { ...oldTable, pureName: tmpTable }
+      );
+
+      if (autoinc) {
+        this.allowIdentityInsert(newTable, false);
+      }
+
+      if (this.dialect.dropForeignKey) {
+        newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
+      }
+
+      this.dropTable({ ...oldTable, pureName: tmpTable });
+    } else {
+      // we have to preserve old table as long as possible
+      this.createTable({ ...newTable, pureName: tmpTable });
+
+      this.putCmd(
+        '^insert ^into %f (%,i) select %,s ^from %f',
+        { ...newTable, pureName: tmpTable },
+        columnPairs.map(x => x.newcol.columnName),
+        columnPairs.map(x => x.oldcol.columnName),
+        oldTable
+      );
+
+      this.dropTable(oldTable);
+      this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName);
     }
-
-    this.putCmd(
-      '^insert ^into %f (%,i) select %,s ^from %f',
-      newTable,
-      columnPairs.map(x => x.newcol.columnName),
-      columnPairs.map(x => x.oldcol.columnName),
-      { ...oldTable, pureName: tmpTable }
-    );
-
-    if (autoinc) {
-      this.allowIdentityInsert(newTable, false);
-    }
-
-    if (this.dialect.dropForeignKey) {
-      newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
-    }
-
-    this.dropTable({ ...oldTable, pureName: tmpTable });
   }
 
   createSqlObject(obj: SqlObjectInfo) {

packages/tools/src/alterPlan.ts

@@ -284,6 +284,7 @@ export class AlterPlan {
       : [];
     const constraints = _.compact([
       dependencyDefinition?.includes('primaryKey') ? table.primaryKey : null,
+      dependencyDefinition?.includes('sortingKey') ? table.sortingKey : null,
       ...(dependencyDefinition?.includes('foreignKeys') ? table.foreignKeys : []),
       ...(dependencyDefinition?.includes('indexes') ? table.indexes : []),
       ...(dependencyDefinition?.includes('uniques') ? table.uniques : []),
@@ -400,6 +401,7 @@ export class AlterPlan {
 
   _canCreateConstraint(cnt: ConstraintInfo) {
     if (cnt.constraintType == 'primaryKey') return this.dialect.createPrimaryKey;
+    if (cnt.constraintType == 'sortingKey') return this.dialect.createPrimaryKey;
     if (cnt.constraintType == 'foreignKey') return this.dialect.createForeignKey;
     if (cnt.constraintType == 'index') return this.dialect.createIndex;
     if (cnt.constraintType == 'unique') return this.dialect.createUnique;
@@ -409,6 +411,7 @@ export class AlterPlan {
 
   _canDropConstraint(cnt: ConstraintInfo) {
     if (cnt.constraintType == 'primaryKey') return this.dialect.dropPrimaryKey;
+    if (cnt.constraintType == 'sortingKey') return this.dialect.dropPrimaryKey;
     if (cnt.constraintType == 'foreignKey') return this.dialect.dropForeignKey;
     if (cnt.constraintType == 'index') return this.dialect.dropIndex;
     if (cnt.constraintType == 'unique') return this.dialect.dropUnique;

packages/tools/src/database-info-alter-processor.ts

@@ -11,6 +11,7 @@ import {
   UniqueInfo,
   SqlObjectInfo,
   NamedObjectInfo,
+  ColumnsConstraintInfo,
 } from '../../types';
 
 export class DatabaseInfoAlterProcessor {
@@ -59,6 +60,9 @@ export class DatabaseInfoAlterProcessor {
       case 'primaryKey':
         table.primaryKey = constraint as PrimaryKeyInfo;
         break;
+      case 'sortingKey':
+        table.sortingKey = constraint as ColumnsConstraintInfo;
+        break;
       case 'foreignKey':
         table.foreignKeys.push(constraint as ForeignKeyInfo);
         break;
@@ -86,6 +90,9 @@ export class DatabaseInfoAlterProcessor {
       case 'primaryKey':
         table.primaryKey = null;
         break;
+      case 'sortingKey':
+        table.sortingKey = null;
+        break;
       case 'foreignKey':
         table.foreignKeys = table.foreignKeys.filter(x => x.constraintName != constraint.constraintName);
         break;

packages/tools/src/diffTools.ts

@@ -46,6 +46,14 @@ export function generateTablePairingId(table: TableInfo): TableInfo {
   if (!table.pairingId) {
     return {
       ...table,
+      primaryKey: table.primaryKey && {
+        ...table.primaryKey,
+        pairingId: table.primaryKey.pairingId || uuidv1(),
+      },
+      sortingKey: table.sortingKey && {
+        ...table.sortingKey,
+        pairingId: table.sortingKey.pairingId || uuidv1(),
+      },
       columns: table.columns?.map(col => ({
         ...col,
         pairingId: col.pairingId || uuidv1(),
@@ -335,6 +343,7 @@ export function testEqualTypes(a: ColumnInfo, b: ColumnInfo, opts: DbDiffOptions
 function getTableConstraints(table: TableInfo) {
   const res = [];
   if (table.primaryKey) res.push(table.primaryKey);
+  if (table.sortingKey) res.push(table.sortingKey);
   if (table.foreignKeys) res.push(...table.foreignKeys);
   if (table.indexes) res.push(...table.indexes);
   if (table.uniques) res.push(...table.uniques);
@@ -345,7 +354,9 @@ function getTableConstraints(table: TableInfo) {
 function createPairs(oldList, newList, additionalCondition = null) {
   const res = [];
   for (const a of oldList) {
-    const b = newList.find(x => x.pairingId == a.pairingId || (additionalCondition && additionalCondition(a, x)));
+    const b = newList.find(
+      x => (a.pairingId && x.pairingId == a.pairingId) || (additionalCondition && additionalCondition(a, x))
+    );
     if (b) {
       res.push([a, b]);
     } else {
@@ -381,9 +392,14 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
   const constraintPairs = createPairs(
     getTableConstraints(oldTable),
     getTableConstraints(newTable),
-    (a, b) => a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey'
+    (a, b) =>
+      (a.constraintType == 'primaryKey' && b.constraintType == 'primaryKey') ||
+      (a.constraintType == 'sortingKey' && b.constraintType == 'sortingKey')
   );
-  // console.log('constraintPairs SOURCE', getTableConstraints(oldTable), getTableConstraints(newTable));
+  // console.log('constraintPairs OLD TABLE', oldTable);
+  // console.log('constraintPairs NEW TABLE', newTable);
+  // console.log('constraintPairs SOURCE OLD', getTableConstraints(oldTable));
+  // console.log('constraintPairs SOURCE NEW', getTableConstraints(newTable));
   // console.log('constraintPairs', constraintPairs);
 
   if (!opts.noDropConstraint) {
@@ -427,6 +443,10 @@ function planAlterTable(plan: AlterPlan, oldTable: TableInfo, newTable: TableInf
   planTablePreload(plan, oldTable, newTable);
 
   planChangeTableOptions(plan, oldTable, newTable, opts);
+
+  // console.log('oldTable', oldTable);
+  // console.log('newTable', newTable);
+  // console.log('plan.operations', plan.operations);
 }
 
 function planChangeTableOptions(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo, opts: DbDiffOptions) {

packages/tools/src/schemaEditorTools.ts

@@ -2,6 +2,7 @@ import uuidv1 from 'uuid/v1';
 import _omit from 'lodash/omit';
 import type {
   ColumnInfo,
+  ColumnsConstraintInfo,
   ConstraintInfo,
   ForeignKeyInfo,
   IndexInfo,
@@ -195,6 +196,13 @@ export function editorAddConstraint(table: TableInfo, constraint: ConstraintInfo
     } as PrimaryKeyInfo;
   }
 
+  if (constraint.constraintType == 'sortingKey') {
+    res.sortingKey = {
+      pairingId: uuidv1(),
+      ...constraint,
+    } as ColumnsConstraintInfo;
+  }
+
   if (constraint.constraintType == 'foreignKey') {
     res.foreignKeys = [
       ...(res.foreignKeys || []),
@@ -240,6 +248,13 @@ export function editorModifyConstraint(table: TableInfo, constraint: ConstraintI
     };
   }
 
+  if (constraint.constraintType == 'sortingKey') {
+    res.sortingKey = {
+      ...res.sortingKey,
+      ...constraint,
+    };
+  }
+
   if (constraint.constraintType == 'foreignKey') {
     res.foreignKeys = table.foreignKeys.map(fk =>
       fk.pairingId == constraint.pairingId ? { ...fk, ...constraint } : fk

packages/tools/src/yamlModelConv.ts

@@ -27,6 +27,7 @@ export interface TableInfoYaml {
   // schema?: string;
   columns: ColumnInfoYaml[];
   primaryKey?: string[];
+  sortingKey?: string[];
 
   insertKey?: string[];
   insertOnly?: string[];
@@ -91,6 +92,9 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
   if (tableCopy.primaryKey && !tableCopy.primaryKey['_dumped']) {
     res.primaryKey = tableCopy.primaryKey.columns.map(x => x.columnName);
   }
+  if (tableCopy.sortingKey && !tableCopy.sortingKey['_dumped']) {
+    res.sortingKey = tableCopy.sortingKey.columns.map(x => x.columnName);
+  }
   // const foreignKeys = (tableCopy.foreignKeys || []).filter(x => !x['_dumped']).map(foreignKeyInfoToYaml);
   return res;
 }
@@ -132,6 +136,13 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
       columns: table.primaryKey.map(columnName => ({ columnName })),
     };
   }
+  if (table.sortingKey) {
+    res.sortingKey = {
+      pureName: table.name,
+      constraintType: 'sortingKey',
+      columns: table.sortingKey.map(columnName => ({ columnName })),
+    };
+  }
   res.preloadedRows = table.data;
   res.preloadedRowsKey = table.insertKey;
   res.preloadedRowsInsertOnly = table.insertOnly;

plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js

@@ -46,10 +46,10 @@ class Analyser extends DatabaseAnalyser {
             ...extractDataType(col.dataType),
           })),
         primaryKey: table.primaryKeyColumns
-          ? { columns: (table.primaryKeyColumns || '').split(',').map((columnName) => ({ columnName })) }
+          ? { columns: (table.primaryKeyColumns || '').split(',').map((x) => ({ columnName: x.trim() })) }
           : null,
         sortingKey: table.sortingKeyColumns
-          ? { columns: (table.sortingKeyColumns || '').split(',').map((columnName) => ({ columnName })) }
+          ? { columns: (table.sortingKeyColumns || '').split(',').map((x) => ({ columnName: x.trim() })) }
           : null,
         foreignKeys: [],
       })),

plugins/dbgate-plugin-clickhouse/src/frontend/Dumper.js

@@ -23,6 +23,20 @@ class Dumper extends SqlDumper {
   renameColumn(column, newcol) {
     this.putCmd('^alter ^table %f ^rename ^column %i ^to %i', column, column.columnName, newcol);
   }
+
+  renameTable(obj, newName) {
+    this.putCmd('^rename ^table %f ^to %i', obj, newName);
+  }
+
+  tableOptions(table) {
+    super.tableOptions(table);
+    if (table.sortingKey) {
+      this.put(
+        '&n^order ^by (%,i)',
+        table.sortingKey.columns.map((x) => x.columnName)
+      );
+    }
+  }
 }
 
 module.exports = Dumper;