sql generator

packages/api/package.json

@@ -49,7 +49,7 @@
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "nodemon src/index.js",
+    "start": "node src/index.js",
     "start:portal": "env-cmd nodemon src/index.js",
     "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
     "ts": "tsc",

packages/tools/src/SqlDumper.ts

@@ -1,11 +1,20 @@
 import {
   ColumnInfo,
+  ConstraintInfo,
   EngineDriver,
   ForeignKeyInfo,
+  FunctionInfo,
   NamedObjectInfo,
+  PrimaryKeyInfo,
+  ProcedureInfo,
   SqlDialect,
   TableInfo,
   TransformType,
+  TriggerInfo,
+  ViewInfo,
+  IndexInfo,
+  UniqueInfo,
+  CheckInfo,
 } from 'dbgate-types';
 import _isString from 'lodash/isString';
 import _isNumber from 'lodash/isNumber';
@@ -261,4 +270,189 @@ export class SqlDumper {
   }
 
   allowIdentityInsert(table: NamedObjectInfo, allow: boolean) {}
+  enableConstraints(table: NamedObjectInfo, enabled: boolean) {}
+
+  comment(value: string) {
+    if (!value) return;
+    for (const line of value.split('\n')) {
+      this.put(' -- %s', line.trimRight());
+    }
+  }
+
+  createView(obj: ViewInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropView(obj: ViewInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^view  %f', obj);
+  }
+  alterView(obj: ViewInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW'));
+    this.endCommand();
+  }
+  changeViewSchema(obj: ViewInfo, newSchema: string) {}
+  renameView(obj: ViewInfo, newSchema: string) {}
+
+  createProcedure(obj: ProcedureInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropProcedure(obj: ProcedureInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^procedure  %f', obj);
+  }
+  alterProcedure(obj: ProcedureInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+procedure/i, 'ALTER PROCEDURE'));
+    this.endCommand();
+  }
+  changeProcedureSchema(obj: ProcedureInfo, newSchema: string) {}
+  renameProcedure(obj: ProcedureInfo, newSchema: string) {}
+
+  createFunction(obj: FunctionInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropFunction(obj: FunctionInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^function  %f', obj);
+  }
+  alterFunction(obj: FunctionInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+function/i, 'ALTER FUNCTION'));
+    this.endCommand();
+  }
+  changeFunctionSchema(obj: FunctionInfo, newSchema: string) {}
+  renameFunction(obj: FunctionInfo, newSchema: string) {}
+
+  createTrigger(obj: TriggerInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropTrigger(obj: TriggerInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^trigger  %f', obj);
+  }
+  alterTrigger(obj: TriggerInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+trigger/i, 'ALTER TRIGGER'));
+    this.endCommand();
+  }
+  changeTriggerSchema(obj: TriggerInfo, newSchema: string) {}
+  renameTrigger(obj: TriggerInfo, newSchema: string) {}
+
+  dropConstraint(cnt: ConstraintInfo) {
+    this.putCmd('^alter ^table %f ^drop ^constraint %i', cnt, cnt.constraintName);
+  }
+  dropForeignKey(fk: ForeignKeyInfo) {
+    if (this.dialect.explicitDropConstraint) {
+      this.putCmd('^alter ^table %f ^drop ^foreign ^key %i', fk, fk.constraintName);
+    } else {
+      this.dropConstraint(fk);
+    }
+  }
+  createForeignKey(fk: ForeignKeyInfo) {
+    this.put('^alter ^table %f ^add ', fk);
+    this.createForeignKeyFore(fk);
+    this.endCommand();
+  }
+  dropPrimaryKey(pk: PrimaryKeyInfo) {
+    if (this.dialect.explicitDropConstraint) {
+      this.putCmd('^alter ^table %f ^drop ^primary ^key', pk);
+    } else {
+      this.dropConstraint(pk);
+    }
+  }
+  createPrimaryKey(pk: PrimaryKeyInfo) {
+    this.putCmd(
+      '^alter ^table %f ^add ^constraint %i ^primary ^key (%,i)',
+      pk,
+      pk.constraintName,
+      pk.columns.map(x => x.columnName)
+    );
+  }
+
+  dropIndex(ix: IndexInfo) {}
+  createIndex(ix: IndexInfo) {}
+
+  dropUnique(uq: UniqueInfo) {
+    this.dropConstraint(uq);
+  }
+  createUniqueCore(uq: UniqueInfo) {
+    this.put(
+      '^constraint %i ^unique (%,i)',
+      uq.constraintName,
+      uq.columns.map(x => x.columnName)
+    );
+  }
+
+  createUnique(uq: UniqueInfo) {
+    this.put('^alter ^table %f ^add ', uq);
+    this.createUniqueCore(uq);
+    this.endCommand();
+  }
+
+  dropCheck(ch: CheckInfo) {
+    this.dropConstraint(ch);
+  }
+
+  createCheckCore(ch: CheckInfo) {
+    this.put('^constraint %i ^check (%s)', ch.constraintName, ch.definition);
+  }
+
+  createCheck(ch: CheckInfo) {
+    this.put('^alter ^table %f ^add ', ch);
+    this.createCheckCore(ch);
+    this.endCommand();
+  }
+
+  renameConstraint(constraint: ConstraintInfo, newName: string) {}
+
+  createColumn(table: TableInfo, column: ColumnInfo, constraints: ConstraintInfo[]) {
+    this.put('^alter ^table %f ^add %i ', table, column.columnName);
+    this.columnDefinition(column);
+    this.inlineConstraints(constraints);
+    this.endCommand();
+  }
+
+  inlineConstraints(constrains: ConstraintInfo[]) {
+    if (constrains == null) return;
+    for (const cnt of constrains) {
+      if (cnt.constraintType == 'primaryKey') {
+        if (cnt.constraintName != null && !this.dialect.anonymousPrimaryKey) {
+          this.put(' ^constraint %i', cnt.constraintName);
+        }
+        this.put(' ^primary ^key ');
+      }
+    }
+  }
+
+  dropColumn(column: ColumnInfo) {
+    this.putCmd('^alter ^table %f ^drop ^column %i', column, column.columnName);
+  }
+
+  renameColumn(column: ColumnInfo, newName: string) {}
+
+  changeColumn(oldcol: ColumnInfo, newcol: ColumnInfo, constraints: ConstraintInfo[]) {}
+
+  dropTable(obj: TableInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^table %f', obj);
+  }
+
+  changeTableSchema(obj: TableInfo, schema: string) {}
+
+  renameTable(obj: TableInfo, newname: string) {}
+
+  beginTransaction() {
+    this.putCmd('^begin ^transaction');
+  }
+
+  commitTransaction() {
+    this.putCmd('^commit');
+  }
+
+  alterProlog() {}
+  alterEpilog() {}
+
+  selectTableIntoNewTable(sourceName: NamedObjectInfo, targetName: NamedObjectInfo) {
+    this.putCmd('^select * ^into %f ^from %f', targetName, sourceName);
+  }
+
+  truncateTable(name: NamedObjectInfo) {
+    this.putCmd('^delete ^from %f', name);
+  }
 }

packages/tools/src/SqlGenerator.ts

@@ -1,5 +1,15 @@
-import { DatabaseInfo, EngineDriver, FunctionInfo, ProcedureInfo, TableInfo, ViewInfo } from 'dbgate-types';
+import {
+  DatabaseInfo,
+  EngineDriver,
+  FunctionInfo,
+  ProcedureInfo,
+  TableInfo,
+  TriggerInfo,
+  ViewInfo,
+} from 'dbgate-types';
+import _ from 'lodash';
 import { SqlDumper } from './SqlDumper';
+import { extendDatabaseInfo } from './structureTools';
 
 interface SqlGeneratorOptions {
   dropTables: boolean;
@@ -14,6 +24,22 @@ interface SqlGeneratorOptions {
   disableConstraints: boolean;
   omitNulls: boolean;
   truncate: boolean;
+
+  dropViews: boolean;
+  checkIfViewExists: boolean;
+  createViews: boolean;
+
+  dropProcedures: boolean;
+  checkIfProcedureExists: boolean;
+  createProcedures: boolean;
+
+  dropFunctions: boolean;
+  checkIfFunctionExists: boolean;
+  createFunctions: boolean;
+
+  dropTriggers: boolean;
+  checkIfTriggerExists: boolean;
+  createTriggers: boolean;
 }
 
 interface SqlGeneratorObject {
@@ -27,57 +53,177 @@ export class SqlGenerator {
   private views: ViewInfo[];
   private procedures: ProcedureInfo[];
   private functions: FunctionInfo[];
+  private triggers: TriggerInfo[];
+  public dbinfo: DatabaseInfo;
+
   constructor(
-    public dbinfo: DatabaseInfo,
+    dbinfo: DatabaseInfo,
     public options: SqlGeneratorOptions,
     public objects: SqlGeneratorObject[],
     public dmp: SqlDumper,
     public driver: EngineDriver,
     public pool
   ) {
+    this.dbinfo = extendDatabaseInfo(dbinfo);
     this.tables = this.extract('tables');
     this.views = this.extract('views');
     this.procedures = this.extract('procedures');
     this.functions = this.extract('functions');
+    this.triggers = this.extract('triggers');
+  }
+
+  async dump() {
+    this.dropObjects(this.procedures, 'Procedure');
+    if (this.checkDumper()) return;
+    this.dropObjects(this.functions, 'Function');
+    if (this.checkDumper()) return;
+    this.dropObjects(this.views, 'View');
+    if (this.checkDumper()) return;
+    this.dropObjects(this.triggers, 'Trigger');
+    if (this.checkDumper()) return;
+
+    this.dropTables();
+    if (this.checkDumper()) return;
+
+    this.createTables();
+    if (this.checkDumper()) return;
+
+    this.truncateTables();
+    if (this.checkDumper()) return;
+
+    await this.insertData();
+    if (this.checkDumper()) return;
+
+    this.createForeignKeys();
+    if (this.checkDumper()) return;
+
+    this.createObjects(this.procedures, 'Procedure');
+    if (this.checkDumper()) return;
+    this.createObjects(this.functions, 'Function');
+    if (this.checkDumper()) return;
+    this.createObjects(this.views, 'View');
+    if (this.checkDumper()) return;
+    this.createObjects(this.triggers, 'Trigger');
+    if (this.checkDumper()) return;
+  }
+
+  createForeignKeys() {
+    const fks = [];
+    if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || [])));
+    if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || [])));
+    for (const fk of _.uniqBy(fks, 'constraintName')) {
+      this.dmp.createForeignKey(fk);
+      if (this.checkDumper()) return;
+    }
+  }
+
+  truncateTables() {
+    if (this.options.truncate) {
+      for (const table of this.tables) {
+        this.dmp.truncateTable(table);
+        if (this.checkDumper()) return;
+      }
+    }
+  }
+
+  createTables() {
+    if (this.options.createTables) {
+      for (const table of this.tables) {
+        this.dmp.createTable({
+          ...table,
+          foreignKeys: [],
+          dependencies: [],
+          indexes: [],
+        });
+        if (this.checkDumper()) return;
+      }
+    }
+    if (this.options.createIndexes) {
+      for (const index of _.flatten(this.tables.map(x => x.indexes || []))) {
+        this.dmp.createIndex(index);
+      }
+    }
+  }
+
+  async insertData() {
+    if (!this.options.insert) return;
+
+    this.enableConstraints(false);
+
+    for (const table of this.tables) {
+      await this.insertTableData(table);
+      if (this.checkDumper()) return;
+    }
+
+    this.enableConstraints(true);
   }
 
   checkDumper() {
     return false;
   }
 
-  async dump() {
-    if (this.options.createTables) {
-      for (const table of this.tables) {
-        this.dmp.createTable(table);
+  dropObjects(list, name) {
+    if (this.options[`drop${name}s`]) {
+      for (const item of list) {
+        this.dmp[`drop${name}`](item, { testIfExists: this.options[`checkIf${name}Exists`] });
         if (this.checkDumper()) return;
       }
     }
-    if (this.options.insert) {
-      for (const table of this.tables) {
-        await this.insertTableData(table);
+  }
+
+  createObjects(list, name) {
+    if (this.options[`create${name}s`]) {
+      for (const item of list) {
+        this.dmp[`create${name}`](item);
         if (this.checkDumper()) return;
       }
     }
   }
 
+  dropTables() {
+    if (this.options.dropReferences) {
+      for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) {
+        this.dmp.dropForeignKey(fk);
+      }
+    }
+
+    if (this.options.dropTables) {
+      for (const table of this.tables) {
+        this.dmp.dropTable(table, { testIfExists: this.options.checkIfTableExists });
+      }
+    }
+  }
+
   async insertTableData(table: TableInfo) {
-    const dmp = this.driver.createDumper();
-    dmp.put('^select * ^from %f', table);
-    const readable = await this.driver.readQuery(this.pool, dmp.s, table);
+    const dmpLocal = this.driver.createDumper();
+    dmpLocal.put('^select * ^from %f', table);
+
+    const autoinc = table.columns.find(x => x.autoIncrement);
+    if (autoinc && !this.options.skipAutoincrementColumn) {
+      this.dmp.allowIdentityInsert(table, true);
+    }
+
+    const readable = await this.driver.readQuery(this.pool, dmpLocal.s, table);
     await this.processReadable(table, readable);
+
+    if (autoinc && !this.options.skipAutoincrementColumn) {
+      this.dmp.allowIdentityInsert(table, false);
+    }
   }
 
   processReadable(table: TableInfo, readable) {
-    const columnNames = table.columns.map(x => x.columnName);
+    const columnsFiltered = this.options.skipAutoincrementColumn
+      ? table.columns.filter(x => !x.autoIncrement)
+      : table.columns;
+    const columnNames = columnsFiltered.map(x => x.columnName);
     return new Promise(resolve => {
       readable.on('data', chunk => {
-        // const chunk = readable.read();
-        // if (!chunk) return;
+        const columnNamesCopy = this.options.omitNulls ? columnNames.filter(col => chunk[col] != null) : columnNames;
         this.dmp.put(
           '^insert ^into %f (%,i) ^values (%,v);&n',
           table,
-          columnNames,
-          columnNames.map(col => chunk[col])
+          columnNamesCopy,
+          columnNamesCopy.map(col => chunk[col])
         );
       });
       readable.on('end', () => {
@@ -93,4 +239,16 @@ export class SqlGenerator {
       )
     );
   }
+
+  enableConstraints(enabled) {
+    if (this.options.disableConstraints) {
+      if (this.driver.dialect.enableConstraintsPerTable) {
+        for (const table of this.tables) {
+          this.dmp.enableConstraints(table, enabled);
+        }
+      } else {
+        this.dmp.enableConstraints(null, enabled);
+      }
+    }
+  }
 }

packages/tools/src/index.ts

@@ -9,3 +9,4 @@ export * from './SqlDumper';
 export * from './testPermission';
 export * from './splitPostgresQuery';
 export * from './SqlGenerator';
+export * from './structureTools';

packages/tools/src/structureTools.ts

@@ -0,0 +1,31 @@
+import { DatabaseInfo } from 'dbgate-types';
+import _ from 'lodash';
+
+export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
+  const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys));
+  return {
+    ...db,
+    tables: db.tables.map(table => ({
+      ...table,
+      dependencies: allForeignKeys.filter(x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName),
+    })),
+  };
+}
+
+function fillTableExtendedInfo(db: DatabaseInfo) {
+  return {
+    ...db,
+    tables: db.tables.map(table => ({
+      ...table,
+      columns: (table.columns || []).map(column => ({
+        pureName: table.pureName,
+        schemaName: table.schemaName,
+        ...column,
+      })),
+    })),
+  };
+}
+
+export function extendDatabaseInfo(db: DatabaseInfo): DatabaseInfo {
+  return fillTableExtendedInfo(addTableDependencies(db));
+}

packages/types/dbinfo.d.ts

@@ -10,7 +10,7 @@ export interface ColumnReference {
 
 export interface ConstraintInfo extends NamedObjectInfo {
   constraintName: string;
-  constraintType: string;
+  constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique';
 }
 
 export interface ColumnsConstraintInfo extends ConstraintInfo {
@@ -26,7 +26,18 @@ export interface ForeignKeyInfo extends ColumnsConstraintInfo {
   deleteAction: string;
 }
 
-export interface ColumnInfo {
+export interface IndexInfo extends ColumnsConstraintInfo {
+  isUnique: boolean;
+  indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext';
+}
+
+export interface UniqueInfo extends ColumnsConstraintInfo {}
+
+export interface CheckInfo extends ConstraintInfo {
+  definition: string;
+}
+
+export interface ColumnInfo extends NamedObjectInfo {
   columnName: string;
   notNull: boolean;
   autoIncrement: boolean;
@@ -58,6 +69,8 @@ export interface TableInfo extends DatabaseObjectInfo {
   primaryKey?: PrimaryKeyInfo;
   foreignKeys: ForeignKeyInfo[];
   dependencies?: ForeignKeyInfo[];
+  indexes?: IndexInfo[];
+  checks?: CheckInfo[];
 }
 
 export interface ViewInfo extends SqlObjectInfo {

packages/types/dialect.d.ts

@@ -5,4 +5,7 @@ export interface SqlDialect {
   offsetFetchRangeSyntax?: boolean;
   quoteIdentifier(s: string): string;
   fallbackDataType?: string;
+  explicitDropConstraint?: boolean;
+  anonymousPrimaryKey?: boolean;
+  enableConstraintsPerTable?: boolean;
 }

packages/web/src/modals/SqlGeneratorModal.svelte

@@ -3,6 +3,7 @@
   import { writable } from 'svelte/store';
   import AppObjectList from '../appobj/AppObjectList.svelte';
   import * as databaseObjectAppObject from '../appobj/DatabaseObjectAppObject.svelte';
+  import uuidv1 from 'uuid/v1';
 
   import HorizontalSplitter from '../elements/HorizontalSplitter.svelte';
 
@@ -18,6 +19,7 @@
   import FontIcon from '../icons/FontIcon.svelte';
   import SqlEditor from '../query/SqlEditor.svelte';
   import axiosInstance from '../utility/axiosInstance';
+  import createRef from '../utility/createRef';
   import { useDatabaseInfo } from '../utility/metadataLoaders';
   import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte';
   import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte';
@@ -35,7 +37,11 @@
   let sqlPreview = '';
 
   const checkedObjectsStore = writable([]);
-  const valuesStore = writable({});
+  const valuesStore = writable({
+    checkIfTableExists: true,
+    disableConstraints: true,
+  });
+  const loadRef = createRef(null);
 
   $: console.log('checkedObjectsStore', $checkedObjectsStore);
 
@@ -53,8 +59,19 @@
   );
 
   async function generatePreview(options, objects) {
+    const loadid = uuidv1();
+    loadRef.set(loadid);
     busy = true;
-    const response = await axiosInstance.post('database-connections/sql-preview', { conid, database, objects, options });
+    const response = await axiosInstance.post('database-connections/sql-preview', {
+      conid,
+      database,
+      objects,
+      options,
+    });
+    if (loadRef.get() != loadid) {
+      // newer load exists
+      return;
+    }
     if (_.isString(response.data)) {
       sqlPreview = response.data;
     }
@@ -96,8 +113,8 @@
               <FormValues let:values>
                 <FormCheckboxField label="Drop tables" name="dropTables" />
                 {#if values.dropTables}
-                  <div class="ml-1">
-                    <FormCheckboxField label="Test if exists" name="checkIfTableExists" defaultValue={true} />
+                  <div class="ml-2">
+                    <FormCheckboxField label="Test if exists" name="checkIfTableExists" />
                   </div>
                 {/if}
                 <FormCheckboxField label="Drop references" name="dropReferences" />
@@ -109,9 +126,9 @@
 
                 <FormCheckboxField label="Insert" name="insert" />
                 {#if values.insert}
-                  <div class="ml-1">
+                  <div class="ml-2">
                     <FormCheckboxField label="Skip autoincrement column" name="skipAutoincrementColumn" />
-                    <FormCheckboxField label="Disable constraints" name="disableConstraints" defaultValue={true} />
+                    <FormCheckboxField label="Disable constraints" name="disableConstraints" />
                     <FormCheckboxField label="Omit NULL values" name="omitNulls" />
                   </div>
                 {/if}

packages/web/src/utility/metadataLoaders.ts

@@ -7,23 +7,13 @@ import socket from './socket';
 import getAsArray from './getAsArray';
 import { DatabaseInfo } from 'dbgate-types';
 import { derived } from 'svelte/store';
+import { extendDatabaseInfo } from 'dbgate-tools';
 
 const databaseInfoLoader = ({ conid, database }) => ({
   url: 'database-connections/structure',
   params: { conid, database },
   reloadTrigger: `database-structure-changed-${conid}-${database}`,
-  transform: (db: DatabaseInfo) => {
-    const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys));
-    return {
-      ...db,
-      tables: db.tables.map(table => ({
-        ...table,
-        dependencies: allForeignKeys.filter(
-          x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName
-        ),
-      })),
-    };
-  },
+  transform: extendDatabaseInfo,
 });
 
 // const tableInfoLoader = ({ conid, database, schemaName, pureName }) => ({