db deploy fixes

packages/api/src/shell/dataTypeMapperTransform.js

@@ -0,0 +1,21 @@
+const dataTypeMapperTransform = (oldType, newType) => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        columns: table.columns.map(column => {
+          if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
+            return {
+              ...column,
+              dataType: newType,
+            };
+          }
+          return column;
+        }),
+      };
+    }),
+  };
+};
+
+module.exports = dataTypeMapperTransform;

packages/api/src/shell/deployDb.js

@@ -1,7 +1,15 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
 
-async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+async function deployDb({
+  connection,
+  systemConnection,
+  driver,
+  analysedStructure,
+  modelFolder,
+  loadedDbModel,
+  modelTransforms,
+}) {
   const { sql } = await generateDeploySql({
     connection,
     systemConnection,
@@ -9,9 +17,10 @@ async function deployDb({ connection, systemConnection, driver, analysedStructur
     analysedStructure,
     modelFolder,
     loadedDbModel,
+    modelTransforms,
   });
   // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql });
+  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
 }
 
 module.exports = deployDb;

packages/api/src/shell/executeQuery.js

@@ -1,11 +1,19 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');
 
-async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
+async function executeQuery({
-  logger.info({ sql }, `Execute query`);
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  sql,
+  logScriptItems = false,
+}) {
+  if (!logScriptItems) {
+    logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
+  }
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
@@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   try {
     logger.info(`Connected.`);
 
-    await driver.script(dbhan, sql);
+    await driver.script(dbhan, sql, { logScriptItems });
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

packages/api/src/shell/generateDeploySql.js

@@ -18,6 +18,7 @@ async function generateDeploySql({
   analysedStructure = undefined,
   modelFolder = undefined,
   loadedDbModel = undefined,
+  modelTransforms = undefined,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -28,9 +29,15 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
-    const deployedModel = generateDbPairingId(
+    let deployedModelSource = loadedDbModel
-      extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
+      ? databaseInfoFromYamlModel(loadedDbModel)
-    );
+      : await importDbModel(modelFolder);
+
+    for (const transform of modelTransforms || []) {
+      deployedModelSource = transform(deployedModelSource);
+    }
+
+    const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {
       ...modelCompareDbDiffOptions,

packages/api/src/shell/index.js

@@ -30,6 +30,8 @@ const dataDuplicator = require('./dataDuplicator');
 const dbModelToJson = require('./dbModelToJson');
 const jsonToDbModel = require('./jsonToDbModel');
 const jsonReader = require('./jsonReader');
+const dataTypeMapperTransform = require('./dataTypeMapperTransform');
+const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 
 const dbgateApi = {
   queryReader,
@@ -63,6 +65,8 @@ const dbgateApi = {
   dataDuplicator,
   dbModelToJson,
   jsonToDbModel,
+  dataTypeMapperTransform,
+  sqlTextReplacementTransform,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/sqlTextReplacementTransform.js

@@ -0,0 +1,28 @@
+function replaceInText(text, replacements) {
+  let result = text;
+  for (const key of Object.keys(replacements)) {
+    result = result.split(key).join(replacements[key]);
+  }
+  return result;
+}
+
+function replaceInCollection(collection, replacements) {
+  return collection.map(item => {
+    if (item.createSql) {
+      return {
+        ...item,
+        createSql: replaceInText(item.createSql, replacements),
+      };
+    }
+    return item;
+  });
+}
+
+const sqlTextReplacementTransform = replacements => database => {
+  return {
+    ...database,
+    views: replaceInCollection(database.views, replacements),
+  };
+};
+
+module.exports = sqlTextReplacementTransform;

packages/tools/package.json

@@ -37,6 +37,7 @@
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",
     "pinomin": "^1.0.4",
+    "toposort": "^2.0.2",
     "uuid": "^3.4.0"
   }
 }

packages/tools/src/SqlDumper.ts

@@ -686,7 +686,8 @@ export class SqlDumper implements AlterProcessor {
   }
 
   createSqlObject(obj: SqlObjectInfo) {
-    this.putCmd(obj.createSql);
+    this.putRaw(obj.createSql);
+    this.endCommand();
   }
 
   getSqlObjectSqlName(ojectTypeField: string) {

packages/tools/src/alterPlan.ts

@@ -498,22 +498,25 @@ export class AlterPlan {
       return this.operations;
     }
     const fks = [];
-    const res = this.operations.map(op => {
+    const res = this.operations
-      if (op.operationType == 'createTable') {
+      .filter(op => op.operationType != 'createConstraint')
-        fks.push(...(op.newObject.foreignKeys || []));
+      .map(op => {
-        return {
+        if (op.operationType == 'createTable') {
-          ...op,
+          fks.push(...(op.newObject.foreignKeys || []));
-          newObject: {
+          return {
-            ...op.newObject,
+            ...op,
-            foreignKeys: [],
+            newObject: {
-          },
+              ...op.newObject,
-        };
+              foreignKeys: [],
-      }
+            },
-      return op;
+          };
-    });
+        }
+        return op;
+      });
 
     return [
       ...res,
+      ...this.operations.filter(op => op.operationType == 'createConstraint'),
       ...fks.map(
         fk =>
           ({

packages/tools/src/diffTools.ts

@@ -7,6 +7,7 @@ import type {
   SqlDialect,
   SqlObjectInfo,
   TableInfo,
+  ViewInfo,
 } from 'dbgate-types';
 import uuidv1 from 'uuid/v1';
 import { AlterPlan } from './alterPlan';
@@ -15,6 +16,8 @@ import _omit from 'lodash/omit';
 import _cloneDeep from 'lodash/cloneDeep';
 import _isEqual from 'lodash/isEqual';
 import _pick from 'lodash/pick';
+import _compact from 'lodash/compact';
+import toposort from 'toposort';
 
 type DbDiffSchemaMode = 'strict' | 'ignore' | 'ignoreImplicit';
 
@@ -504,6 +507,40 @@ export function createAlterTablePlan(
   return plan;
 }
 
+function sortViewsByDependency(views: ViewInfo[]): ViewInfo[] {
+  const viewNames: string[] = [];
+  const viewDict: { [name: string]: string } = {};
+
+  for (const view of views) {
+    if (!viewNames.includes(view.pureName)) {
+      viewNames.push(view.pureName);
+    }
+    viewDict[view.pureName] = viewDict[view.pureName]
+      ? `${viewDict[view.pureName]} ${view.createSql}}`
+      : view.createSql;
+  }
+
+  const edges = [];
+  for (const viewName of viewNames) {
+    edges.push([viewName, null]);
+    const viewText = viewDict[viewName];
+    for (const otherView of viewNames) {
+      if (otherView === viewName) continue;
+      if ((' ' + viewText + ' ').match('[\\W]' + otherView + '[\\W]')) {
+        edges.push([otherView, viewName]);
+      }
+    }
+  }
+
+  const ordered: string[] = _compact(toposort(edges));
+
+  const res: ViewInfo[] = [];
+  for (const viewName of ordered) {
+    res.push(...views.filter(x => x.pureName == viewName));
+  }
+  return res;
+}
+
 export function createAlterDatabasePlan(
   oldDb: DatabaseInfo,
   newDb: DatabaseInfo,
@@ -539,7 +576,13 @@ export function createAlterDatabasePlan(
         }
       }
     }
-    for (const newobj of newDb[objectTypeField] || []) {
+
+    let newList = newDb[objectTypeField] || [];
+    if (objectTypeField == 'views') {
+      newList = sortViewsByDependency(newList);
+    }
+
+    for (const newobj of newList) {
       const oldobj = (oldDb[objectTypeField] || []).find(x => x.pairingId == newobj.pairingId);
       if (objectTypeField == 'tables') {
         if (oldobj == null) {

packages/tools/src/driverBase.ts

@@ -4,6 +4,10 @@ import { splitQuery } from 'dbgate-query-splitter';
 import { dumpSqlSelect } from 'dbgate-sqltree';
 import { EngineDriver, QueryResult, RunScriptOptions } from 'dbgate-types';
 import { detectSqlFilterBehaviour } from './detectSqlFilterBehaviour';
+import { getLogger } from './getLogger';
+import { getLimitedQuery } from './stringTools';
+
+const logger = getLogger('driverBase');
 
 const dialect = {
   limitSelect: true,
@@ -71,6 +75,9 @@ export const driverBase = {
     }
     for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
       try {
+        if (options?.logScriptItems) {
+          logger.info({ sql: getLimitedQuery(sqlItem as string) }, `Execute script item`);
+        }
         await this.query(pool, sqlItem, { discardResult: true, ...options?.queryOptions });
       } catch (err) {
         if (options?.useTransaction && this.supportsTransactions) {

packages/tools/src/stringTools.ts

@@ -511,3 +511,13 @@ export function safeFormatDate(date) {
     return date?.toString();
   }
 }
+
+export function getLimitedQuery(sql: string): string {
+  if (!sql) {
+    return sql;
+  }
+  if (sql.length > 1000) {
+    return sql.substring(0, 1000) + '...';
+  }
+  return sql;
+}

packages/tools/src/yamlModelConv.ts

@@ -74,10 +74,11 @@ function columnInfoFromYaml(column: ColumnInfoYaml, table: TableInfoYaml): Colum
   const res: ColumnInfo = {
     pureName: table.name,
     columnName: column.name,
-    dataType: column.length ? `${column.type}(${column.length})` : column.type,
+    dataType: column.length ? `${column.type}(${column.length < 0 ? 'max' : column.length})` : column.type,
     autoIncrement: column.autoIncrement,
     notNull: column.notNull || (table.primaryKey && table.primaryKey.includes(column.name)),
     defaultValue: column.default,
+    defaultConstraint: column.default != null ? `DF_${table.name}_${column.name}` : undefined,
   };
   return res;
 }
@@ -108,6 +109,7 @@ function convertForeignKeyFromYaml(
   if (!refTable || !refTable.primaryKey) return null;
   return {
     constraintType: 'foreignKey',
+    constraintName: `FK_${table.name}_${col.name}`,
     pureName: table.name,
     refTableName: col.references,
     deleteAction: col.refDeleteAction,
@@ -133,6 +135,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
     res.primaryKey = {
       pureName: table.name,
       constraintType: 'primaryKey',
+      constraintName: `PK_${table.name}`,
       columns: table.primaryKey.map(columnName => ({ columnName })),
     };
   }
@@ -140,6 +143,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
     res.sortingKey = {
       pureName: table.name,
       constraintType: 'sortingKey',
+      constraintName: `SK_${table.name}`,
       columns: table.sortingKey.map(columnName => ({ columnName })),
     };
   }

packages/types/engines.d.ts

@@ -25,6 +25,7 @@ export interface StreamOptions {
 
 export interface RunScriptOptions {
   useTransaction: boolean;
+  logScriptItems?: boolean;
   queryOptions?: QueryOptions;
 }
 

plugins/dbgate-plugin-mssql/src/frontend/MsSqlDumper.js

@@ -108,13 +108,13 @@ class MsSqlDumper extends SqlDumper {
   }
 
   guessDefaultName(col) {
-    return col.defaultConstraint || `DF${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
+    return col.defaultConstraint || `DF_${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
   }
 
   createDefault(col) {
-    if (!col.defaultValue) return;
+    if (col.defaultValue == null) return;
-    const defsql = col.defaultValue;
+    const defsql = col.defaultValue?.toString();
-    if (!defsql) {
+    if (defsql) {
       const defname = this.guessDefaultName(col);
       this.putCmd('^alter ^table %f ^add ^constraint %i ^default %s for %i', col, defname, defsql, col.columnName);
     }

test/.gitignore

@@ -0,0 +1,2 @@
+deployTest.js
+database

yarn.lock

@@ -11173,6 +11173,11 @@ toidentifier@1.0.1:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
+toposort@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/toposort/-/toposort-2.0.2.tgz#ae21768175d1559d48bef35420b2f4962f09c330"
+  integrity sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==
+
 totalist@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df"