improved modification detection algorithm - for mssql

packages/tools/src/DatabaseAnalyser.ts

@@ -2,6 +2,7 @@ import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types';
 import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
+import _ from 'lodash';
 
 const fp_pick = arg => array => _pick(array, arg);
 export class DatabaseAnalyser {
@@ -16,17 +17,15 @@ export class DatabaseAnalyser {
     return DatabaseAnalyser.createEmptyStructure();
   }
 
-  async _computeSingleObjectId() {}
-
-  /** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
-  async getModifications() {
-    if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
-
+  async _getFastSnapshot(): Promise<DatabaseInfo> {
     return null;
   }
 
+  async _computeSingleObjectId() {}
+
   async fullAnalysis() {
-    return this._runAnalysis();
+    const res = await this._runAnalysis();
+    return res;
   }
 
   async singleObjectAnalysis(name, typeField) {
@@ -49,7 +48,7 @@ export class DatabaseAnalyser {
     }
     if (this.modifications.length == 0) return null;
     console.log('DB modifications detected:', this.modifications);
-    return this._runAnalysis();
+    return this.mergeAnalyseResult(await this._runAnalysis());
   }
 
   mergeAnalyseResult(newlyAnalysed) {
@@ -69,7 +68,7 @@ export class DatabaseAnalyser {
       const addedChangedIds = newArray.map(x => x.objectId);
       const removeAllIds = [...removedIds, ...addedChangedIds];
       res[field] = _sortBy(
-        [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
+        [...(this.structure[field] || []).filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
         x => x.pureName
       );
     }
@@ -109,6 +108,11 @@ export class DatabaseAnalyser {
     if (!this.modifications || !typeFields || this.modifications.length == 0) {
       res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
     } else {
+      if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) {
+        // do not filter objects
+        res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+      }
+
       const filterIds = this.modifications
         .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
         .map(x => x.objectId);
@@ -121,6 +125,72 @@ export class DatabaseAnalyser {
     return res;
   }
 
+  getDeletedObjectsForField(snapshot, objectTypeField) {
+    const items = snapshot[objectTypeField];
+    if (!items) return [];
+    if (!this.structure[objectTypeField]) return [];
+    return this.structure[objectTypeField]
+      .filter(x => !items.find(y => x.objectId == y.objectId))
+      .map(x => ({
+        oldName: _.pick(x, ['schemaName', 'pureName']),
+        objectId: x.objectId,
+        action: 'remove',
+        objectTypeField,
+      }));
+  }
+
+  getDeletedObjects(snapshot) {
+    return [
+      ...this.getDeletedObjectsForField(snapshot, 'tables'),
+      ...this.getDeletedObjectsForField(snapshot, 'collections'),
+      ...this.getDeletedObjectsForField(snapshot, 'views'),
+      ...this.getDeletedObjectsForField(snapshot, 'procedures'),
+      ...this.getDeletedObjectsForField(snapshot, 'functions'),
+      ...this.getDeletedObjectsForField(snapshot, 'triggers'),
+    ];
+  }
+
+  async getModifications() {
+    const snapshot = await this._getFastSnapshot();
+    if (!snapshot) return null;
+
+    // console.log('STRUCTURE', this.structure);
+    // console.log('SNAPSHOT', snapshot);
+
+    const res = [];
+    for (const field in snapshot) {
+      const items = snapshot[field];
+      if (items === null) {
+        res.push({ objectTypeField: field, action: 'all' });
+        continue;
+      }
+      for (const item of items) {
+        const { objectId, schemaName, pureName, contentHash } = item;
+        const obj = this.structure[field].find(x => x.objectId == objectId);
+
+        if (obj && contentHash && obj.contentHash == contentHash) continue;
+
+        const action = obj
+          ? {
+              newName: { schemaName, pureName },
+              oldName: _.pick(obj, ['schemaName', 'pureName']),
+              action: 'change',
+              objectTypeField: field,
+              objectId,
+            }
+          : {
+              newName: { schemaName, pureName },
+              action: 'add',
+              objectTypeField: field,
+              objectId,
+            };
+        res.push(action);
+      }
+
+      return [..._.compact(res), ...this.getDeletedObjects(snapshot)];
+    }
+  }
+
   static createEmptyStructure(): DatabaseInfo {
     return {
       tables: [],

packages/types/engines.d.ts

@@ -78,6 +78,6 @@ export interface DatabaseModification {
   oldName?: NamedObjectInfo;
   newName?: NamedObjectInfo;
   objectId?: string;
-  action: 'add' | 'remove' | 'change';
+  action: 'add' | 'remove' | 'change' | 'all';
   objectTypeField: keyof DatabaseInfo;
 }

plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js

@@ -88,6 +88,7 @@ class MsSqlAnalyser extends DatabaseAnalyser {
 
     const tables = tablesRows.rows.map(row => ({
       ...row,
+      contentHash: row.modifyDate.toISOString(),
       columns: columnsRows.rows.filter(col => col.objectId == row.objectId).map(getColumnInfo),
       primaryKey: DatabaseAnalyser.extractPrimaryKeys(row, pkColumnsRows.rows),
       foreignKeys: DatabaseAnalyser.extractForeignKeys(row, fkColumnsRows.rows),
@@ -95,6 +96,7 @@ class MsSqlAnalyser extends DatabaseAnalyser {
 
     const views = viewsRows.rows.map(row => ({
       ...row,
+      contentHash: row.modifyDate.toISOString(),
       createSql: getCreateSql(row),
       columns: viewColumnRows.rows.filter(col => col.objectId == row.objectId).map(getColumnInfo),
     }));
@@ -103,6 +105,7 @@ class MsSqlAnalyser extends DatabaseAnalyser {
       .filter(x => x.sqlObjectType.trim() == 'P')
       .map(row => ({
         ...row,
+        contentHash: row.modifyDate.toISOString(),
         createSql: getCreateSql(row),
       }));
 
@@ -110,75 +113,36 @@ class MsSqlAnalyser extends DatabaseAnalyser {
       .filter(x => ['FN', 'IF', 'TF'].includes(x.sqlObjectType.trim()))
       .map(row => ({
         ...row,
+        contentHash: row.modifyDate.toISOString(),
         createSql: getCreateSql(row),
       }));
 
-    return this.mergeAnalyseResult({
+    return {
       tables,
       views,
       procedures,
       functions,
       schemas,
-    });
+    };
   }
 
-  getDeletedObjectsForField(idArray, objectTypeField) {
-    return this.structure[objectTypeField]
-      .filter(x => !idArray.includes(x.objectId))
-      .map(x => ({
-        oldName: _.pick(x, ['schemaName', 'pureName']),
-        objectId: x.objectId,
-        action: 'remove',
-        objectTypeField,
-      }));
-  }
-
-  getDeletedObjects(idArray) {
-    return [
-      ...this.getDeletedObjectsForField(idArray, 'tables'),
-      ...this.getDeletedObjectsForField(idArray, 'views'),
-      ...this.getDeletedObjectsForField(idArray, 'procedures'),
-      ...this.getDeletedObjectsForField(idArray, 'functions'),
-      ...this.getDeletedObjectsForField(idArray, 'triggers'),
-    ];
-  }
-
-  async getModifications() {
+  async _getFastSnapshot() {
     const modificationsQueryData = await this.driver.query(this.pool, this.createQuery('modifications'));
-    // console.log('MOD - SRC', modifications);
-    // console.log(
-    //   'MODs',
-    //   this.structure.tables.map((x) => x.modifyDate)
-    // );
-    const modifications = modificationsQueryData.rows.map(x => {
-      const { type, objectId, modifyDate, schemaName, pureName } = x;
+
+    const res = DatabaseAnalyser.createEmptyStructure();
+    for (const item of modificationsQueryData.rows) {
+      const { type, objectId, modifyDate, schemaName, pureName } = item;
       const field = objectTypeToField(type);
-      if (!this.structure[field]) return null;
-      // @ts-ignore
-      const obj = this.structure[field].find(x => x.objectId == objectId);
+      if (!field || !res[field]) continue;
 
-      // object not modified
-      if (obj && Math.abs(new Date(modifyDate).getTime() - new Date(obj.modifyDate).getTime()) < 1000) return null;
-
-      /** @type {import('dbgate-types').DatabaseModification} */
-      const action = obj
-        ? {
-            newName: { schemaName, pureName },
-            oldName: _.pick(obj, ['schemaName', 'pureName']),
-            action: 'change',
-            objectTypeField: field,
-            objectId,
-          }
-        : {
-            newName: { schemaName, pureName },
-            action: 'add',
-            objectTypeField: field,
-            objectId,
-          };
-      return action;
-    });
-
-    return [..._.compact(modifications), ...this.getDeletedObjects(modificationsQueryData.rows.map(x => x.objectId))];
+      res[field].push({
+        objectId,
+        contentHash: modifyDate.toISOString(),
+        schemaName,
+        pureName,
+      });
+    }
+    return res;
   }
 }