From cf5afb43eba822a399bcbba1759330040741045f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 15 May 2021 21:14:00 +0200 Subject: [PATCH] improved modification detection algorithm - for mssql --- packages/tools/src/DatabaseAnalyser.ts | 88 +++++++++++++++++-- packages/types/engines.d.ts | 2 +- .../src/backend/MsSqlAnalyser.js | 76 +++++----------- 3 files changed, 100 insertions(+), 66 deletions(-) diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 92dd053f..2bcd547c 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -2,6 +2,7 @@ import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types'; import _sortBy from 'lodash/sortBy'; import _groupBy from 'lodash/groupBy'; import _pick from 'lodash/pick'; +import _ from 'lodash'; const fp_pick = arg => array => _pick(array, arg); export class DatabaseAnalyser { @@ -16,17 +17,15 @@ export class DatabaseAnalyser { return DatabaseAnalyser.createEmptyStructure(); } - async _computeSingleObjectId() {} - - /** @returns {Promise} */ - async getModifications() { - if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled'); - + async _getFastSnapshot(): Promise { return null; } + async _computeSingleObjectId() {} + async fullAnalysis() { - return this._runAnalysis(); + const res = await this._runAnalysis(); + return res; } async singleObjectAnalysis(name, typeField) { @@ -49,7 +48,7 @@ export class DatabaseAnalyser { } if (this.modifications.length == 0) return null; console.log('DB modifications detected:', this.modifications); - return this._runAnalysis(); + return this.mergeAnalyseResult(await this._runAnalysis()); } mergeAnalyseResult(newlyAnalysed) { @@ -69,7 +68,7 @@ export class DatabaseAnalyser { const addedChangedIds = newArray.map(x => x.objectId); const removeAllIds = [...removedIds, ...addedChangedIds]; res[field] = _sortBy( - [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray], + [...(this.structure[field] || []).filter(x => !removeAllIds.includes(x.objectId)), ...newArray], x => x.pureName ); } @@ -109,6 +108,11 @@ export class DatabaseAnalyser { if (!this.modifications || !typeFields || this.modifications.length == 0) { res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null'); } else { + if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) { + // do not filter objects + res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null'); + } + const filterIds = this.modifications .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change')) .map(x => x.objectId); @@ -121,6 +125,72 @@ export class DatabaseAnalyser { return res; } + getDeletedObjectsForField(snapshot, objectTypeField) { + const items = snapshot[objectTypeField]; + if (!items) return []; + if (!this.structure[objectTypeField]) return []; + return this.structure[objectTypeField] + .filter(x => !items.find(y => x.objectId == y.objectId)) + .map(x => ({ + oldName: _.pick(x, ['schemaName', 'pureName']), + objectId: x.objectId, + action: 'remove', + objectTypeField, + })); + } + + getDeletedObjects(snapshot) { + return [ + ...this.getDeletedObjectsForField(snapshot, 'tables'), + ...this.getDeletedObjectsForField(snapshot, 'collections'), + ...this.getDeletedObjectsForField(snapshot, 'views'), + ...this.getDeletedObjectsForField(snapshot, 'procedures'), + ...this.getDeletedObjectsForField(snapshot, 'functions'), + ...this.getDeletedObjectsForField(snapshot, 'triggers'), + ]; + } + + async getModifications() { + const snapshot = await this._getFastSnapshot(); + if (!snapshot) return null; + + // console.log('STRUCTURE', this.structure); + // console.log('SNAPSHOT', snapshot); + + const res = []; + for (const field in snapshot) { + const items = snapshot[field]; + if (items === null) { + res.push({ objectTypeField: field, action: 'all' }); + continue; + } + for (const item of items) { + const { objectId, schemaName, pureName, contentHash } = item; + const obj = this.structure[field].find(x => x.objectId == objectId); + + if (obj && contentHash && obj.contentHash == contentHash) continue; + + const action = obj + ? { + newName: { schemaName, pureName }, + oldName: _.pick(obj, ['schemaName', 'pureName']), + action: 'change', + objectTypeField: field, + objectId, + } + : { + newName: { schemaName, pureName }, + action: 'add', + objectTypeField: field, + objectId, + }; + res.push(action); + } + + return [..._.compact(res), ...this.getDeletedObjects(snapshot)]; + } + } + static createEmptyStructure(): DatabaseInfo { return { tables: [], diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index c5dbc29a..27efb0b6 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -78,6 +78,6 @@ export interface DatabaseModification { oldName?: NamedObjectInfo; newName?: NamedObjectInfo; objectId?: string; - action: 'add' | 'remove' | 'change'; + action: 'add' | 'remove' | 'change' | 'all'; objectTypeField: keyof DatabaseInfo; } diff --git a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js index e1ec6e0f..811a90b6 100644 --- a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js +++ b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js @@ -88,6 +88,7 @@ class MsSqlAnalyser extends DatabaseAnalyser { const tables = tablesRows.rows.map(row => ({ ...row, + contentHash: row.modifyDate.toISOString(), columns: columnsRows.rows.filter(col => col.objectId == row.objectId).map(getColumnInfo), primaryKey: DatabaseAnalyser.extractPrimaryKeys(row, pkColumnsRows.rows), foreignKeys: DatabaseAnalyser.extractForeignKeys(row, fkColumnsRows.rows), @@ -95,6 +96,7 @@ class MsSqlAnalyser extends DatabaseAnalyser { const views = viewsRows.rows.map(row => ({ ...row, + contentHash: row.modifyDate.toISOString(), createSql: getCreateSql(row), columns: viewColumnRows.rows.filter(col => col.objectId == row.objectId).map(getColumnInfo), })); @@ -103,6 +105,7 @@ class MsSqlAnalyser extends DatabaseAnalyser { .filter(x => x.sqlObjectType.trim() == 'P') .map(row => ({ ...row, + contentHash: row.modifyDate.toISOString(), createSql: getCreateSql(row), })); @@ -110,75 +113,36 @@ class MsSqlAnalyser extends DatabaseAnalyser { .filter(x => ['FN', 'IF', 'TF'].includes(x.sqlObjectType.trim())) .map(row => ({ ...row, + contentHash: row.modifyDate.toISOString(), createSql: getCreateSql(row), })); - return this.mergeAnalyseResult({ + return { tables, views, procedures, functions, schemas, - }); + }; } - getDeletedObjectsForField(idArray, objectTypeField) { - return this.structure[objectTypeField] - .filter(x => !idArray.includes(x.objectId)) - .map(x => ({ - oldName: _.pick(x, ['schemaName', 'pureName']), - objectId: x.objectId, - action: 'remove', - objectTypeField, - })); - } - - getDeletedObjects(idArray) { - return [ - ...this.getDeletedObjectsForField(idArray, 'tables'), - ...this.getDeletedObjectsForField(idArray, 'views'), - ...this.getDeletedObjectsForField(idArray, 'procedures'), - ...this.getDeletedObjectsForField(idArray, 'functions'), - ...this.getDeletedObjectsForField(idArray, 'triggers'), - ]; - } - - async getModifications() { + async _getFastSnapshot() { const modificationsQueryData = await this.driver.query(this.pool, this.createQuery('modifications')); - // console.log('MOD - SRC', modifications); - // console.log( - // 'MODs', - // this.structure.tables.map((x) => x.modifyDate) - // ); - const modifications = modificationsQueryData.rows.map(x => { - const { type, objectId, modifyDate, schemaName, pureName } = x; + + const res = DatabaseAnalyser.createEmptyStructure(); + for (const item of modificationsQueryData.rows) { + const { type, objectId, modifyDate, schemaName, pureName } = item; const field = objectTypeToField(type); - if (!this.structure[field]) return null; - // @ts-ignore - const obj = this.structure[field].find(x => x.objectId == objectId); + if (!field || !res[field]) continue; - // object not modified - if (obj && Math.abs(new Date(modifyDate).getTime() - new Date(obj.modifyDate).getTime()) < 1000) return null; - - /** @type {import('dbgate-types').DatabaseModification} */ - const action = obj - ? { - newName: { schemaName, pureName }, - oldName: _.pick(obj, ['schemaName', 'pureName']), - action: 'change', - objectTypeField: field, - objectId, - } - : { - newName: { schemaName, pureName }, - action: 'add', - objectTypeField: field, - objectId, - }; - return action; - }); - - return [..._.compact(modifications), ...this.getDeletedObjects(modificationsQueryData.rows.map(x => x.objectId))]; + res[field].push({ + objectId, + contentHash: modifyDate.toISOString(), + schemaName, + pureName, + }); + } + return res; } }