From b06d7473994c73edd2d4f70ac757cacaecf88926 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 6 Jan 2023 18:40:47 +0100 Subject: [PATCH 1/6] #451 loading fks on postgres cleanup & fix --- packages/tools/src/DatabaseAnalyser.ts | 6 +- .../src/backend/Analyser.js | 103 +++++++++--------- 2 files changed, 53 insertions(+), 56 deletions(-) diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 2d103e28..5d79ccd0 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -170,9 +170,9 @@ export class DatabaseAnalyser { // return this.structure.tables.find((x) => x.objectId == id); // } - containsObjectIdCondition(typeFields) { - return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null'; - } + // containsObjectIdCondition(typeFields) { + // return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null'; + // } createQuery(template, typeFields) { return this.createQueryCore(template, typeFields); diff --git a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js index 90d8563e..1363d999 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js @@ -57,8 +57,8 @@ class Analyser extends DatabaseAnalyser { createQuery(resFileName, typeFields) { const query = super.createQuery(sql[resFileName], typeFields); - if (query) return query.replace('#REFTABLECOND#', this.driver.__analyserInternals.refTableCond); - return null; + // if (query) return query.replace('#REFTABLECOND#', this.driver.__analyserInternals.refTableCond); + return query; } async _computeSingleObjectId() { @@ -80,63 +80,60 @@ class Analyser extends DatabaseAnalyser { let fkColumns = null; // if (true) { - if (this.containsObjectIdCondition(['tables']) || this.driver.__analyserInternals.refTableCond) { - this.feedback({ analysingMessage: 'Loading foreign keys' }); - fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); - } else { - this.feedback({ analysingMessage: 'Loading foreign key constraints' }); - const fk_tableConstraints = await this.driver.query( - this.pool, - this.createQuery('fk_tableConstraints', ['tables']) + // if (this.containsObjectIdCondition(['tables']) || this.driver.__analyserInternals.refTableCond) { + // this.feedback({ analysingMessage: 'Loading foreign keys' }); + // fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); + // } else { + this.feedback({ analysingMessage: 'Loading foreign key constraints' }); + const fk_tableConstraints = await this.driver.query(this.pool, this.createQuery('fk_tableConstraints', ['tables'])); + + this.feedback({ analysingMessage: 'Loading foreign key refs' }); + const fk_referentialConstraints = await this.driver.query( + this.pool, + this.createQuery('fk_referentialConstraints', ['tables']) + ); + + this.feedback({ analysingMessage: 'Loading foreign key columns' }); + const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables'])); + + const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`; + const fkRows = []; + const fkConstraintDct = _.keyBy(fk_tableConstraints.rows, cntKey); + for (const fkRef of fk_referentialConstraints.rows) { + const cntBase = fkConstraintDct[cntKey(fkRef)]; + const cntRef = fkConstraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`]; + if (!cntBase || !cntRef) continue; + const baseCols = _.sortBy( + fk_keyColumnUsage.rows.filter( + x => x.table_name == cntBase.table_name && x.constraint_name == cntBase.constraint_name + ), + 'ordinal_position' ); - - this.feedback({ analysingMessage: 'Loading foreign key refs' }); - const fk_referentialConstraints = await this.driver.query( - this.pool, - this.createQuery('fk_referentialConstraints', ['tables']) + const refCols = _.sortBy( + fk_keyColumnUsage.rows.filter( + x => x.table_name == cntRef.table_name && x.constraint_name == cntRef.constraint_name + ), + 'ordinal_position' ); + if (baseCols.length != refCols.length) continue; - this.feedback({ analysingMessage: 'Loading foreign key columns' }); - const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables'])); + for (let i = 0; i < baseCols.length; i++) { + const baseCol = baseCols[i]; + const refCol = refCols[i]; - const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`; - const rows = []; - const constraintDct = _.keyBy(fk_tableConstraints.rows, cntKey); - for (const fkRef of fk_referentialConstraints.rows) { - const cntBase = constraintDct[cntKey(fkRef)]; - const cntRef = constraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`]; - if (!cntBase || !cntRef) continue; - const baseCols = _.sortBy( - fk_keyColumnUsage.rows.filter( - x => x.table_name == cntBase.table_name && x.constraint_name == cntBase.constraint_name - ), - 'ordinal_position' - ); - const refCols = _.sortBy( - fk_keyColumnUsage.rows.filter( - x => x.table_name == cntRef.table_name && x.constraint_name == cntRef.constraint_name - ), - 'ordinal_position' - ); - if (baseCols.length != refCols.length) continue; - - for (let i = 0; i < baseCols.length; i++) { - const baseCol = baseCols[i]; - const refCol = refCols[i]; - - rows.push({ - ...fkRef, - pure_name: cntBase.table_name, - schema_name: cntBase.table_schema, - ref_table_name: cntRef.table_name, - ref_schema_name: cntRef.table_schema, - column_name: baseCol.column_name, - ref_column_name: refCol.column_name, - }); - } + fkRows.push({ + ...fkRef, + pure_name: cntBase.table_name, + schema_name: cntBase.table_schema, + ref_table_name: cntRef.table_name, + ref_schema_name: cntRef.table_schema, + column_name: baseCol.column_name, + ref_column_name: refCol.column_name, + }); } - fkColumns = { rows }; } + fkColumns = { rows: fkRows }; + // } this.feedback({ analysingMessage: 'Loading views' }); const views = await this.driver.query(this.pool, this.createQuery('views', ['views'])); From 451f6714264089122695715b2849b4acc7d81a8b Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 6 Jan 2023 18:49:09 +0100 Subject: [PATCH 2/6] v5.2.2-beta.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fe85d695..42265b9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "5.2.1", + "version": "5.2.2-beta.1", "name": "dbgate-all", "workspaces": [ "packages/*", From 3dfae351a6b4d29f2f208f28a96a776fa32ebdf1 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 21 Jan 2023 09:34:29 +0100 Subject: [PATCH 3/6] foreign key loading optimalization #451 --- .../src/backend/Analyser.js | 7 ------ .../src/backend/drivers.js | 2 -- .../src/backend/sql/fk_key_column_usage.js | 1 + .../src/backend/sql/fk_table_constraints.js | 1 + .../src/backend/sql/foreignKeys.js | 24 ------------------- .../src/backend/sql/index.js | 2 -- .../src/frontend/drivers.js | 9 ++----- 7 files changed, 4 insertions(+), 42 deletions(-) delete mode 100644 plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeys.js diff --git a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js index 1363d999..b15dc6b4 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js @@ -57,7 +57,6 @@ class Analyser extends DatabaseAnalyser { createQuery(resFileName, typeFields) { const query = super.createQuery(sql[resFileName], typeFields); - // if (query) return query.replace('#REFTABLECOND#', this.driver.__analyserInternals.refTableCond); return query; } @@ -79,11 +78,6 @@ class Analyser extends DatabaseAnalyser { let fkColumns = null; - // if (true) { - // if (this.containsObjectIdCondition(['tables']) || this.driver.__analyserInternals.refTableCond) { - // this.feedback({ analysingMessage: 'Loading foreign keys' }); - // fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); - // } else { this.feedback({ analysingMessage: 'Loading foreign key constraints' }); const fk_tableConstraints = await this.driver.query(this.pool, this.createQuery('fk_tableConstraints', ['tables'])); @@ -133,7 +127,6 @@ class Analyser extends DatabaseAnalyser { } } fkColumns = { rows: fkRows }; - // } this.feedback({ analysingMessage: 'Loading views' }); const views = await this.driver.query(this.pool, this.createQuery('views', ['views'])); diff --git a/plugins/dbgate-plugin-postgres/src/backend/drivers.js b/plugins/dbgate-plugin-postgres/src/backend/drivers.js index 4faf653a..8165c719 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/backend/drivers.js @@ -78,8 +78,6 @@ const drivers = driverBases.map(driverBase => ({ }; } - console.log('OPTIONS', options); - const client = new pg.Client(options); await client.connect(); diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js index ea5a7382..f7f6e300 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js @@ -7,4 +7,5 @@ select basecol.table_name, basecol.ordinal_position from information_schema.key_column_usage basecol +where ('tables:' || basecol.table_schema || '.' || basecol.table_name) =OBJECT_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js index 51354f21..faca2c5f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js @@ -5,4 +5,5 @@ select base.constraint_name as "constraint_name", base.constraint_schema as "constraint_schema" from information_schema.table_constraints base +where ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeys.js b/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeys.js deleted file mode 100644 index 81b96636..00000000 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/foreignKeys.js +++ /dev/null @@ -1,24 +0,0 @@ -module.exports = ` -select - fk.constraint_name as "constraint_name", - fk.constraint_schema as "constraint_schema", - base.table_name as "pure_name", - base.table_schema as "schema_name", - fk.update_rule as "update_action", - fk.delete_rule as "delete_action", - ref.table_name as "ref_table_name", - ref.table_schema as "ref_schema_name", - basecol.column_name as "column_name", - refcol.column_name as "ref_column_name" -from information_schema.referential_constraints fk -inner join information_schema.table_constraints base on fk.constraint_name = base.constraint_name and fk.constraint_schema = base.constraint_schema -inner join information_schema.table_constraints ref on fk.unique_constraint_name = ref.constraint_name and fk.unique_constraint_schema = ref.constraint_schema #REFTABLECOND# -inner join information_schema.key_column_usage basecol on base.table_name = basecol.table_name and base.constraint_name = basecol.constraint_name -inner join information_schema.key_column_usage refcol on ref.table_name = refcol.table_name and ref.constraint_name = refcol.constraint_name and basecol.ordinal_position = refcol.ordinal_position -where - base.table_schema <> 'information_schema' - and base.table_schema <> 'pg_catalog' - and base.table_schema !~ '^pg_toast' - and ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION -order by basecol.ordinal_position -`; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/index.js b/plugins/dbgate-plugin-postgres/src/backend/sql/index.js index 4c2f500a..53a858ab 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/index.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/index.js @@ -4,7 +4,6 @@ const tableList = require('./tableList'); const viewModifications = require('./viewModifications'); const matviewModifications = require('./matviewModifications'); const primaryKeys = require('./primaryKeys'); -const foreignKeys = require('./foreignKeys'); const views = require('./views'); const matviews = require('./matviews'); const routines = require('./routines'); @@ -26,7 +25,6 @@ module.exports = { tableList, viewModifications, primaryKeys, - foreignKeys, fk_keyColumnUsage, fk_referentialConstraints, fk_tableConstraints, diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index 3c1ac062..93aee6de 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -140,9 +140,7 @@ const postgresDriverBase = { return connection; }, - __analyserInternals: { - refTableCond: '', - }, + __analyserInternals: {}, getNewObjectTemplates() { return [ @@ -212,9 +210,7 @@ const cockroachDriver = { dropColumnDependencies: ['primaryKey', 'dependencies'], dropPrimaryKey: false, }, - __analyserInternals: { - refTableCond: 'and fk.referenced_table_name = ref.table_name', - }, + __analyserInternals: {}, }; /** @type {import('dbgate-types').EngineDriver} */ @@ -225,7 +221,6 @@ const redshiftDriver = { stringAgg: false, }, __analyserInternals: { - refTableCond: '', skipIndexes: true, }, engine: 'redshift@dbgate-plugin-postgres', From f9f879272ba5f4ce1ac86f96916b507025bd4289 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 21 Jan 2023 10:13:08 +0100 Subject: [PATCH 4/6] analyser refactor + optimalization --- packages/tools/src/DatabaseAnalyser.ts | 11 +- .../src/backend/MsSqlAnalyser.js | 41 ++++--- .../src/backend/Analyser.js | 30 ++--- .../src/backend/Analyser.js | 112 +++++++++--------- .../src/backend/Analyser.js | 49 ++++---- 5 files changed, 116 insertions(+), 127 deletions(-) diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 5d79ccd0..126c77ab 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -197,7 +197,7 @@ export class DatabaseAnalyser { .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change')) .map(x => x.objectId); if (filterIds.length == 0) { - return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'"); + return null; } return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`); } @@ -293,7 +293,14 @@ export class DatabaseAnalyser { return [..._compact(res), ...this.getDeletedObjects(snapshot)]; } - async safeQuery(sql) { + async analyserQuery(template, typeFields) { + const sql = this.createQuery(template, typeFields); + + if (!sql) { + return { + rows: [], + }; + } try { return await this.driver.query(this.pool, sql); } catch (err) { diff --git a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js index 76f0f91d..cb21583e 100644 --- a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js +++ b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js @@ -82,33 +82,35 @@ class MsSqlAnalyser extends DatabaseAnalyser { async _runAnalysis() { this.feedback({ analysingMessage: 'Loading tables' }); - const tablesRows = await this.driver.query(this.pool, this.createQuery('tables', ['tables'])); + const tablesRows = await this.analyserQuery('tables', ['tables']); this.feedback({ analysingMessage: 'Loading columns' }); - const columnsRows = await this.driver.query(this.pool, this.createQuery('columns', ['tables'])); + const columnsRows = await this.analyserQuery('columns', ['tables']); this.feedback({ analysingMessage: 'Loading primary keys' }); - const pkColumnsRows = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables'])); + const pkColumnsRows = await this.analyserQuery('primaryKeys', ['tables']); this.feedback({ analysingMessage: 'Loading foreign keys' }); - const fkColumnsRows = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); + const fkColumnsRows = await this.analyserQuery('foreignKeys', ['tables']); this.feedback({ analysingMessage: 'Loading schemas' }); - const schemaRows = await this.driver.query(this.pool, this.createQuery('getSchemas')); + const schemaRows = await this.analyserQuery('getSchemas'); this.feedback({ analysingMessage: 'Loading indexes' }); - const indexesRows = await this.driver.query(this.pool, this.createQuery('indexes', ['tables'])); + const indexesRows = await this.analyserQuery('indexes', ['tables']); this.feedback({ analysingMessage: 'Loading index columns' }); - const indexcolsRows = await this.driver.query(this.pool, this.createQuery('indexcols', ['tables'])); + const indexcolsRows = await this.analyserQuery('indexcols', ['tables']); this.feedback({ analysingMessage: 'Loading default schema' }); - const defaultSchemaRows = await this.driver.query(this.pool, 'SELECT SCHEMA_NAME() as name'); + const defaultSchemaRows = await this.query(this.pool, 'SELECT SCHEMA_NAME() as name'); this.feedback({ analysingMessage: 'Loading table sizes' }); - const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes')); + const tableSizes = await this.analyserQuery('tableSizes'); const schemas = schemaRows.rows; const tableSizesDict = _.mapValues(_.keyBy(tableSizes.rows, 'objectId'), 'tableRowCount'); this.feedback({ analysingMessage: 'Loading SQL code' }); - const sqlCodeRows = await this.driver.query( - this.pool, - this.createQuery('loadSqlCode', ['views', 'procedures', 'functions', 'triggers']) - ); + const sqlCodeRows = await this.analyserQuery('loadSqlCode', [ + 'views', + 'procedures', + 'functions', + 'triggers', + ]); const getCreateSql = row => sqlCodeRows.rows .filter(x => x.pureName == row.pureName && x.schemaName == row.schemaName) @@ -116,14 +118,11 @@ class MsSqlAnalyser extends DatabaseAnalyser { .join(''); this.feedback({ analysingMessage: 'Loading views' }); - const viewsRows = await this.driver.query(this.pool, this.createQuery('views', ['views'])); + const viewsRows = await this.analyserQuery('views', ['views']); this.feedback({ analysingMessage: 'Loading procedures & functions' }); - const programmableRows = await this.driver.query( - this.pool, - this.createQuery('programmables', ['procedures', 'functions']) - ); + const programmableRows = await this.analyserQuery('programmables', ['procedures', 'functions']); this.feedback({ analysingMessage: 'Loading view columns' }); - const viewColumnRows = await this.driver.query(this.pool, this.createQuery('viewColumns', ['views'])); + const viewColumnRows = await this.analyserQuery('viewColumns', ['views']); this.feedback({ analysingMessage: 'Finalizing DB structure' }); const tables = tablesRows.rows.map(row => ({ @@ -190,8 +189,8 @@ class MsSqlAnalyser extends DatabaseAnalyser { } async _getFastSnapshot() { - const modificationsQueryData = await this.driver.query(this.pool, this.createQuery('modifications')); - const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes')); + const modificationsQueryData = await this.analyserQuery('modifications'); + const tableSizes = await this.analyserQuery('tableSizes'); const res = DatabaseAnalyser.createEmptyStructure(); for (const item of modificationsQueryData.rows) { diff --git a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js index f0f8edae..1656893d 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-mysql/src/backend/Analyser.js @@ -57,7 +57,7 @@ class Analyser extends DatabaseAnalyser { async getViewTexts(allViewNames) { const res = {}; - const views = await this.safeQuery(this.createQuery('viewTexts', ['views'])); + const views = await this.analyserQuery('viewTexts', ['views']); for (const view of views.rows) { res[view.pureName] = `CREATE VIEW \`${view.pureName}\` AS ${view.viewDefinition}`; } @@ -76,24 +76,24 @@ class Analyser extends DatabaseAnalyser { async _runAnalysis() { this.feedback({ analysingMessage: 'Loading tables' }); - const tables = await this.driver.query(this.pool, this.createQuery('tables', ['tables'])); + const tables = await this.analyserQuery('tables', ['tables']); this.feedback({ analysingMessage: 'Loading columns' }); - const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views'])); + const columns = await this.analyserQuery('columns', ['tables', 'views']); this.feedback({ analysingMessage: 'Loading primary keys' }); - const pkColumns = await this.safeQuery(this.createQuery('primaryKeys', ['tables'])); + const pkColumns = await this.analyserQuery('primaryKeys', ['tables']); this.feedback({ analysingMessage: 'Loading foreign keys' }); - const fkColumns = await this.safeQuery(this.createQuery('foreignKeys', ['tables'])); + const fkColumns = await this.analyserQuery('foreignKeys', ['tables']); this.feedback({ analysingMessage: 'Loading views' }); - const views = await this.safeQuery(this.createQuery('views', ['views'])); + const views = await this.analyserQuery('views', ['views']); this.feedback({ analysingMessage: 'Loading programmables' }); - const programmables = await this.safeQuery(this.createQuery('programmables', ['procedures', 'functions'])); + const programmables = await this.analyserQuery('programmables', ['procedures', 'functions']); this.feedback({ analysingMessage: 'Loading view texts' }); const viewTexts = await this.getViewTexts(views.rows.map(x => x.pureName)); this.feedback({ analysingMessage: 'Loading indexes' }); - const indexes = await this.safeQuery(this.createQuery('indexes', ['tables'])); + const indexes = await this.analyserQuery('indexes', ['tables']); this.feedback({ analysingMessage: 'Loading uniques' }); - const uniqueNames = await this.safeQuery(this.createQuery('uniqueNames', ['tables'])); + const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']); this.feedback({ analysingMessage: 'Finalizing DB structure' }); const res = { @@ -169,15 +169,9 @@ class Analyser extends DatabaseAnalyser { } async _getFastSnapshot() { - const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications')); - const procedureModificationsQueryData = await this.driver.query( - this.pool, - this.createQuery('procedureModifications') - ); - const functionModificationsQueryData = await this.driver.query( - this.pool, - this.createQuery('functionModifications') - ); + const tableModificationsQueryData = await this.analyserQuery('tableModifications'); + const procedureModificationsQueryData = await this.analyserQuery('procedureModifications'); + const functionModificationsQueryData = await this.analyserQuery('functionModifications'); return { tables: tableModificationsQueryData.rows diff --git a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js index f1130880..45aeb777 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js @@ -68,45 +68,40 @@ class Analyser extends DatabaseAnalyser { async _runAnalysis() { this.feedback({ analysingMessage: 'Loading tables' }); - const tables = await this.driver.query( - this.pool, - this.createQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables']) - ); + const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables']); this.feedback({ analysingMessage: 'Loading columns' }); - const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views'])); + const columns = await this.analyserQuery('columns', ['tables', 'views']); this.feedback({ analysingMessage: 'Loading primary keys' }); - const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables'])); + const pkColumns = await this.analyserQuery('primaryKeys', ['tables']); //let fkColumns = null; - this.feedback({ analysingMessage: 'Loading foreign keys' }); - const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables'])); + this.feedback({ analysingMessage: 'Loading foreign keys' }); + const fkColumns = await this.analyserQuery('foreignKeys', ['tables']); this.feedback({ analysingMessage: 'Loading views' }); - const views = await this.driver.query(this.pool, this.createQuery('views', ['views'])); + const views = await this.analyserQuery('views', ['views']); let geometryColumns = { rows: [] }; let geographyColumns = { rows: [] }; this.feedback({ analysingMessage: 'Loading materialized views' }); - const matviews = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews'])) - : null; + const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null; this.feedback({ analysingMessage: 'Loading materialized view columns' }); const matviewColumns = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews'])) + ? await this.analyserQuery('matviewColumns', ['matviews']) : null; this.feedback({ analysingMessage: 'Loading routines' }); - const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions'])); + const routines = await this.analyserQuery('routines', ['procedures', 'functions']); this.feedback({ analysingMessage: 'Loading indexes' }); const indexes = this.driver.__analyserInternals.skipIndexes ? { rows: [] } - : await this.driver.query(this.pool, this.createQuery('indexes', ['tables'])); + : await this.analyserQuery('indexes', ['tables']); this.feedback({ analysingMessage: 'Loading index columns' }); -// const indexcols = this.driver.__analyserInternals.skipIndexes -// ? { rows: [] } -// : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables'])); + // const indexcols = this.driver.__analyserInternals.skipIndexes + // ? { rows: [] } + // : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables'])); this.feedback({ analysingMessage: 'Loading unique names' }); - const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables'])); + const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']); this.feedback({ analysingMessage: 'Finalizing DB structure' }); const columnColumnsMapped = fkColumns.rows.map(x => ({ @@ -144,34 +139,35 @@ class Analyser extends DatabaseAnalyser { .map(col => getColumnInfo(col, newTable, geometryColumns, geographyColumns)), primaryKey: DatabaseAnalyser.extractPrimaryKeys(newTable, pkColumnsMapped), foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, columnColumnsMapped), - indexes: _.uniqBy( - indexes.rows.filter( - idx => - idx.tableName == table.pureName && !uniqueNames.rows.find(x => x.constraintName == idx.constraintName) - ), - 'constraintName' - ).map(idx => ({ - ..._.pick(idx, ['constraintName', 'indexType']), - isUnique: idx.Unique === 'UNIQUE', - columns: indexes.rows - .filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName) - .map(col => ({ - ..._.pick(col, ['columnName']), - })), - })), - uniques: _.uniqBy( - indexes.rows.filter( - idx => idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName) - ), - 'constraintName' - ).map(idx => ({ - ..._.pick(idx, ['constraintName']), - columns: indexes.rows - .filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName) - .map(col => ({ - ..._.pick(col, ['columnName']), - })), - })), + indexes: _.uniqBy( + indexes.rows.filter( + idx => + idx.tableName == table.pureName && !uniqueNames.rows.find(x => x.constraintName == idx.constraintName) + ), + 'constraintName' + ).map(idx => ({ + ..._.pick(idx, ['constraintName', 'indexType']), + isUnique: idx.Unique === 'UNIQUE', + columns: indexes.rows + .filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName) + .map(col => ({ + ..._.pick(col, ['columnName']), + })), + })), + uniques: _.uniqBy( + indexes.rows.filter( + idx => + idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName) + ), + 'constraintName' + ).map(idx => ({ + ..._.pick(idx, ['constraintName']), + columns: indexes.rows + .filter(col => col.tableName == idx.tableName && col.constraintName == idx.constraintName) + .map(col => ({ + ..._.pick(col, ['columnName']), + })), + })), }; }), views: views.rows.map(view => ({ @@ -225,13 +221,13 @@ class Analyser extends DatabaseAnalyser { return null; const tableModificationsQueryData = this.driver.dialect.stringAgg - ? await this.driver.query(this.pool, this.createQuery('tableModifications')) + ? await this.analyserQuery('tableModifications') : null; - const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications')); + const viewModificationsQueryData = await this.analyserQuery('viewModifications'); const matviewModificationsQueryData = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviewModifications')) + ? await this.analyserQuery('matviewModifications') : null; - const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications')); + const routineModificationsQueryData = await this.analyserQuery('routineModifications'); return { tables: tableModificationsQueryData @@ -243,13 +239,13 @@ class Analyser extends DatabaseAnalyser { })) : null, views: viewModificationsQueryData - ? viewModificationsQueryData.rows.map(x => ({ - objectId: `views:${x.schema_name}.${x.pure_name}`, - pureName: x.pure_name, - schemaName: x.schema_name, - contentHash: x.hash_code, - })) - : undefined, + ? viewModificationsQueryData.rows.map(x => ({ + objectId: `views:${x.schema_name}.${x.pure_name}`, + pureName: x.pure_name, + schemaName: x.schema_name, + contentHash: x.hash_code, + })) + : undefined, matviews: matviewModificationsQueryData ? matviewModificationsQueryData.rows.map(x => ({ objectId: `matviews:${x.schema_name}.${x.pure_name}`, diff --git a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js index b15dc6b4..fb536c5f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js @@ -67,28 +67,23 @@ class Analyser extends DatabaseAnalyser { async _runAnalysis() { this.feedback({ analysingMessage: 'Loading tables' }); - const tables = await this.driver.query( - this.pool, - this.createQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', ['tables']) - ); + const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', [ + 'tables', + ]); this.feedback({ analysingMessage: 'Loading columns' }); - const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views'])); + const columns = await this.analyserQuery('columns', ['tables', 'views']); this.feedback({ analysingMessage: 'Loading primary keys' }); - const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables'])); + const pkColumns = await this.analyserQuery('primaryKeys', ['tables']); let fkColumns = null; this.feedback({ analysingMessage: 'Loading foreign key constraints' }); - const fk_tableConstraints = await this.driver.query(this.pool, this.createQuery('fk_tableConstraints', ['tables'])); + const fk_tableConstraints = await this.analyserQuery('fk_tableConstraints', ['tables']); this.feedback({ analysingMessage: 'Loading foreign key refs' }); - const fk_referentialConstraints = await this.driver.query( - this.pool, - this.createQuery('fk_referentialConstraints', ['tables']) - ); - + const fk_referentialConstraints = await this.analyserQuery('fk_referentialConstraints', ['tables']); this.feedback({ analysingMessage: 'Loading foreign key columns' }); - const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables'])); + const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']); const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`; const fkRows = []; @@ -129,37 +124,35 @@ class Analyser extends DatabaseAnalyser { fkColumns = { rows: fkRows }; this.feedback({ analysingMessage: 'Loading views' }); - const views = await this.driver.query(this.pool, this.createQuery('views', ['views'])); + const views = await this.analyserQuery('views', ['views']); this.feedback({ analysingMessage: 'Loading materialized views' }); - const matviews = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews'])) - : null; + const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null; this.feedback({ analysingMessage: 'Loading materialized view columns' }); const matviewColumns = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews'])) + ? await this.analyserQuery('matviewColumns', ['matviews']) : null; this.feedback({ analysingMessage: 'Loading routines' }); - const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions'])); + const routines = await this.analyserQuery('routines', ['procedures', 'functions']); this.feedback({ analysingMessage: 'Loading indexes' }); const indexes = this.driver.__analyserInternals.skipIndexes ? { rows: [] } - : await this.driver.query(this.pool, this.createQuery('indexes', ['tables'])); + : await this.analyserQuery('indexes', ['tables']); this.feedback({ analysingMessage: 'Loading index columns' }); const indexcols = this.driver.__analyserInternals.skipIndexes ? { rows: [] } - : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables'])); + : await this.analyserQuery('indexcols', ['tables']); this.feedback({ analysingMessage: 'Loading unique names' }); - const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables'])); + const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']); let geometryColumns = { rows: [] }; if (views.rows.find(x => x.pure_name == 'geometry_columns' && x.schema_name == 'public')) { this.feedback({ analysingMessage: 'Loading geometry columns' }); - geometryColumns = await this.safeQuery(this.createQuery('geometryColumns', ['tables'])); + geometryColumns = await this.analyserQuery('geometryColumns', ['tables']); } let geographyColumns = { rows: [] }; if (views.rows.find(x => x.pure_name == 'geography_columns' && x.schema_name == 'public')) { this.feedback({ analysingMessage: 'Loading geography columns' }); - geographyColumns = await this.safeQuery(this.createQuery('geographyColumns', ['tables'])); + geographyColumns = await this.analyserQuery('geographyColumns', ['tables']); } this.feedback({ analysingMessage: 'Finalizing DB structure' }); @@ -289,13 +282,13 @@ class Analyser extends DatabaseAnalyser { async _getFastSnapshot() { const tableModificationsQueryData = this.driver.dialect.stringAgg - ? await this.driver.query(this.pool, this.createQuery('tableModifications')) + ? await this.analyserQuery('tableModifications') : null; - const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications')); + const viewModificationsQueryData = await this.analyserQuery('viewModifications'); const matviewModificationsQueryData = this.driver.dialect.materializedViews - ? await this.driver.query(this.pool, this.createQuery('matviewModifications')) + ? await this.analyserQuery('matviewModifications') : null; - const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications')); + const routineModificationsQueryData = await this.analyserQuery('routineModifications'); return { tables: tableModificationsQueryData From cf07123f515b876fb2e0f25b60cd16b829d89cfa Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 21 Jan 2023 10:52:50 +0100 Subject: [PATCH 5/6] fixed msql analyser --- plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js index cb21583e..6be71c47 100644 --- a/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js +++ b/plugins/dbgate-plugin-mssql/src/backend/MsSqlAnalyser.js @@ -96,7 +96,7 @@ class MsSqlAnalyser extends DatabaseAnalyser { this.feedback({ analysingMessage: 'Loading index columns' }); const indexcolsRows = await this.analyserQuery('indexcols', ['tables']); this.feedback({ analysingMessage: 'Loading default schema' }); - const defaultSchemaRows = await this.query(this.pool, 'SELECT SCHEMA_NAME() as name'); + const defaultSchemaRows = await this.driver.query(this.pool, 'SELECT SCHEMA_NAME() as name'); this.feedback({ analysingMessage: 'Loading table sizes' }); const tableSizes = await this.analyserQuery('tableSizes'); From 44dadcd2563924b47f5500324cdbe06babf04d4e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 21 Jan 2023 11:01:19 +0100 Subject: [PATCH 6/6] fixed sqlite analyser --- plugins/dbgate-plugin-sqlite/src/backend/Analyser.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/dbgate-plugin-sqlite/src/backend/Analyser.js b/plugins/dbgate-plugin-sqlite/src/backend/Analyser.js index 6192328c..1aefdae9 100644 --- a/plugins/dbgate-plugin-sqlite/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-sqlite/src/backend/Analyser.js @@ -53,12 +53,9 @@ class Analyser extends DatabaseAnalyser { } async _runAnalysis() { - const objects = await this.driver.query( - this.pool, - super.createQuery( - "select * from sqlite_master where (type='table' or type='view') and name =OBJECT_ID_CONDITION", - ['tables', 'views'] - ) + const objects = await this.analyserQuery( + "select * from sqlite_master where (type='table' or type='view') and name =OBJECT_ID_CONDITION", + ['tables', 'views'] ); const tables = objects.rows.filter((x) => x.type == 'table'); const views = objects.rows.filter((x) => x.type == 'view');