Merge branch 'develop'

This commit is contained in:
Jan Prochazka 2023-01-21 13:56:55 +01:00
commit 16238f8f94
13 changed files with 163 additions and 218 deletions

View File

@ -1,6 +1,6 @@
{
"private": true,
"version": "5.2.1",
"version": "5.2.2-beta.1",
"name": "dbgate-all",
"workspaces": [
"packages/*",

View File

@ -170,9 +170,9 @@ export class DatabaseAnalyser {
// return this.structure.tables.find((x) => x.objectId == id);
// }
containsObjectIdCondition(typeFields) {
return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
}
// containsObjectIdCondition(typeFields) {
// return this.createQueryCore('=OBJECT_ID_CONDITION', typeFields) != ' is not null';
// }
createQuery(template, typeFields) {
return this.createQueryCore(template, typeFields);
@ -197,7 +197,7 @@ export class DatabaseAnalyser {
.filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
.map(x => x.objectId);
if (filterIds.length == 0) {
return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'");
return null;
}
return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`);
}
@ -293,7 +293,14 @@ export class DatabaseAnalyser {
return [..._compact(res), ...this.getDeletedObjects(snapshot)];
}
async safeQuery(sql) {
async analyserQuery(template, typeFields) {
const sql = this.createQuery(template, typeFields);
if (!sql) {
return {
rows: [],
};
}
try {
return await this.driver.query(this.pool, sql);
} catch (err) {

View File

@ -82,33 +82,35 @@ class MsSqlAnalyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tablesRows = await this.driver.query(this.pool, this.createQuery('tables', ['tables']));
const tablesRows = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columnsRows = await this.driver.query(this.pool, this.createQuery('columns', ['tables']));
const columnsRows = await this.analyserQuery('columns', ['tables']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumnsRows = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumnsRows = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumnsRows = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
const fkColumnsRows = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading schemas' });
const schemaRows = await this.driver.query(this.pool, this.createQuery('getSchemas'));
const schemaRows = await this.analyserQuery('getSchemas');
this.feedback({ analysingMessage: 'Loading indexes' });
const indexesRows = await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
const indexesRows = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
const indexcolsRows = await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
const indexcolsRows = await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading default schema' });
const defaultSchemaRows = await this.driver.query(this.pool, 'SELECT SCHEMA_NAME() as name');
this.feedback({ analysingMessage: 'Loading table sizes' });
const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes'));
const tableSizes = await this.analyserQuery('tableSizes');
const schemas = schemaRows.rows;
const tableSizesDict = _.mapValues(_.keyBy(tableSizes.rows, 'objectId'), 'tableRowCount');
this.feedback({ analysingMessage: 'Loading SQL code' });
const sqlCodeRows = await this.driver.query(
this.pool,
this.createQuery('loadSqlCode', ['views', 'procedures', 'functions', 'triggers'])
);
const sqlCodeRows = await this.analyserQuery('loadSqlCode', [
'views',
'procedures',
'functions',
'triggers',
]);
const getCreateSql = row =>
sqlCodeRows.rows
.filter(x => x.pureName == row.pureName && x.schemaName == row.schemaName)
@ -116,14 +118,11 @@ class MsSqlAnalyser extends DatabaseAnalyser {
.join('');
this.feedback({ analysingMessage: 'Loading views' });
const viewsRows = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const viewsRows = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading procedures & functions' });
const programmableRows = await this.driver.query(
this.pool,
this.createQuery('programmables', ['procedures', 'functions'])
);
const programmableRows = await this.analyserQuery('programmables', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading view columns' });
const viewColumnRows = await this.driver.query(this.pool, this.createQuery('viewColumns', ['views']));
const viewColumnRows = await this.analyserQuery('viewColumns', ['views']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const tables = tablesRows.rows.map(row => ({
@ -190,8 +189,8 @@ class MsSqlAnalyser extends DatabaseAnalyser {
}
async _getFastSnapshot() {
const modificationsQueryData = await this.driver.query(this.pool, this.createQuery('modifications'));
const tableSizes = await this.driver.query(this.pool, this.createQuery('tableSizes'));
const modificationsQueryData = await this.analyserQuery('modifications');
const tableSizes = await this.analyserQuery('tableSizes');
const res = DatabaseAnalyser.createEmptyStructure();
for (const item of modificationsQueryData.rows) {

View File

@ -57,7 +57,7 @@ class Analyser extends DatabaseAnalyser {
async getViewTexts(allViewNames) {
const res = {};
const views = await this.safeQuery(this.createQuery('viewTexts', ['views']));
const views = await this.analyserQuery('viewTexts', ['views']);
for (const view of views.rows) {
res[view.pureName] = `CREATE VIEW \`${view.pureName}\` AS ${view.viewDefinition}`;
}
@ -76,24 +76,24 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(this.pool, this.createQuery('tables', ['tables']));
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.safeQuery(this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumns = await this.safeQuery(this.createQuery('foreignKeys', ['tables']));
const fkColumns = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.safeQuery(this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading programmables' });
const programmables = await this.safeQuery(this.createQuery('programmables', ['procedures', 'functions']));
const programmables = await this.analyserQuery('programmables', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading view texts' });
const viewTexts = await this.getViewTexts(views.rows.map(x => x.pureName));
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = await this.safeQuery(this.createQuery('indexes', ['tables']));
const indexes = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading uniques' });
const uniqueNames = await this.safeQuery(this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const res = {
@ -169,15 +169,9 @@ class Analyser extends DatabaseAnalyser {
}
async _getFastSnapshot() {
const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications'));
const procedureModificationsQueryData = await this.driver.query(
this.pool,
this.createQuery('procedureModifications')
);
const functionModificationsQueryData = await this.driver.query(
this.pool,
this.createQuery('functionModifications')
);
const tableModificationsQueryData = await this.analyserQuery('tableModifications');
const procedureModificationsQueryData = await this.analyserQuery('procedureModifications');
const functionModificationsQueryData = await this.analyserQuery('functionModifications');
return {
tables: tableModificationsQueryData.rows

View File

@ -68,45 +68,40 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(
this.pool,
this.createQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables'])
);
const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableList' : 'tableList', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
//let fkColumns = null;
this.feedback({ analysingMessage: 'Loading foreign keys' });
const fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
const fkColumns = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
let geometryColumns = { rows: [] };
let geographyColumns = { rows: [] };
this.feedback({ analysingMessage: 'Loading materialized views' });
const matviews = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews']))
: null;
const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null;
this.feedback({ analysingMessage: 'Loading materialized view columns' });
const matviewColumns = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews']))
? await this.analyserQuery('matviewColumns', ['matviews'])
: null;
this.feedback({ analysingMessage: 'Loading routines' });
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
const routines = await this.analyserQuery('routines', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
: await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
// const indexcols = this.driver.__analyserInternals.skipIndexes
// ? { rows: [] }
// : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
// const indexcols = this.driver.__analyserInternals.skipIndexes
// ? { rows: [] }
// : await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
this.feedback({ analysingMessage: 'Loading unique names' });
const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
const columnColumnsMapped = fkColumns.rows.map(x => ({
@ -161,7 +156,8 @@ class Analyser extends DatabaseAnalyser {
})),
uniques: _.uniqBy(
indexes.rows.filter(
idx => idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
idx =>
idx.tableName == table.pureName && uniqueNames.rows.find(x => x.constraintName == idx.constraintName)
),
'constraintName'
).map(idx => ({
@ -225,13 +221,13 @@ class Analyser extends DatabaseAnalyser {
return null;
const tableModificationsQueryData = this.driver.dialect.stringAgg
? await this.driver.query(this.pool, this.createQuery('tableModifications'))
? await this.analyserQuery('tableModifications')
: null;
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const viewModificationsQueryData = await this.analyserQuery('viewModifications');
const matviewModificationsQueryData = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewModifications'))
? await this.analyserQuery('matviewModifications')
: null;
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const routineModificationsQueryData = await this.analyserQuery('routineModifications');
return {
tables: tableModificationsQueryData

View File

@ -57,8 +57,7 @@ class Analyser extends DatabaseAnalyser {
createQuery(resFileName, typeFields) {
const query = super.createQuery(sql[resFileName], typeFields);
if (query) return query.replace('#REFTABLECOND#', this.driver.__analyserInternals.refTableCond);
return null;
return query;
}
async _computeSingleObjectId() {
@ -68,43 +67,30 @@ class Analyser extends DatabaseAnalyser {
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
const tables = await this.driver.query(
this.pool,
this.createQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', ['tables'])
);
const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', [
'tables',
]);
this.feedback({ analysingMessage: 'Loading columns' });
const columns = await this.driver.query(this.pool, this.createQuery('columns', ['tables', 'views']));
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
const pkColumns = await this.driver.query(this.pool, this.createQuery('primaryKeys', ['tables']));
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
let fkColumns = null;
// if (true) {
if (this.containsObjectIdCondition(['tables']) || this.driver.__analyserInternals.refTableCond) {
this.feedback({ analysingMessage: 'Loading foreign keys' });
fkColumns = await this.driver.query(this.pool, this.createQuery('foreignKeys', ['tables']));
} else {
this.feedback({ analysingMessage: 'Loading foreign key constraints' });
const fk_tableConstraints = await this.driver.query(
this.pool,
this.createQuery('fk_tableConstraints', ['tables'])
);
const fk_tableConstraints = await this.analyserQuery('fk_tableConstraints', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key refs' });
const fk_referentialConstraints = await this.driver.query(
this.pool,
this.createQuery('fk_referentialConstraints', ['tables'])
);
const fk_referentialConstraints = await this.analyserQuery('fk_referentialConstraints', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key columns' });
const fk_keyColumnUsage = await this.driver.query(this.pool, this.createQuery('fk_keyColumnUsage', ['tables']));
const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']);
const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`;
const rows = [];
const constraintDct = _.keyBy(fk_tableConstraints.rows, cntKey);
const fkRows = [];
const fkConstraintDct = _.keyBy(fk_tableConstraints.rows, cntKey);
for (const fkRef of fk_referentialConstraints.rows) {
const cntBase = constraintDct[cntKey(fkRef)];
const cntRef = constraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`];
const cntBase = fkConstraintDct[cntKey(fkRef)];
const cntRef = fkConstraintDct[`${fkRef.unique_constraint_name}|${fkRef.unique_constraint_schema}`];
if (!cntBase || !cntRef) continue;
const baseCols = _.sortBy(
fk_keyColumnUsage.rows.filter(
@ -124,7 +110,7 @@ class Analyser extends DatabaseAnalyser {
const baseCol = baseCols[i];
const refCol = refCols[i];
rows.push({
fkRows.push({
...fkRef,
pure_name: cntBase.table_name,
schema_name: cntBase.table_schema,
@ -135,41 +121,38 @@ class Analyser extends DatabaseAnalyser {
});
}
}
fkColumns = { rows };
}
fkColumns = { rows: fkRows };
this.feedback({ analysingMessage: 'Loading views' });
const views = await this.driver.query(this.pool, this.createQuery('views', ['views']));
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading materialized views' });
const matviews = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviews', ['matviews']))
: null;
const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null;
this.feedback({ analysingMessage: 'Loading materialized view columns' });
const matviewColumns = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewColumns', ['matviews']))
? await this.analyserQuery('matviewColumns', ['matviews'])
: null;
this.feedback({ analysingMessage: 'Loading routines' });
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
const routines = await this.analyserQuery('routines', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading indexes' });
const indexes = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexes', ['tables']));
: await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
const indexcols = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.driver.query(this.pool, this.createQuery('indexcols', ['tables']));
: await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading unique names' });
const uniqueNames = await this.driver.query(this.pool, this.createQuery('uniqueNames', ['tables']));
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
let geometryColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geometry_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geometry columns' });
geometryColumns = await this.safeQuery(this.createQuery('geometryColumns', ['tables']));
geometryColumns = await this.analyserQuery('geometryColumns', ['tables']);
}
let geographyColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geography_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geography columns' });
geographyColumns = await this.safeQuery(this.createQuery('geographyColumns', ['tables']));
geographyColumns = await this.analyserQuery('geographyColumns', ['tables']);
}
this.feedback({ analysingMessage: 'Finalizing DB structure' });
@ -299,13 +282,13 @@ class Analyser extends DatabaseAnalyser {
async _getFastSnapshot() {
const tableModificationsQueryData = this.driver.dialect.stringAgg
? await this.driver.query(this.pool, this.createQuery('tableModifications'))
? await this.analyserQuery('tableModifications')
: null;
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const viewModificationsQueryData = await this.analyserQuery('viewModifications');
const matviewModificationsQueryData = this.driver.dialect.materializedViews
? await this.driver.query(this.pool, this.createQuery('matviewModifications'))
? await this.analyserQuery('matviewModifications')
: null;
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const routineModificationsQueryData = await this.analyserQuery('routineModifications');
return {
tables: tableModificationsQueryData

View File

@ -78,8 +78,6 @@ const drivers = driverBases.map(driverBase => ({
};
}
console.log('OPTIONS', options);
const client = new pg.Client(options);
await client.connect();

View File

@ -7,4 +7,5 @@ select
basecol.table_name,
basecol.ordinal_position
from information_schema.key_column_usage basecol
where ('tables:' || basecol.table_schema || '.' || basecol.table_name) =OBJECT_ID_CONDITION
`;

View File

@ -5,4 +5,5 @@ select
base.constraint_name as "constraint_name",
base.constraint_schema as "constraint_schema"
from information_schema.table_constraints base
where ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION
`;

View File

@ -1,24 +0,0 @@
module.exports = `
select
fk.constraint_name as "constraint_name",
fk.constraint_schema as "constraint_schema",
base.table_name as "pure_name",
base.table_schema as "schema_name",
fk.update_rule as "update_action",
fk.delete_rule as "delete_action",
ref.table_name as "ref_table_name",
ref.table_schema as "ref_schema_name",
basecol.column_name as "column_name",
refcol.column_name as "ref_column_name"
from information_schema.referential_constraints fk
inner join information_schema.table_constraints base on fk.constraint_name = base.constraint_name and fk.constraint_schema = base.constraint_schema
inner join information_schema.table_constraints ref on fk.unique_constraint_name = ref.constraint_name and fk.unique_constraint_schema = ref.constraint_schema #REFTABLECOND#
inner join information_schema.key_column_usage basecol on base.table_name = basecol.table_name and base.constraint_name = basecol.constraint_name
inner join information_schema.key_column_usage refcol on ref.table_name = refcol.table_name and ref.constraint_name = refcol.constraint_name and basecol.ordinal_position = refcol.ordinal_position
where
base.table_schema <> 'information_schema'
and base.table_schema <> 'pg_catalog'
and base.table_schema !~ '^pg_toast'
and ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION
order by basecol.ordinal_position
`;

View File

@ -4,7 +4,6 @@ const tableList = require('./tableList');
const viewModifications = require('./viewModifications');
const matviewModifications = require('./matviewModifications');
const primaryKeys = require('./primaryKeys');
const foreignKeys = require('./foreignKeys');
const views = require('./views');
const matviews = require('./matviews');
const routines = require('./routines');
@ -26,7 +25,6 @@ module.exports = {
tableList,
viewModifications,
primaryKeys,
foreignKeys,
fk_keyColumnUsage,
fk_referentialConstraints,
fk_tableConstraints,

View File

@ -140,9 +140,7 @@ const postgresDriverBase = {
return connection;
},
__analyserInternals: {
refTableCond: '',
},
__analyserInternals: {},
getNewObjectTemplates() {
return [
@ -212,9 +210,7 @@ const cockroachDriver = {
dropColumnDependencies: ['primaryKey', 'dependencies'],
dropPrimaryKey: false,
},
__analyserInternals: {
refTableCond: 'and fk.referenced_table_name = ref.table_name',
},
__analyserInternals: {},
};
/** @type {import('dbgate-types').EngineDriver} */
@ -225,7 +221,6 @@ const redshiftDriver = {
stringAgg: false,
},
__analyserInternals: {
refTableCond: '',
skipIndexes: true,
},
engine: 'redshift@dbgate-plugin-postgres',

View File

@ -53,12 +53,9 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
const objects = await this.driver.query(
this.pool,
super.createQuery(
const objects = await this.analyserQuery(
"select * from sqlite_master where (type='table' or type='view') and name =OBJECT_ID_CONDITION",
['tables', 'views']
)
);
const tables = objects.rows.filter((x) => x.type == 'table');
const views = objects.rows.filter((x) => x.type == 'view');