mirrors/dbgate
1
0
Fork 0
mirror of https://github.com/dbgate/dbgate synced 2024-11-07 20:26:23 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

postgre incremental analysis, fixed mysql incremental analysis

Browse Source
This commit is contained in:
Jan Prochazka 2020-09-28 11:33:25 +02:00
parent fbd963bfb1
commit a88e38dcf7
14 changed files with 194 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/engines/default/DatabaseAnalyser.js
View File

@ -46,7 +46,7 @@ class DatabaseAnalyser {
return this._runAnalysis();
}
mergeAnalyseResult(newlyAnalysed) {
mergeAnalyseResult(newlyAnalysed, extractObjectId) {
if (this.structure == null) {
return {
...DatabaseAnalyser.createEmptyStructure(),
@ -58,12 +58,12 @@ class DatabaseAnalyser {
for (const field of ['tables', 'views', 'functions', 'procedures', 'triggers']) {
const removedIds = this.modifications
.filter((x) => x.action == 'remove' && x.objectTypeField == field)
.map((x) => x.objectId);
.map((x) => extractObjectId(x));
const newArray = newlyAnalysed[field] || [];
const addedChangedIds = newArray.map((x) => x.objectId);
const addedChangedIds = newArray.map((x) => extractObjectId(x));
const removeAllIds = [...removedIds, ...addedChangedIds];
res[field] = _.sortBy(
[...this.structure[field].filter((x) => !removeAllIds.includes(x.objectId)), ...newArray],
[...this.structure[field].filter((x) => !removeAllIds.includes(extractObjectId(x))), ...newArray],
(x) => x.pureName
);
}

17
packages/engines/mssql/MsSqlAnalyser.js
View File

@ -137,13 +137,16 @@ class MsSqlAnalyser extends DatabaseAnalyser {
createSql: getCreateSql(row),
}));
return this.mergeAnalyseResult({
tables,
views,
procedures,
functions,
schemas,
});
return this.mergeAnalyseResult(
{
tables,
views,
procedures,
functions,
schemas,
},
(x) => x.objectId
);
}
getDeletedObjectsForField(idArray, objectTypeField) {

38
packages/engines/mysql/MySqlAnalyser.js
View File

@ -53,7 +53,8 @@ class MySqlAnalyser extends DatabaseAnalayser {
} else {
const filterNames = this.modifications
.filter((x) => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
.map((x) => x.objectId);
.map((x) => x.newName && x.newName.pureName)
.filter(Boolean);
if (filterNames.length == 0) {
res = res.replace('=[OBJECT_NAME_CONDITION]', ' IS NULL');
} else {
@ -97,22 +98,25 @@ class MySqlAnalyser extends DatabaseAnalayser {
const viewTexts = await this.getViewTexts(views.rows.map((x) => x.pureName));
return this.mergeAnalyseResult({
tables: tables.rows.map((table) => ({
...table,
columns: columns.rows.filter((col) => col.pureName == table.pureName).map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,
columns: columns.rows.filter((col) => col.pureName == view.pureName).map(getColumnInfo),
createSql: viewTexts[view.pureName],
requiresFormat: true,
})),
procedures: programmables.rows.filter((x) => x.objectType == 'PROCEDURE').map(fp.omit(['objectType'])),
functions: programmables.rows.filter((x) => x.objectType == 'FUNCTION').map(fp.omit(['objectType'])),
});
return this.mergeAnalyseResult(
{
tables: tables.rows.map((table) => ({
...table,
columns: columns.rows.filter((col) => col.pureName == table.pureName).map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,
columns: columns.rows.filter((col) => col.pureName == view.pureName).map(getColumnInfo),
createSql: viewTexts[view.pureName],
requiresFormat: true,
})),
procedures: programmables.rows.filter((x) => x.objectType == 'PROCEDURE').map(fp.omit(['objectType'])),
functions: programmables.rows.filter((x) => x.objectType == 'FUNCTION').map(fp.omit(['objectType'])),
},
(x) => x.pureName
);
}
getDeletedObjectsForField(nameArray, objectTypeField) {

131
packages/engines/postgres/PostgreAnalyser.js
View File

@ -42,8 +42,31 @@ class PostgreAnalyser extends DatabaseAnalayser {
createQuery(resFileName, typeFields) {
let res = sql[resFileName];
res = res.replace('=[OBJECT_ID_CONDITION]', ' is not null');
if (this.singleObjectFilter) {
const { typeField, schemaName, pureName } = this.singleObjectFilter;
if (!typeFields || !typeFields.includes(typeField)) return null;
res = res.replace(/=OBJECT_ID_CONDITION/g, ` = '${typeField}:${schemaName}.${pureName}'`);
return res;
}
if (!this.modifications || !typeFields || this.modifications.length == 0) {
res = res.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
} else {
const filterNames = this.modifications
.filter((x) => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
.filter((x) => x.newName)
.map((x) => `${x.objectTypeField}:${x.newName.schemaName}.${x.newName.pureName}`);
if (filterNames.length == 0) {
res = res.replace(/=OBJECT_ID_CONDITION/g, ' IS NULL');
} else {
res = res.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterNames.map((x) => `'${x}'`).join(',')})`);
}
}
return res;
// let res = sql[resFileName];
// res = res.replace('=[OBJECT_ID_CONDITION]', ' is not null');
// return res;
}
async _runAnalysis() {
const tables = await this.driver.query(this.pool, this.createQuery('tableModifications', ['tables']));
@ -54,24 +77,96 @@ class PostgreAnalyser extends DatabaseAnalayser {
const routines = await this.driver.query(this.pool, this.createQuery('routines', ['procedures', 'functions']));
// console.log('PG fkColumns', fkColumns.rows);
return this.mergeAnalyseResult({
tables: tables.rows.map((table) => ({
...table,
columns: columns.rows
.filter((col) => col.pureName == table.pureName && col.schemaName == table.schemaName)
.map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,
columns: columns.rows
.filter((col) => col.pureName == view.pureName && col.schemaName == view.schemaName)
.map(getColumnInfo),
})),
procedures: routines.rows.filter((x) => x.objectType == 'PROCEDURE'),
functions: routines.rows.filter((x) => x.objectType == 'FUNCTION'),
return this.mergeAnalyseResult(
{
tables: tables.rows.map((table) => ({
...table,
columns: columns.rows
.filter((col) => col.pureName == table.pureName && col.schemaName == table.schemaName)
.map(getColumnInfo),
primaryKey: DatabaseAnalayser.extractPrimaryKeys(table, pkColumns.rows),
foreignKeys: DatabaseAnalayser.extractForeignKeys(table, fkColumns.rows),
})),
views: views.rows.map((view) => ({
...view,
columns: columns.rows
.filter((col) => col.pureName == view.pureName && col.schemaName == view.schemaName)
.map(getColumnInfo),
})),
procedures: routines.rows.filter((x) => x.objectType == 'PROCEDURE'),
functions: routines.rows.filter((x) => x.objectType == 'FUNCTION'),
},
(x) => `${x.objectTypeField}:${x.schemaName}.${x.pureName}`
);
}
async getModifications() {
const tableModificationsQueryData = await this.driver.query(this.pool, this.createQuery('tableModifications'));
const viewModificationsQueryData = await this.driver.query(this.pool, this.createQuery('viewModifications'));
const routineModificationsQueryData = await this.driver.query(this.pool, this.createQuery('routineModifications'));
const allModifications = _.compact([
...tableModificationsQueryData.rows.map((x) => ({ ...x, objectTypeField: 'tables' })),
...viewModificationsQueryData.rows.map((x) => ({ ...x, objectTypeField: 'views' })),
...routineModificationsQueryData.rows
.filter((x) => x.objectType == 'PROCEDURE')
.map((x) => ({ ...x, objectTypeField: 'procedures' })),
...routineModificationsQueryData.rows
.filter((x) => x.objectType == 'FUNCTION')
.map((x) => ({ ...x, objectTypeField: 'functions' })),
]);
const modifications = allModifications.map((x) => {
const { objectTypeField, hashCode, pureName, schemaName } = x;
if (!objectTypeField || !this.structure[objectTypeField]) return null;
const obj = this.structure[objectTypeField].find((x) => x.pureName == pureName && x.schemaName == schemaName);
// object not modified
if (obj && obj.hashCode == hashCode) return null;
// console.log('MODIFICATION OF ', objectTypeField, schemaName, pureName);
/** @type {import('@dbgate/types').DatabaseModification} */
const action = obj
? {
newName: { schemaName, pureName },
oldName: _.pick(obj, ['schemaName', 'pureName']),
action: 'change',
objectTypeField,
}
: {
newName: { schemaName, pureName },
action: 'add',
objectTypeField,
};
return action;
});
return [
..._.compact(modifications),
...this.getDeletedObjects([...allModifications.map((x) => `${x.schemaName}.${x.pureName}`)]),
];
}
getDeletedObjectsForField(nameArray, objectTypeField) {
return this.structure[objectTypeField]
.filter((x) => !nameArray.includes(`${x.schemaName}.${x.pureName}`))
.map((x) => ({
oldName: _.pick(x, ['schemaName', 'pureName']),
action: 'remove',
objectTypeField,
}));
}
getDeletedObjects(nameArray) {
return [
...this.getDeletedObjectsForField(nameArray, 'tables'),
...this.getDeletedObjectsForField(nameArray, 'views'),
...this.getDeletedObjectsForField(nameArray, 'procedures'),
...this.getDeletedObjectsForField(nameArray, 'functions'),
...this.getDeletedObjectsForField(nameArray, 'triggers'),
];
}
}

10
packages/engines/postgres/index.js
View File

@ -97,6 +97,16 @@ const driver = {
return stream;
},
async analyseSingleObject(pool, name, typeField = 'tables') {
const analyser = new PostgreAnalyser(pool, this);
analyser.singleObjectFilter = { ...name, typeField };
const res = await analyser.fullAnalysis();
return res.tables[0];
},
// @ts-ignore
analyseSingleTable(pool, name) {
return this.analyseSingleObject(pool, name, 'tables');
},
async getVersion(client) {
const { rows } = await this.query(client, 'SELECT version()');
const { version } = rows[0];

2
packages/engines/postgres/sql/columns.js
View File

@ -14,6 +14,6 @@ where
table_schema <> 'information_schema'
and table_schema <> 'pg_catalog'
and table_schema !~ '^pg_toast'
and 'table:' || table_schema || '.' || table_name =[OBJECT_ID_CONDITION]
and 'tables:' || table_schema || '.' || table_name =OBJECT_ID_CONDITION
order by ordinal_position
`;

2
packages/engines/postgres/sql/foreignKeys.js
View File

@ -19,6 +19,6 @@ where
base.table_schema <> 'information_schema'
and base.table_schema <> 'pg_catalog'
and base.table_schema !~ '^pg_toast'
and 'table:' || base.table_schema || '.' || base.table_name =[OBJECT_ID_CONDITION]
and 'tables:' || base.table_schema || '.' || base.table_name =OBJECT_ID_CONDITION
order by basecol.ordinal_position
`;

4
packages/engines/postgres/sql/index.js
View File

@ -1,15 +1,19 @@
const columns = require('./columns');
const tableModifications = require('./tableModifications');
const viewModifications = require('./viewModifications');
const primaryKeys = require('./primaryKeys');
const foreignKeys = require('./foreignKeys');
const views = require('./views');
const routines = require('./routines');
const routineModifications = require('./routineModifications');
module.exports = {
columns,
tableModifications,
viewModifications,
primaryKeys,
foreignKeys,
views,
routines,
routineModifications,
};

2
packages/engines/postgres/sql/primaryKeys.js
View File

@ -12,6 +12,6 @@ where
and table_constraints.table_schema <> 'pg_catalog'
and table_constraints.table_schema !~ '^pg_toast'
and table_constraints.constraint_type = 'PRIMARY KEY'
and 'table:' || table_constraints.table_schema || '.' || table_constraints.table_name =[OBJECT_ID_CONDITION]
and 'tables:' || table_constraints.table_schema || '.' || table_constraints.table_name =OBJECT_ID_CONDITION
order by key_column_usage.ordinal_position
`;

10
packages/engines/postgres/sql/routineModifications.js Normal file
View File

@ -0,0 +1,10 @@
module.exports = `
select
routine_name as "pureName",
routine_schema as "schemaName",
md5(routine_definition) as "hashCode",
routine_type as "objectType"
from
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog'
and routine_type in ('PROCEDURE', 'FUNCTION')
`;

7
packages/engines/postgres/sql/routines.js
View File

@ -6,5 +6,10 @@ select
md5(routine_definition) as "hashCode",
routine_type as "objectType"
from
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog' and routine_type is not null
information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog'
and (
(routine_type = 'PROCEDURE' and ('procedures:' || routine_schema || '.' || routine_schema) =OBJECT_ID_CONDITION)
or
(routine_type = 'FUNCTION' and ('functions:' || routine_schema || '.' || routine_schema) =OBJECT_ID_CONDITION)
)
`;

4
packages/engines/postgres/sql/tableModifications.js
View File

@ -20,7 +20,7 @@ SELECT oid as "objectId", nspname as "schemaName", relname as "pureName",
' ' || column_name || ' ' || type || ' '|| not_null
)
, E',\\n'
) || E'\\n);\\n' || (select pkey from pkey where pkey.conrelid = oid)) as "hashCode"
) || E'\\n);\\n' || coalesce((select pkey from pkey where pkey.conrelid = oid),'NO_PK')) as "hashCode"
from
(
SELECT
@ -47,6 +47,6 @@ from
AND n.nspname !~ '^pg_toast'
ORDER BY a.attnum
) as tabledefinition
where 'table:' || nspname || '.' || relname =[OBJECT_ID_CONDITION]
where ('tables:' || nspname || '.' || relname) =OBJECT_ID_CONDITION
group by relname, nspname, oid
`;

8
packages/engines/postgres/sql/viewModifications.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = `
select
table_name as "pureName",
table_schema as "schemaName",
md5(view_definition) as "hashCode"
from
information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog'
`;

4
packages/engines/postgres/sql/views.js
View File

@ -5,5 +5,7 @@ select
view_definition as "createSql",
md5(view_definition) as "hashCode"
from
information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog'
information_schema.views
where table_schema != 'information_schema' and table_schema != 'pg_catalog'
and ('views:' || table_schema || '.' || table_name) =OBJECT_ID_CONDITION
`;