db deploy fixes

This commit is contained in:
SPRINX0\prochazka 2024-10-29 14:28:26 +01:00
parent 456d3ba42e
commit 2704825d03
17 changed files with 184 additions and 30 deletions

View File

@ -0,0 +1,21 @@
const dataTypeMapperTransform = (oldType, newType) => database => {
return {
...database,
tables: database.tables.map(table => {
return {
...table,
columns: table.columns.map(column => {
if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
return {
...column,
dataType: newType,
};
}
return column;
}),
};
}),
};
};
module.exports = dataTypeMapperTransform;

View File

@ -1,7 +1,15 @@
const generateDeploySql = require('./generateDeploySql');
const executeQuery = require('./executeQuery');
async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
async function deployDb({
connection,
systemConnection,
driver,
analysedStructure,
modelFolder,
loadedDbModel,
modelTransforms,
}) {
const { sql } = await generateDeploySql({
connection,
systemConnection,
@ -9,9 +17,10 @@ async function deployDb({ connection, systemConnection, driver, analysedStructur
analysedStructure,
modelFolder,
loadedDbModel,
modelTransforms,
});
// console.log('RUNNING DEPLOY SCRIPT:', sql);
await executeQuery({ connection, systemConnection, driver, sql });
await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
}
module.exports = deployDb;

View File

@ -1,11 +1,19 @@
const requireEngineDriver = require('../utility/requireEngineDriver');
const connectUtility = require('../utility/connectUtility');
const { getLogger } = require('dbgate-tools');
const { getLogger, getLimitedQuery } = require('dbgate-tools');
const logger = getLogger('execQuery');
async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
logger.info({ sql }, `Execute query`);
async function executeQuery({
connection = undefined,
systemConnection = undefined,
driver = undefined,
sql,
logScriptItems = false,
}) {
if (!logScriptItems) {
logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
}
if (!driver) driver = requireEngineDriver(connection);
const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
try {
logger.info(`Connected.`);
await driver.script(dbhan, sql);
await driver.script(dbhan, sql, { logScriptItems });
} finally {
if (!systemConnection) {
await driver.close(dbhan);

View File

@ -18,6 +18,7 @@ async function generateDeploySql({
analysedStructure = undefined,
modelFolder = undefined,
loadedDbModel = undefined,
modelTransforms = undefined,
}) {
if (!driver) driver = requireEngineDriver(connection);
@ -28,9 +29,15 @@ async function generateDeploySql({
analysedStructure = await driver.analyseFull(dbhan);
}
const deployedModel = generateDbPairingId(
extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
);
let deployedModelSource = loadedDbModel
? databaseInfoFromYamlModel(loadedDbModel)
: await importDbModel(modelFolder);
for (const transform of modelTransforms || []) {
deployedModelSource = transform(deployedModelSource);
}
const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
const opts = {
...modelCompareDbDiffOptions,
@ -57,7 +64,7 @@ async function generateDeploySql({
deployedModel,
driver
);
return res;
} finally {
if (!systemConnection) {

View File

@ -30,6 +30,8 @@ const dataDuplicator = require('./dataDuplicator');
const dbModelToJson = require('./dbModelToJson');
const jsonToDbModel = require('./jsonToDbModel');
const jsonReader = require('./jsonReader');
const dataTypeMapperTransform = require('./dataTypeMapperTransform');
const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
const dbgateApi = {
queryReader,
@ -63,6 +65,8 @@ const dbgateApi = {
dataDuplicator,
dbModelToJson,
jsonToDbModel,
dataTypeMapperTransform,
sqlTextReplacementTransform,
};
requirePlugin.initializeDbgateApi(dbgateApi);

View File

@ -0,0 +1,28 @@
function replaceInText(text, replacements) {
let result = text;
for (const key of Object.keys(replacements)) {
result = result.split(key).join(replacements[key]);
}
return result;
}
function replaceInCollection(collection, replacements) {
return collection.map(item => {
if (item.createSql) {
return {
...item,
createSql: replaceInText(item.createSql, replacements),
};
}
return item;
});
}
const sqlTextReplacementTransform = replacements => database => {
return {
...database,
views: replaceInCollection(database.views, replacements),
};
};
module.exports = sqlTextReplacementTransform;

View File

@ -37,6 +37,7 @@
"json-stable-stringify": "^1.0.1",
"lodash": "^4.17.21",
"pinomin": "^1.0.4",
"toposort": "^2.0.2",
"uuid": "^3.4.0"
}
}

View File

@ -686,7 +686,8 @@ export class SqlDumper implements AlterProcessor {
}
createSqlObject(obj: SqlObjectInfo) {
this.putCmd(obj.createSql);
this.putRaw(obj.createSql);
this.endCommand();
}
getSqlObjectSqlName(ojectTypeField: string) {

View File

@ -498,22 +498,25 @@ export class AlterPlan {
return this.operations;
}
const fks = [];
const res = this.operations.map(op => {
if (op.operationType == 'createTable') {
fks.push(...(op.newObject.foreignKeys || []));
return {
...op,
newObject: {
...op.newObject,
foreignKeys: [],
},
};
}
return op;
});
const res = this.operations
.filter(op => op.operationType != 'createConstraint')
.map(op => {
if (op.operationType == 'createTable') {
fks.push(...(op.newObject.foreignKeys || []));
return {
...op,
newObject: {
...op.newObject,
foreignKeys: [],
},
};
}
return op;
});
return [
...res,
...this.operations.filter(op => op.operationType == 'createConstraint'),
...fks.map(
fk =>
({

View File

@ -7,6 +7,7 @@ import type {
SqlDialect,
SqlObjectInfo,
TableInfo,
ViewInfo,
} from 'dbgate-types';
import uuidv1 from 'uuid/v1';
import { AlterPlan } from './alterPlan';
@ -15,6 +16,8 @@ import _omit from 'lodash/omit';
import _cloneDeep from 'lodash/cloneDeep';
import _isEqual from 'lodash/isEqual';
import _pick from 'lodash/pick';
import _compact from 'lodash/compact';
import toposort from 'toposort';
type DbDiffSchemaMode = 'strict' | 'ignore' | 'ignoreImplicit';
@ -504,6 +507,40 @@ export function createAlterTablePlan(
return plan;
}
function sortViewsByDependency(views: ViewInfo[]): ViewInfo[] {
const viewNames: string[] = [];
const viewDict: { [name: string]: string } = {};
for (const view of views) {
if (!viewNames.includes(view.pureName)) {
viewNames.push(view.pureName);
}
viewDict[view.pureName] = viewDict[view.pureName]
? `${viewDict[view.pureName]} ${view.createSql}}`
: view.createSql;
}
const edges = [];
for (const viewName of viewNames) {
edges.push([viewName, null]);
const viewText = viewDict[viewName];
for (const otherView of viewNames) {
if (otherView === viewName) continue;
if ((' ' + viewText + ' ').match('[\\W]' + otherView + '[\\W]')) {
edges.push([otherView, viewName]);
}
}
}
const ordered: string[] = _compact(toposort(edges));
const res: ViewInfo[] = [];
for (const viewName of ordered) {
res.push(...views.filter(x => x.pureName == viewName));
}
return res;
}
export function createAlterDatabasePlan(
oldDb: DatabaseInfo,
newDb: DatabaseInfo,
@ -539,7 +576,13 @@ export function createAlterDatabasePlan(
}
}
}
for (const newobj of newDb[objectTypeField] || []) {
let newList = newDb[objectTypeField] || [];
if (objectTypeField == 'views') {
newList = sortViewsByDependency(newList);
}
for (const newobj of newList) {
const oldobj = (oldDb[objectTypeField] || []).find(x => x.pairingId == newobj.pairingId);
if (objectTypeField == 'tables') {
if (oldobj == null) {

View File

@ -4,6 +4,10 @@ import { splitQuery } from 'dbgate-query-splitter';
import { dumpSqlSelect } from 'dbgate-sqltree';
import { EngineDriver, QueryResult, RunScriptOptions } from 'dbgate-types';
import { detectSqlFilterBehaviour } from './detectSqlFilterBehaviour';
import { getLogger } from './getLogger';
import { getLimitedQuery } from './stringTools';
const logger = getLogger('driverBase');
const dialect = {
limitSelect: true,
@ -71,6 +75,9 @@ export const driverBase = {
}
for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
try {
if (options?.logScriptItems) {
logger.info({ sql: getLimitedQuery(sqlItem as string) }, `Execute script item`);
}
await this.query(pool, sqlItem, { discardResult: true, ...options?.queryOptions });
} catch (err) {
if (options?.useTransaction && this.supportsTransactions) {

View File

@ -511,3 +511,13 @@ export function safeFormatDate(date) {
return date?.toString();
}
}
export function getLimitedQuery(sql: string): string {
if (!sql) {
return sql;
}
if (sql.length > 1000) {
return sql.substring(0, 1000) + '...';
}
return sql;
}

View File

@ -74,10 +74,11 @@ function columnInfoFromYaml(column: ColumnInfoYaml, table: TableInfoYaml): Colum
const res: ColumnInfo = {
pureName: table.name,
columnName: column.name,
dataType: column.length ? `${column.type}(${column.length})` : column.type,
dataType: column.length ? `${column.type}(${column.length < 0 ? 'max' : column.length})` : column.type,
autoIncrement: column.autoIncrement,
notNull: column.notNull || (table.primaryKey && table.primaryKey.includes(column.name)),
defaultValue: column.default,
defaultConstraint: column.default != null ? `DF_${table.name}_${column.name}` : undefined,
};
return res;
}
@ -108,6 +109,7 @@ function convertForeignKeyFromYaml(
if (!refTable || !refTable.primaryKey) return null;
return {
constraintType: 'foreignKey',
constraintName: `FK_${table.name}_${col.name}`,
pureName: table.name,
refTableName: col.references,
deleteAction: col.refDeleteAction,
@ -133,6 +135,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
res.primaryKey = {
pureName: table.name,
constraintType: 'primaryKey',
constraintName: `PK_${table.name}`,
columns: table.primaryKey.map(columnName => ({ columnName })),
};
}
@ -140,6 +143,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
res.sortingKey = {
pureName: table.name,
constraintType: 'sortingKey',
constraintName: `SK_${table.name}`,
columns: table.sortingKey.map(columnName => ({ columnName })),
};
}

View File

@ -25,6 +25,7 @@ export interface StreamOptions {
export interface RunScriptOptions {
useTransaction: boolean;
logScriptItems?: boolean;
queryOptions?: QueryOptions;
}

View File

@ -108,13 +108,13 @@ class MsSqlDumper extends SqlDumper {
}
guessDefaultName(col) {
return col.defaultConstraint || `DF${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
return col.defaultConstraint || `DF_${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
}
createDefault(col) {
if (!col.defaultValue) return;
const defsql = col.defaultValue;
if (!defsql) {
if (col.defaultValue == null) return;
const defsql = col.defaultValue?.toString();
if (defsql) {
const defname = this.guessDefaultName(col);
this.putCmd('^alter ^table %f ^add ^constraint %i ^default %s for %i', col, defname, defsql, col.columnName);
}

2
test/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
deployTest.js
database

View File

@ -11173,6 +11173,11 @@ toidentifier@1.0.1:
resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
toposort@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/toposort/-/toposort-2.0.2.tgz#ae21768175d1559d48bef35420b2f4962f09c330"
integrity sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==
totalist@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df"