mirror of
https://github.com/dbgate/dbgate
synced 2024-11-21 07:26:43 +00:00
db deploy fixes
This commit is contained in:
parent
456d3ba42e
commit
2704825d03
21
packages/api/src/shell/dataTypeMapperTransform.js
Normal file
21
packages/api/src/shell/dataTypeMapperTransform.js
Normal file
@ -0,0 +1,21 @@
|
||||
const dataTypeMapperTransform = (oldType, newType) => database => {
|
||||
return {
|
||||
...database,
|
||||
tables: database.tables.map(table => {
|
||||
return {
|
||||
...table,
|
||||
columns: table.columns.map(column => {
|
||||
if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
|
||||
return {
|
||||
...column,
|
||||
dataType: newType,
|
||||
};
|
||||
}
|
||||
return column;
|
||||
}),
|
||||
};
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = dataTypeMapperTransform;
|
@ -1,7 +1,15 @@
|
||||
const generateDeploySql = require('./generateDeploySql');
|
||||
const executeQuery = require('./executeQuery');
|
||||
|
||||
async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
|
||||
async function deployDb({
|
||||
connection,
|
||||
systemConnection,
|
||||
driver,
|
||||
analysedStructure,
|
||||
modelFolder,
|
||||
loadedDbModel,
|
||||
modelTransforms,
|
||||
}) {
|
||||
const { sql } = await generateDeploySql({
|
||||
connection,
|
||||
systemConnection,
|
||||
@ -9,9 +17,10 @@ async function deployDb({ connection, systemConnection, driver, analysedStructur
|
||||
analysedStructure,
|
||||
modelFolder,
|
||||
loadedDbModel,
|
||||
modelTransforms,
|
||||
});
|
||||
// console.log('RUNNING DEPLOY SCRIPT:', sql);
|
||||
await executeQuery({ connection, systemConnection, driver, sql });
|
||||
await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
|
||||
}
|
||||
|
||||
module.exports = deployDb;
|
||||
|
@ -1,11 +1,19 @@
|
||||
const requireEngineDriver = require('../utility/requireEngineDriver');
|
||||
const connectUtility = require('../utility/connectUtility');
|
||||
const { getLogger } = require('dbgate-tools');
|
||||
const { getLogger, getLimitedQuery } = require('dbgate-tools');
|
||||
|
||||
const logger = getLogger('execQuery');
|
||||
|
||||
async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
|
||||
logger.info({ sql }, `Execute query`);
|
||||
async function executeQuery({
|
||||
connection = undefined,
|
||||
systemConnection = undefined,
|
||||
driver = undefined,
|
||||
sql,
|
||||
logScriptItems = false,
|
||||
}) {
|
||||
if (!logScriptItems) {
|
||||
logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
|
||||
}
|
||||
|
||||
if (!driver) driver = requireEngineDriver(connection);
|
||||
const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
|
||||
@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
|
||||
try {
|
||||
logger.info(`Connected.`);
|
||||
|
||||
await driver.script(dbhan, sql);
|
||||
await driver.script(dbhan, sql, { logScriptItems });
|
||||
} finally {
|
||||
if (!systemConnection) {
|
||||
await driver.close(dbhan);
|
||||
|
@ -18,6 +18,7 @@ async function generateDeploySql({
|
||||
analysedStructure = undefined,
|
||||
modelFolder = undefined,
|
||||
loadedDbModel = undefined,
|
||||
modelTransforms = undefined,
|
||||
}) {
|
||||
if (!driver) driver = requireEngineDriver(connection);
|
||||
|
||||
@ -28,9 +29,15 @@ async function generateDeploySql({
|
||||
analysedStructure = await driver.analyseFull(dbhan);
|
||||
}
|
||||
|
||||
const deployedModel = generateDbPairingId(
|
||||
extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
|
||||
);
|
||||
let deployedModelSource = loadedDbModel
|
||||
? databaseInfoFromYamlModel(loadedDbModel)
|
||||
: await importDbModel(modelFolder);
|
||||
|
||||
for (const transform of modelTransforms || []) {
|
||||
deployedModelSource = transform(deployedModelSource);
|
||||
}
|
||||
|
||||
const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
|
||||
const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
|
||||
const opts = {
|
||||
...modelCompareDbDiffOptions,
|
||||
@ -57,7 +64,7 @@ async function generateDeploySql({
|
||||
deployedModel,
|
||||
driver
|
||||
);
|
||||
|
||||
|
||||
return res;
|
||||
} finally {
|
||||
if (!systemConnection) {
|
||||
|
@ -30,6 +30,8 @@ const dataDuplicator = require('./dataDuplicator');
|
||||
const dbModelToJson = require('./dbModelToJson');
|
||||
const jsonToDbModel = require('./jsonToDbModel');
|
||||
const jsonReader = require('./jsonReader');
|
||||
const dataTypeMapperTransform = require('./dataTypeMapperTransform');
|
||||
const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
|
||||
|
||||
const dbgateApi = {
|
||||
queryReader,
|
||||
@ -63,6 +65,8 @@ const dbgateApi = {
|
||||
dataDuplicator,
|
||||
dbModelToJson,
|
||||
jsonToDbModel,
|
||||
dataTypeMapperTransform,
|
||||
sqlTextReplacementTransform,
|
||||
};
|
||||
|
||||
requirePlugin.initializeDbgateApi(dbgateApi);
|
||||
|
28
packages/api/src/shell/sqlTextReplacementTransform.js
Normal file
28
packages/api/src/shell/sqlTextReplacementTransform.js
Normal file
@ -0,0 +1,28 @@
|
||||
function replaceInText(text, replacements) {
|
||||
let result = text;
|
||||
for (const key of Object.keys(replacements)) {
|
||||
result = result.split(key).join(replacements[key]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function replaceInCollection(collection, replacements) {
|
||||
return collection.map(item => {
|
||||
if (item.createSql) {
|
||||
return {
|
||||
...item,
|
||||
createSql: replaceInText(item.createSql, replacements),
|
||||
};
|
||||
}
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
const sqlTextReplacementTransform = replacements => database => {
|
||||
return {
|
||||
...database,
|
||||
views: replaceInCollection(database.views, replacements),
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = sqlTextReplacementTransform;
|
@ -37,6 +37,7 @@
|
||||
"json-stable-stringify": "^1.0.1",
|
||||
"lodash": "^4.17.21",
|
||||
"pinomin": "^1.0.4",
|
||||
"toposort": "^2.0.2",
|
||||
"uuid": "^3.4.0"
|
||||
}
|
||||
}
|
||||
|
@ -686,7 +686,8 @@ export class SqlDumper implements AlterProcessor {
|
||||
}
|
||||
|
||||
createSqlObject(obj: SqlObjectInfo) {
|
||||
this.putCmd(obj.createSql);
|
||||
this.putRaw(obj.createSql);
|
||||
this.endCommand();
|
||||
}
|
||||
|
||||
getSqlObjectSqlName(ojectTypeField: string) {
|
||||
|
@ -498,22 +498,25 @@ export class AlterPlan {
|
||||
return this.operations;
|
||||
}
|
||||
const fks = [];
|
||||
const res = this.operations.map(op => {
|
||||
if (op.operationType == 'createTable') {
|
||||
fks.push(...(op.newObject.foreignKeys || []));
|
||||
return {
|
||||
...op,
|
||||
newObject: {
|
||||
...op.newObject,
|
||||
foreignKeys: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
return op;
|
||||
});
|
||||
const res = this.operations
|
||||
.filter(op => op.operationType != 'createConstraint')
|
||||
.map(op => {
|
||||
if (op.operationType == 'createTable') {
|
||||
fks.push(...(op.newObject.foreignKeys || []));
|
||||
return {
|
||||
...op,
|
||||
newObject: {
|
||||
...op.newObject,
|
||||
foreignKeys: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
return op;
|
||||
});
|
||||
|
||||
return [
|
||||
...res,
|
||||
...this.operations.filter(op => op.operationType == 'createConstraint'),
|
||||
...fks.map(
|
||||
fk =>
|
||||
({
|
||||
|
@ -7,6 +7,7 @@ import type {
|
||||
SqlDialect,
|
||||
SqlObjectInfo,
|
||||
TableInfo,
|
||||
ViewInfo,
|
||||
} from 'dbgate-types';
|
||||
import uuidv1 from 'uuid/v1';
|
||||
import { AlterPlan } from './alterPlan';
|
||||
@ -15,6 +16,8 @@ import _omit from 'lodash/omit';
|
||||
import _cloneDeep from 'lodash/cloneDeep';
|
||||
import _isEqual from 'lodash/isEqual';
|
||||
import _pick from 'lodash/pick';
|
||||
import _compact from 'lodash/compact';
|
||||
import toposort from 'toposort';
|
||||
|
||||
type DbDiffSchemaMode = 'strict' | 'ignore' | 'ignoreImplicit';
|
||||
|
||||
@ -504,6 +507,40 @@ export function createAlterTablePlan(
|
||||
return plan;
|
||||
}
|
||||
|
||||
function sortViewsByDependency(views: ViewInfo[]): ViewInfo[] {
|
||||
const viewNames: string[] = [];
|
||||
const viewDict: { [name: string]: string } = {};
|
||||
|
||||
for (const view of views) {
|
||||
if (!viewNames.includes(view.pureName)) {
|
||||
viewNames.push(view.pureName);
|
||||
}
|
||||
viewDict[view.pureName] = viewDict[view.pureName]
|
||||
? `${viewDict[view.pureName]} ${view.createSql}}`
|
||||
: view.createSql;
|
||||
}
|
||||
|
||||
const edges = [];
|
||||
for (const viewName of viewNames) {
|
||||
edges.push([viewName, null]);
|
||||
const viewText = viewDict[viewName];
|
||||
for (const otherView of viewNames) {
|
||||
if (otherView === viewName) continue;
|
||||
if ((' ' + viewText + ' ').match('[\\W]' + otherView + '[\\W]')) {
|
||||
edges.push([otherView, viewName]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ordered: string[] = _compact(toposort(edges));
|
||||
|
||||
const res: ViewInfo[] = [];
|
||||
for (const viewName of ordered) {
|
||||
res.push(...views.filter(x => x.pureName == viewName));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
export function createAlterDatabasePlan(
|
||||
oldDb: DatabaseInfo,
|
||||
newDb: DatabaseInfo,
|
||||
@ -539,7 +576,13 @@ export function createAlterDatabasePlan(
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const newobj of newDb[objectTypeField] || []) {
|
||||
|
||||
let newList = newDb[objectTypeField] || [];
|
||||
if (objectTypeField == 'views') {
|
||||
newList = sortViewsByDependency(newList);
|
||||
}
|
||||
|
||||
for (const newobj of newList) {
|
||||
const oldobj = (oldDb[objectTypeField] || []).find(x => x.pairingId == newobj.pairingId);
|
||||
if (objectTypeField == 'tables') {
|
||||
if (oldobj == null) {
|
||||
|
@ -4,6 +4,10 @@ import { splitQuery } from 'dbgate-query-splitter';
|
||||
import { dumpSqlSelect } from 'dbgate-sqltree';
|
||||
import { EngineDriver, QueryResult, RunScriptOptions } from 'dbgate-types';
|
||||
import { detectSqlFilterBehaviour } from './detectSqlFilterBehaviour';
|
||||
import { getLogger } from './getLogger';
|
||||
import { getLimitedQuery } from './stringTools';
|
||||
|
||||
const logger = getLogger('driverBase');
|
||||
|
||||
const dialect = {
|
||||
limitSelect: true,
|
||||
@ -71,6 +75,9 @@ export const driverBase = {
|
||||
}
|
||||
for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
|
||||
try {
|
||||
if (options?.logScriptItems) {
|
||||
logger.info({ sql: getLimitedQuery(sqlItem as string) }, `Execute script item`);
|
||||
}
|
||||
await this.query(pool, sqlItem, { discardResult: true, ...options?.queryOptions });
|
||||
} catch (err) {
|
||||
if (options?.useTransaction && this.supportsTransactions) {
|
||||
|
@ -511,3 +511,13 @@ export function safeFormatDate(date) {
|
||||
return date?.toString();
|
||||
}
|
||||
}
|
||||
|
||||
export function getLimitedQuery(sql: string): string {
|
||||
if (!sql) {
|
||||
return sql;
|
||||
}
|
||||
if (sql.length > 1000) {
|
||||
return sql.substring(0, 1000) + '...';
|
||||
}
|
||||
return sql;
|
||||
}
|
||||
|
@ -74,10 +74,11 @@ function columnInfoFromYaml(column: ColumnInfoYaml, table: TableInfoYaml): Colum
|
||||
const res: ColumnInfo = {
|
||||
pureName: table.name,
|
||||
columnName: column.name,
|
||||
dataType: column.length ? `${column.type}(${column.length})` : column.type,
|
||||
dataType: column.length ? `${column.type}(${column.length < 0 ? 'max' : column.length})` : column.type,
|
||||
autoIncrement: column.autoIncrement,
|
||||
notNull: column.notNull || (table.primaryKey && table.primaryKey.includes(column.name)),
|
||||
defaultValue: column.default,
|
||||
defaultConstraint: column.default != null ? `DF_${table.name}_${column.name}` : undefined,
|
||||
};
|
||||
return res;
|
||||
}
|
||||
@ -108,6 +109,7 @@ function convertForeignKeyFromYaml(
|
||||
if (!refTable || !refTable.primaryKey) return null;
|
||||
return {
|
||||
constraintType: 'foreignKey',
|
||||
constraintName: `FK_${table.name}_${col.name}`,
|
||||
pureName: table.name,
|
||||
refTableName: col.references,
|
||||
deleteAction: col.refDeleteAction,
|
||||
@ -133,6 +135,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
|
||||
res.primaryKey = {
|
||||
pureName: table.name,
|
||||
constraintType: 'primaryKey',
|
||||
constraintName: `PK_${table.name}`,
|
||||
columns: table.primaryKey.map(columnName => ({ columnName })),
|
||||
};
|
||||
}
|
||||
@ -140,6 +143,7 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
|
||||
res.sortingKey = {
|
||||
pureName: table.name,
|
||||
constraintType: 'sortingKey',
|
||||
constraintName: `SK_${table.name}`,
|
||||
columns: table.sortingKey.map(columnName => ({ columnName })),
|
||||
};
|
||||
}
|
||||
|
1
packages/types/engines.d.ts
vendored
1
packages/types/engines.d.ts
vendored
@ -25,6 +25,7 @@ export interface StreamOptions {
|
||||
|
||||
export interface RunScriptOptions {
|
||||
useTransaction: boolean;
|
||||
logScriptItems?: boolean;
|
||||
queryOptions?: QueryOptions;
|
||||
}
|
||||
|
||||
|
@ -108,13 +108,13 @@ class MsSqlDumper extends SqlDumper {
|
||||
}
|
||||
|
||||
guessDefaultName(col) {
|
||||
return col.defaultConstraint || `DF${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
|
||||
return col.defaultConstraint || `DF_${col.schemaName || 'dbo'}_${col.pureName}_col.columnName`;
|
||||
}
|
||||
|
||||
createDefault(col) {
|
||||
if (!col.defaultValue) return;
|
||||
const defsql = col.defaultValue;
|
||||
if (!defsql) {
|
||||
if (col.defaultValue == null) return;
|
||||
const defsql = col.defaultValue?.toString();
|
||||
if (defsql) {
|
||||
const defname = this.guessDefaultName(col);
|
||||
this.putCmd('^alter ^table %f ^add ^constraint %i ^default %s for %i', col, defname, defsql, col.columnName);
|
||||
}
|
||||
|
2
test/.gitignore
vendored
Normal file
2
test/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
deployTest.js
|
||||
database
|
@ -11173,6 +11173,11 @@ toidentifier@1.0.1:
|
||||
resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
|
||||
integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
|
||||
|
||||
toposort@^2.0.2:
|
||||
version "2.0.2"
|
||||
resolved "https://registry.yarnpkg.com/toposort/-/toposort-2.0.2.tgz#ae21768175d1559d48bef35420b2f4962f09c330"
|
||||
integrity sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==
|
||||
|
||||
totalist@^1.0.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df"
|
||||
|
Loading…
Reference in New Issue
Block a user