nocobase/packages/core/database/src/sync-runner.ts
ChengLei Shao fa97d0a642
feat: application backup and restore (#3268)
* fix: perform load action on boot main app

* feat: add dataType option in collection duplicator

* chore: reset optional dumpable config

* chore: dump command

* chore: dump & restore command

* chore: delay restore

* fix: dump test

* chore: restore command

* chore: dump command action

* chore: dumpable collection api

* chore: client collection option

* feat: backup& restore client

* chore: content disposition header in dump response

* chore: download backup field

* feat: collection origin option

* fix: test

* chore: collection manager collection origin

* chore: upload  backup field

* chore: upload restore file

* chore: upload restore file

* fix: test

* chore: backup and restore support learn more

* refactor: upload restore file

* refactor: upload restore file

* fix: test

* fix: test

* chore: dumpable collection with title

* chore: pg only test

* chore: test

* fix: test

* chore: test sleep

* style: locale improve

* refactor: download backup file

* refactor: start restore

* fix: restore key name

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* chore: unify duplicator option

* fix: dump empty collection

* chore: test

* chore: test

* style: style improve

* refactor: locale improve

* chore: dumpalbe collection orders

* style: style improve

* style: style improve

* style: icon adjust

* chore: nginx body size

* chore: get file status

* feat: run dump task

* feat: download api

* chore: backup files resourcer

* feat: restore destroy api

* chore: backup files resoucer

* feat: list backup files action

* chore: get collection meta from dumped file

* fix: dump file name

* fix: test

* chore: backup and restore ui

* chore: swagger api for backup & restore

* chore: api doc

* chore: api doc

* chore: api doc

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* fix: restore values

* style: style improve

* fix: download field respontype

* fix: restore form local file

* refactor: local improve

* refactor: delete backup file

* fix: in progress status

* refactor: locale improve

* refactor: locale improve

* refactor: style improve

* refactor: style improve

* refactor: style improve

* test: dump collection table attribute

* chore: dump collection with table attributes

* chore: test

* chore: create new table in restore

* fix: import error

* chore: restore table from backup file

* chore: sync collection after restore collections

* fix: restore json data

* style: style improve

* chore: restore with fields

* chore: test

* fix: test

* fix: test with underscored

* style: style improve

* fix: lock file state

* chore: add test file

* refactor: backup & restore plugin

* fix: mysql test

* chore: skip import view collection

* chore: restore collection with inherits topo order

* fix: import

* style: style improve

* fix: restore sequence fields

* fix: themeConfig collection duplicator option

* fix: restore with dialectOnly meta

* fix: throw error

* fix: restore

* fix: import backup file created in postgres into mysql

* fix: repeated items in inherits

* chore: upgrade after restore

* feat: check database env before restore

* feat: handle autoincr val in postgres

* chore: sqlite & mysql queryInterface

* chore: test

* fix: test

* chore: test

* fix: build

* fix: pg test

* fix: restore with date field

* chore: theme-config collection

* chore: chage import collections method to support collection origin

* chore: fallback get autoincr value in mysql

* fix: dataType normalize

* chore: delay restore

* chore: test

* fix: build

* feat: collectin onDump

* feat: collection onDump interface

* chore: dump with view collection

* chore: sync in restore

* refactor: locale improve

* refactor: code improve

* fix: test

* fix: data sync

* chore: rename backup & restore plugin

* chore: skip test

* style: style improve

* style: style improve

* style: style improve

* style: style improve

* chore: import version check

* chore: backup file dir

* chore: build

* fix: bugs

* fix: error

* fix: pageSize

* fix: import origin

* fix: improve code

* fix: remove namespace

* chore: dump rules config

* fix: dump custom collection

* chore: version

* fix: test

* fix: test

* fix: test

* fix: test

* chore: test

* fix: load custom collection

* fix: client

* fix: translation

* chore: code

* fix: bug

* fix:  support shared option

* fix: roles collection dumpRules

* chore: test

* fix: define collections

* chore: collection group

* fix: translation

* fix: translation

* fix: restore options

* chore: restore command

* chore: dump error

* fix: too many open files

---------

Co-authored-by: katherinehhh <katherine_15995@163.com>
Co-authored-by: chenos <chenlinxh@gmail.com>
2024-01-08 18:59:56 +08:00

177 lines
5.3 KiB
TypeScript

import { InheritedCollection } from './inherited-collection';
import lodash from 'lodash';
export class SyncRunner {
static async syncInheritModel(model: any, options: any) {
const { transaction } = options;
const inheritedCollection = model.collection as InheritedCollection;
const db = inheritedCollection.context.database;
const dialect = db.sequelize.getDialect();
const queryInterface = db.sequelize.getQueryInterface();
if (dialect != 'postgres') {
throw new Error('Inherit model is only supported on postgres');
}
const parents = inheritedCollection.parents;
if (!parents) {
throw new Error(
`Inherit model ${inheritedCollection.name} can't be created without parents, parents option is ${lodash
.castArray(inheritedCollection.options.inherits)
.join(', ')}`,
);
}
const tableName = inheritedCollection.getTableNameWithSchema();
const attributes = model.tableAttributes;
const childAttributes = lodash.pickBy(attributes, (value) => {
return !value.inherit;
});
if (
!(await inheritedCollection.existsInDb({
transaction,
}))
) {
let maxSequenceVal = 0;
let maxSequenceName;
// find max sequence
if (childAttributes.id && childAttributes.id.autoIncrement) {
for (const parent of parents) {
const sequenceNameResult = await queryInterface.sequelize.query(
`SELECT column_default
FROM information_schema.columns
WHERE table_name = '${parent.model.tableName}'
and table_schema = '${parent.collectionSchema()}'
and "column_name" = 'id';`,
{
transaction,
},
);
if (!sequenceNameResult[0].length) {
continue;
}
const columnDefault = sequenceNameResult[0][0]['column_default'];
if (!columnDefault) {
throw new Error(`Can't find sequence name of parent collection ${parent.options.name}`);
}
const regex = new RegExp(/nextval\('(.*)'::regclass\)/);
const match = regex.exec(columnDefault);
const sequenceName = match[1];
const sequenceCurrentValResult = await queryInterface.sequelize.query(
`select last_value
from ${sequenceName}`,
{
transaction,
},
);
const sequenceCurrentVal = parseInt(sequenceCurrentValResult[0][0]['last_value']);
if (sequenceCurrentVal > maxSequenceVal) {
maxSequenceName = sequenceName;
maxSequenceVal = sequenceCurrentVal;
}
}
}
await this.createTable(tableName, childAttributes, options, model, parents);
// if we have max sequence, set it to child table
if (maxSequenceName) {
const parentsDeep = Array.from(db.inheritanceMap.getParents(inheritedCollection.name)).map((parent) =>
db.getCollection(parent).getTableNameWithSchema(),
);
const sequenceTables = [...parentsDeep, tableName];
for (const sequenceTable of sequenceTables) {
const tableName = sequenceTable.tableName;
const schemaName = sequenceTable.schema;
const idColumnSql = `SELECT column_name
FROM information_schema.columns
WHERE table_name = '${tableName}'
and column_name = 'id'
and table_schema = '${schemaName}';
`;
const idColumnQuery = await queryInterface.sequelize.query(idColumnSql, {
transaction,
});
if (idColumnQuery[0].length == 0) {
continue;
}
await queryInterface.sequelize.query(
`alter table ${db.utils.quoteTable(sequenceTable)}
alter column id set default nextval('${maxSequenceName}')`,
{
transaction,
},
);
}
}
}
if (options.alter) {
const columns = await queryInterface.describeTable(tableName, options);
for (const attribute in childAttributes) {
const columnName = childAttributes[attribute].field;
if (!columns[columnName]) {
await queryInterface.addColumn(tableName, columnName, childAttributes[columnName], options);
}
}
}
}
static async createTable(tableName, attributes, options, model, parents) {
let sql = '';
options = { ...options };
if (options && options.uniqueKeys) {
lodash.forOwn(options.uniqueKeys, (uniqueKey) => {
if (uniqueKey.customIndex === undefined) {
uniqueKey.customIndex = true;
}
});
}
if (model) {
options.uniqueKeys = options.uniqueKeys || model.uniqueKeys;
}
const queryGenerator = model.queryGenerator;
attributes = lodash.mapValues(attributes, (attribute) => model.sequelize.normalizeAttribute(attribute));
attributes = queryGenerator.attributesToSQL(attributes, { table: tableName, context: 'createTable' });
sql = `${queryGenerator.createTableQuery(tableName, attributes, options)}`.replace(
';',
` INHERITS (${parents
.map((t) => {
return t.getTableNameWithSchema();
})
.join(', ')});`,
);
return await model.sequelize.query(sql, options);
}
}