From 6c718981d60cd77e624133904184885c518cc811 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 15 Apr 2022 20:12:40 +0200 Subject: [PATCH] mysql dumper POC --- packages/api/src/shell/dumpDatabase.js | 38 +++++++++ packages/api/src/shell/index.js | 2 + packages/api/src/utility/connectUtility.js | 4 +- packages/tools/src/ScriptWriter.ts | 14 ++++ packages/types/engines.d.ts | 6 ++ .../web/src/appobj/DatabaseAppObject.svelte | 11 ++- packages/web/src/utility/exportFileTools.ts | 79 +++++++++++++------ plugins/dbgate-plugin-mysql/package.json | 9 ++- .../src/backend/drivers.js | 14 +++- .../src/frontend/drivers.js | 1 + yarn.lock | 14 ++++ 11 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 packages/api/src/shell/dumpDatabase.js diff --git a/packages/api/src/shell/dumpDatabase.js b/packages/api/src/shell/dumpDatabase.js new file mode 100644 index 00000000..f777ba5e --- /dev/null +++ b/packages/api/src/shell/dumpDatabase.js @@ -0,0 +1,38 @@ +const requireEngineDriver = require('../utility/requireEngineDriver'); +const connectUtility = require('../utility/connectUtility'); + +function doDump(dumper) { + return new Promise((resolve, reject) => { + dumper.once('end', () => { + resolve(true); + }); + dumper.once('error', err => { + reject(err); + }); + dumper.run(); + }); +} + +async function dumpDatabase({ + connection = undefined, + systemConnection = undefined, + driver = undefined, + outputFile, + databaseName, + schemaName, +}) { + console.log(`Dumping database`); + + if (!driver) driver = requireEngineDriver(connection); + const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true })); + console.log(`Connected.`); + + const dumper = await driver.createBackupDumper(pool, { + outputFile, + databaseName, + schemaName, + }); + await doDump(dumper); +} + +module.exports = dumpDatabase; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 4a2946c2..e663d4cb 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -21,6 +21,7 @@ const executeQuery = require('./executeQuery'); const loadFile = require('./loadFile'); const deployDb = require('./deployDb'); const initializeApiEnvironment = require('./initializeApiEnvironment'); +const dumpDatabase = require('./dumpDatabase'); const dbgateApi = { queryReader, @@ -45,6 +46,7 @@ const dbgateApi = { loadFile, deployDb, initializeApiEnvironment, + dumpDatabase, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/utility/connectUtility.js b/packages/api/src/utility/connectUtility.js index 4d8f0ee6..5fa33ff6 100644 --- a/packages/api/src/utility/connectUtility.js +++ b/packages/api/src/utility/connectUtility.js @@ -39,7 +39,7 @@ async function loadConnection(driver, storedConnection, connectionMode) { return storedConnection; } -async function connectUtility(driver, storedConnection, connectionMode) { +async function connectUtility(driver, storedConnection, connectionMode, additionalOptions = null) { const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode); const connection = { @@ -93,7 +93,7 @@ async function connectUtility(driver, storedConnection, connectionMode) { } } - const conn = await driver.connect(connection); + const conn = await driver.connect({ ...connection, ...additionalOptions }); return conn; } diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index 23dae116..0e53e2ed 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -49,6 +49,10 @@ export class ScriptWriter { } } + dumpDatabase(options) { + this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`); + } + comment(s) { this._put(`// ${s}`); } @@ -121,6 +125,13 @@ export class ScriptWriterJson { }); } + dumpDatabase(options) { + this.commands.push({ + type: 'dumpDatabase', + options, + }); + } + getScript(schedule = null) { return { type: 'json', @@ -158,6 +169,9 @@ export function jsonScriptToJavascript(json) { case 'comment': script.comment(cmd.text); break; + case 'dumpDatabase': + script.dumpDatabase(cmd.options); + break; } } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 44cd6696..5ee6f2f9 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -51,6 +51,10 @@ export interface SupportedDbKeyType { showItemList?: boolean; } +export interface SqlBackupDumper { + run(); +} + export interface EngineDriver { engine: string; title: string; @@ -60,6 +64,7 @@ export interface EngineDriver { readOnlySessions: boolean; supportedKeyTypes: SupportedDbKeyType[]; supportsDatabaseUrl?: boolean; + supportsDatabaseDump?: boolean; isElectronOnly?: boolean; supportedCreateDatabase?: boolean; showConnectionField?: (field: string, values: any) => boolean; @@ -99,6 +104,7 @@ export interface EngineDriver { dialect: SqlDialect; dialectByVersion(version): SqlDialect; createDumper(options = null): SqlDumper; + createBackupDumper(pool: any, options): Promise; getAuthTypes(): EngineAuthType[]; readCollection(pool: any, options: ReadCollectionOptions): Promise; updateCollection(pool: any, changeSet: any): Promise; diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index a92010ed..c09f0f0a 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -86,6 +86,10 @@ }); }; + const handleSqlDump = () => { + exportSqlDump(connection, name); + }; + const handleShowDiagram = async () => { const db = await getDatabaseInfo({ conid: connection._id, @@ -200,8 +204,10 @@ driver?.databaseEngineTypes?.includes('sql') && { onClick: handleNewTable, text: 'New table' }, driver?.databaseEngineTypes?.includes('document') && { onClick: handleNewCollection, text: 'New collection' }, { divider: true }, - isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import' }, - isSqlOrDoc && { onClick: handleExport, text: 'Export' }, + isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import wizard' }, + isSqlOrDoc && { onClick: handleExport, text: 'Export wizard' }, + driver?.supportsDatabaseDump && { onClick: handleSqlDump, text: 'Backup/export SQL dump' }, + { divider: true }, isSqlOrDoc && { onClick: handleShowDiagram, text: 'Show diagram' }, isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' }, isSqlOrDoc && { onClick: handleOpenJsonModel, text: 'Open model as JSON' }, @@ -267,6 +273,7 @@ import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte'; import { filterAppsForDatabase } from '../utility/appTools'; import newQuery from '../query/newQuery'; + import { exportSqlDump } from '../utility/exportFileTools'; export let data; export let passProps; diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index 93e753df..6f7856f1 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -6,46 +6,28 @@ import { apiCall, apiOff, apiOn } from './api'; import { normalizeExportColumnMap } from '../impexp/createImpExpScript'; import { getCurrentConfig } from '../stores'; -export async function exportQuickExportFile(dataName, reader, format, columnMap = null) { +export async function saveExportedFile(filters, defaultPath, extension, dataName, getScript: (filaPath: string) => {}) { const electron = getElectron(); let filePath; let pureFileName; if (electron) { - const filters = [{ name: format.label, extensions: [format.extension] }]; filePath = await electron.showSaveDialog({ filters, - defaultPath: `${dataName}.${format.extension}`, + defaultPath, properties: ['showOverwriteConfirmation'], }); } else { - const resp = await apiCall('files/generate-uploads-file', { extension: format.extension }); + const resp = await apiCall('files/generate-uploads-file', { extension }); filePath = resp.filePath; pureFileName = resp.fileName; } if (!filePath) return; - const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); + const script = getScript(filePath); - const sourceVar = script.allocVariable(); - script.assign(sourceVar, reader.functionName, reader.props); - - const targetVar = script.allocVariable(); - const writer = format.createWriter(filePath, dataName); - script.assign(targetVar, writer.functionName, writer.props); - - const colmap = normalizeExportColumnMap(columnMap); - let colmapVar = null; - if (colmap) { - colmapVar = script.allocVariable(); - script.assignValue(colmapVar, colmap); - } - - script.copyStream(sourceVar, targetVar, colmapVar); - script.endLine(); - - const resp = await apiCall('runners/start', { script: script.getScript() }); + const resp = await apiCall('runners/start', { script }); const runid = resp.runid; let isCanceled = false; @@ -77,6 +59,57 @@ export async function exportQuickExportFile(dataName, reader, format, columnMap apiOn(`runner-done-${runid}`, handleRunnerDone); } +export async function exportQuickExportFile(dataName, reader, format, columnMap = null) { + await saveExportedFile( + [{ name: format.label, extensions: [format.extension] }], + `${dataName}.${format.extension}`, + format.extension, + dataName, + filePath => { + const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); + + const sourceVar = script.allocVariable(); + script.assign(sourceVar, reader.functionName, reader.props); + + const targetVar = script.allocVariable(); + const writer = format.createWriter(filePath, dataName); + script.assign(targetVar, writer.functionName, writer.props); + + const colmap = normalizeExportColumnMap(columnMap); + let colmapVar = null; + if (colmap) { + colmapVar = script.allocVariable(); + script.assignValue(colmapVar, colmap); + } + + script.copyStream(sourceVar, targetVar, colmapVar); + script.endLine(); + + return script.getScript(); + } + ); +} + +export async function exportSqlDump(connection, databaseName) { + await saveExportedFile( + [{ name: 'SQL files', extensions: ['sql'] }], + `${databaseName}.sql`, + 'sql', + `${databaseName}-dump`, + filePath => { + const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); + + script.dumpDatabase({ + connection, + databaseName, + outputFile: filePath, + }); + + return script.getScript(); + } + ); +} + export async function saveFileToDisk( filePathFunc, options: any = { formatLabel: 'HTML page', formatExtension: 'html' } diff --git a/plugins/dbgate-plugin-mysql/package.json b/plugins/dbgate-plugin-mysql/package.json index 565e0bfe..12735168 100644 --- a/plugins/dbgate-plugin-mysql/package.json +++ b/plugins/dbgate-plugin-mysql/package.json @@ -31,11 +31,12 @@ "prepublishOnly": "yarn build" }, "devDependencies": { + "antares-mysql-dumper": "link:/Users/jena/jenasoft/antares-mysql-dumper", "dbgate-plugin-tools": "^1.0.7", "dbgate-query-splitter": "^4.8.3", - "webpack": "^4.42.0", - "webpack-cli": "^3.3.11", "dbgate-tools": "^4.1.1", - "mysql2": "^2.2.5" + "mysql2": "^2.2.5", + "webpack": "^4.42.0", + "webpack-cli": "^3.3.11" } -} \ No newline at end of file +} diff --git a/plugins/dbgate-plugin-mysql/src/backend/drivers.js b/plugins/dbgate-plugin-mysql/src/backend/drivers.js index b84cd775..cc4526b6 100644 --- a/plugins/dbgate-plugin-mysql/src/backend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/backend/drivers.js @@ -4,6 +4,7 @@ const driverBases = require('../frontend/drivers'); const Analyser = require('./Analyser'); const mysql2 = require('mysql2'); const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools'); +const { MySqlDumper } = require('antares-mysql-dumper'); function extractColumns(fields) { if (fields) { @@ -28,7 +29,7 @@ const drivers = driverBases.map(driverBase => ({ ...driverBase, analyserClass: Analyser, - async connect({ server, port, user, password, database, ssl, isReadOnly }) { + async connect({ server, port, user, password, database, ssl, isReadOnly, forceRowsAsObjects }) { const connection = mysql2.createConnection({ host: server, port, @@ -36,7 +37,7 @@ const drivers = driverBases.map(driverBase => ({ password, database, ssl, - rowsAsArray: true, + rowsAsArray: forceRowsAsObjects ? false : true, supportBigNumbers: true, bigNumberStrings: true, dateStrings: true, @@ -172,6 +173,15 @@ const drivers = driverBases.map(driverBase => ({ // @ts-ignore return createBulkInsertStreamBase(this, stream, pool, name, options); }, + async createBackupDumper(pool, options) { + const { outputFile, databaseName, schemaName } = options; + const res = new MySqlDumper({ + connection: pool, + schema: databaseName || schemaName, + outputFile, + }); + return res; + }, })); module.exports = drivers; diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 77c27ba3..87ba6f93 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -47,6 +47,7 @@ const mysqlDriverBase = { defaultPort: 3306, getQuerySplitterOptions: () => mysqlSplitterOptions, readOnlySessions: true, + supportsDatabaseDump: true, getNewObjectTemplates() { return [ diff --git a/yarn.lock b/yarn.lock index 55e4f343..2209052f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1024,6 +1024,11 @@ resolved "https://registry.yarnpkg.com/@tsconfig/svelte/-/svelte-1.0.10.tgz#30ec7feeee0bdf38b12a50f0686f8a2e7b6b9dc0" integrity sha512-EBrpH2iXXfaf/9z81koiDYkp2mlwW2XzFcAqn6qh7VKyP8zBvHHAQzNhY+W9vH5arAjmGAm5g8ElWq6YmXm3ig== +"@turf/helpers@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e" + integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw== + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": version "7.1.14" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402" @@ -1668,6 +1673,10 @@ ansi-styles@^5.0.0: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== +"antares-mysql-dumper@link:../antares-mysql-dumper": + version "0.0.0" + uid "" + anymatch@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb" @@ -7389,6 +7398,11 @@ moment@^2.24.0: resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== +moment@^2.29.2: + version "2.29.2" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4" + integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg== + mongodb-client-encryption@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/mongodb-client-encryption/-/mongodb-client-encryption-1.2.3.tgz#0078f2cf385762e052b0c12d9be256eb1ef9a347"