From a75e931fd52c3f6973435c0ee18b20c4b64cd1d3 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 18 Apr 2022 14:22:16 +0200 Subject: [PATCH] import SQL dump POC --- packages/api/src/shell/importDatabase.js | 50 +++++++++++ packages/api/src/shell/index.js | 2 + packages/tools/src/ScriptWriter.ts | 14 +++ .../web/src/appobj/DatabaseAppObject.svelte | 8 ++ .../src/modals/ImportDatabaseDumpModal.svelte | 72 +++++++++++++++ packages/web/src/utility/exportFileTools.ts | 90 +++++++++++++------ 6 files changed, 207 insertions(+), 29 deletions(-) create mode 100644 packages/api/src/shell/importDatabase.js create mode 100644 packages/web/src/modals/ImportDatabaseDumpModal.svelte diff --git a/packages/api/src/shell/importDatabase.js b/packages/api/src/shell/importDatabase.js new file mode 100644 index 00000000..b82227b1 --- /dev/null +++ b/packages/api/src/shell/importDatabase.js @@ -0,0 +1,50 @@ +const fs = require('fs'); +const byline = require('byline'); +const requireEngineDriver = require('../utility/requireEngineDriver'); +const connectUtility = require('../utility/connectUtility'); +const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream'); +const download = require('./download'); +const stream = require('stream'); + +class ImportStream extends stream.Transform { + constructor(pool, driver) { + super({ objectMode: true }); + this.pool = pool; + this.driver = driver; + } + async _transform(chunk, encoding, cb) { + await this.driver.script(this.pool, chunk); + cb(); + } +} + +function awaitStreamEnd(stream) { + return new Promise((resolve, reject) => { + stream.once('end', () => { + resolve(true); + }); + stream.once('error', err => { + reject(err); + }); + }); +} + +async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) { + console.log(`Importing database`); + + if (!driver) driver = requireEngineDriver(connection); + const pool = systemConnection || (await connectUtility(driver, connection, 'write')); + console.log(`Connected.`); + + const downloadedFile = await download(inputFile); + + const fileStream = fs.createReadStream(downloadedFile, 'utf-8'); + const lineStream = byline(fileStream); + const splittedStream = splitQueryStream(lineStream, driver.getQuerySplitterOptions()); + const importStream = new ImportStream(pool, driver); + // @ts-ignore + splittedStream.pipe(importStream); + await awaitStreamEnd(splittedStream); +} + +module.exports = importDatabase; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index e663d4cb..8fb76cae 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -22,6 +22,7 @@ const loadFile = require('./loadFile'); const deployDb = require('./deployDb'); const initializeApiEnvironment = require('./initializeApiEnvironment'); const dumpDatabase = require('./dumpDatabase'); +const importDatabase = require('./importDatabase'); const dbgateApi = { queryReader, @@ -47,6 +48,7 @@ const dbgateApi = { deployDb, initializeApiEnvironment, dumpDatabase, + importDatabase, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/tools/src/ScriptWriter.ts b/packages/tools/src/ScriptWriter.ts index 0e53e2ed..cd84df05 100644 --- a/packages/tools/src/ScriptWriter.ts +++ b/packages/tools/src/ScriptWriter.ts @@ -53,6 +53,10 @@ export class ScriptWriter { this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`); } + importDatabase(options) { + this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`); + } + comment(s) { this._put(`// ${s}`); } @@ -132,6 +136,13 @@ export class ScriptWriterJson { }); } + importDatabase(options) { + this.commands.push({ + type: 'importDatabase', + options, + }); + } + getScript(schedule = null) { return { type: 'json', @@ -172,6 +183,9 @@ export function jsonScriptToJavascript(json) { case 'dumpDatabase': script.dumpDatabase(cmd.options); break; + case 'importDatabase': + script.importDatabase(cmd.options); + break; } } diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index c09f0f0a..4ac48d15 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -90,6 +90,12 @@ exportSqlDump(connection, name); }; + const handleSqlRestore = () => { + showModal(ImportDatabaseDumpModal, { + connection: { ...connection, database: name }, + }); + }; + const handleShowDiagram = async () => { const db = await getDatabaseInfo({ conid: connection._id, @@ -207,6 +213,7 @@ isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import wizard' }, isSqlOrDoc && { onClick: handleExport, text: 'Export wizard' }, driver?.supportsDatabaseDump && { onClick: handleSqlDump, text: 'Backup/export SQL dump' }, + driver?.supportsDatabaseDump && { onClick: handleSqlRestore, text: 'Restore/import SQL dump' }, { divider: true }, isSqlOrDoc && { onClick: handleShowDiagram, text: 'Show diagram' }, isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' }, @@ -274,6 +281,7 @@ import { filterAppsForDatabase } from '../utility/appTools'; import newQuery from '../query/newQuery'; import { exportSqlDump } from '../utility/exportFileTools'; + import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte'; export let data; export let passProps; diff --git a/packages/web/src/modals/ImportDatabaseDumpModal.svelte b/packages/web/src/modals/ImportDatabaseDumpModal.svelte new file mode 100644 index 00000000..7353713d --- /dev/null +++ b/packages/web/src/modals/ImportDatabaseDumpModal.svelte @@ -0,0 +1,72 @@ + + + + + Import database dump + +
Source: {inputLabel}
+ {#if electron} + + {:else} + + {/if} + + + + + handleSubmit(e.detail)} /> + + + + diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index 6f7856f1..d4dbd583 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -6,6 +6,57 @@ import { apiCall, apiOff, apiOn } from './api'; import { normalizeExportColumnMap } from '../impexp/createImpExpScript'; import { getCurrentConfig } from '../stores'; +export async function importSqlDump(inputFile, connection) { + const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); + + script.importDatabase({ + inputFile, + connection, + }); + + await runImportExportScript({ + script: script.getScript(), + runningMessage: 'Importing database', + canceledMessage: 'Database import canceled', + finishedMessage: 'Database import finished', + }); +} + +async function runImportExportScript({ script, runningMessage, canceledMessage, finishedMessage, afterFinish = null }) { + const electron = getElectron(); + + const resp = await apiCall('runners/start', { script }); + const runid = resp.runid; + let isCanceled = false; + + const snackId = showSnackbar({ + message: runningMessage, + icon: 'icon loading', + buttons: [ + { + label: 'Cancel', + onClick: () => { + isCanceled = true; + apiCall('runners/cancel', { runid }); + }, + }, + ], + }); + + function handleRunnerDone() { + closeSnackbar(snackId); + apiOff(`runner-done-${runid}`, handleRunnerDone); + if (isCanceled) { + showSnackbarError(canceledMessage); + } else { + showSnackbarInfo(finishedMessage); + if (afterFinish) afterFinish(); + } + } + + apiOn(`runner-done-${runid}`, handleRunnerDone); +} + export async function saveExportedFile(filters, defaultPath, extension, dataName, getScript: (filaPath: string) => {}) { const electron = getElectron(); @@ -27,36 +78,17 @@ export async function saveExportedFile(filters, defaultPath, extension, dataName const script = getScript(filePath); - const resp = await apiCall('runners/start', { script }); - const runid = resp.runid; - let isCanceled = false; - - const snackId = showSnackbar({ - message: `Exporting ${dataName}`, - icon: 'icon loading', - buttons: [ - { - label: 'Cancel', - onClick: () => { - isCanceled = true; - apiCall('runners/cancel', { runid }); - }, - }, - ], + runImportExportScript({ + script, + runningMessage: `Exporting ${dataName}`, + canceledMessage: `Export ${dataName} canceled`, + finishedMessage: `Export ${dataName} finished`, + afterFinish: () => { + if (!electron) { + window.open(`${resolveApi()}/uploads/get?file=${pureFileName}`, '_blank'); + } + }, }); - - function handleRunnerDone() { - closeSnackbar(snackId); - apiOff(`runner-done-${runid}`, handleRunnerDone); - if (isCanceled) showSnackbarError(`Export ${dataName} canceled`); - else showSnackbarInfo(`Export ${dataName} finished`); - - if (!electron) { - window.open(`${resolveApi()}/uploads/get?file=${pureFileName}`, '_blank'); - } - } - - apiOn(`runner-done-${runid}`, handleRunnerDone); } export async function exportQuickExportFile(dataName, reader, format, columnMap = null) {