import SQL dump POC

packages/api/src/shell/importDatabase.js

@@ -0,0 +1,50 @@
+const fs = require('fs');
+const byline = require('byline');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
+const download = require('./download');
+const stream = require('stream');
+
+class ImportStream extends stream.Transform {
+  constructor(pool, driver) {
+    super({ objectMode: true });
+    this.pool = pool;
+    this.driver = driver;
+  }
+  async _transform(chunk, encoding, cb) {
+    await this.driver.script(this.pool, chunk);
+    cb();
+  }
+}
+
+function awaitStreamEnd(stream) {
+  return new Promise((resolve, reject) => {
+    stream.once('end', () => {
+      resolve(true);
+    });
+    stream.once('error', err => {
+      reject(err);
+    });
+  });
+}
+
+async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
+  console.log(`Importing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+  console.log(`Connected.`);
+
+  const downloadedFile = await download(inputFile);
+
+  const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
+  const lineStream = byline(fileStream);
+  const splittedStream = splitQueryStream(lineStream, driver.getQuerySplitterOptions());
+  const importStream = new ImportStream(pool, driver);
+  // @ts-ignore
+  splittedStream.pipe(importStream);
+  await awaitStreamEnd(splittedStream);
+}
+
+module.exports = importDatabase;

packages/api/src/shell/index.js

@@ -22,6 +22,7 @@ const loadFile = require('./loadFile');
 const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 const dumpDatabase = require('./dumpDatabase');
+const importDatabase = require('./importDatabase');
 
 const dbgateApi = {
   queryReader,
@@ -47,6 +48,7 @@ const dbgateApi = {
   deployDb,
   initializeApiEnvironment,
   dumpDatabase,
+  importDatabase,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/tools/src/ScriptWriter.ts

@@ -53,6 +53,10 @@ export class ScriptWriter {
     this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`);
   }
 
+  importDatabase(options) {
+    this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`);
+  }
+
   comment(s) {
     this._put(`// ${s}`);
   }
@@ -132,6 +136,13 @@ export class ScriptWriterJson {
     });
   }
 
+  importDatabase(options) {
+    this.commands.push({
+      type: 'importDatabase',
+      options,
+    });
+  }
+
   getScript(schedule = null) {
     return {
       type: 'json',
@@ -172,6 +183,9 @@ export function jsonScriptToJavascript(json) {
       case 'dumpDatabase':
         script.dumpDatabase(cmd.options);
         break;
+      case 'importDatabase':
+        script.importDatabase(cmd.options);
+        break;
     }
   }
 

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -90,6 +90,12 @@
       exportSqlDump(connection, name);
     };
 
+    const handleSqlRestore = () => {
+      showModal(ImportDatabaseDumpModal, {
+        connection: { ...connection, database: name },
+      });
+    };
+
     const handleShowDiagram = async () => {
       const db = await getDatabaseInfo({
         conid: connection._id,
@@ -207,6 +213,7 @@
       isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import wizard' },
       isSqlOrDoc && { onClick: handleExport, text: 'Export wizard' },
       driver?.supportsDatabaseDump && { onClick: handleSqlDump, text: 'Backup/export SQL dump' },
+      driver?.supportsDatabaseDump && { onClick: handleSqlRestore, text: 'Restore/import SQL dump' },
       { divider: true },
       isSqlOrDoc && { onClick: handleShowDiagram, text: 'Show diagram' },
       isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' },
@@ -274,6 +281,7 @@
   import { filterAppsForDatabase } from '../utility/appTools';
   import newQuery from '../query/newQuery';
   import { exportSqlDump } from '../utility/exportFileTools';
+  import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
 
   export let data;
   export let passProps;

packages/web/src/modals/ImportDatabaseDumpModal.svelte

@@ -0,0 +1,72 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
+  import UploadButton from '../buttons/UploadButton.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormSubmit from '../forms/FormSubmit.svelte';
+  import FormTextField from '../forms/FormTextField.svelte';
+  import ElectronFilesInput from '../impexp/ElectronFilesInput.svelte';
+  import { importSqlDump } from '../utility/exportFileTools';
+  import getElectron from '../utility/getElectron';
+  import { setUploadListener } from '../utility/uploadFiles';
+  import ChangeDownloadUrlModal from './ChangeDownloadUrlModal.svelte';
+  import ModalBase from './ModalBase.svelte';
+  import { closeCurrentModal, showModal } from './modalTools';
+
+  export let connection;
+
+  let inputLabel = '(not selected)';
+  let inputFile = null;
+
+  const handleSubmit = async values => {
+    const { value } = values;
+    closeCurrentModal();
+    // onConfirm(value);
+    importSqlDump(inputFile, connection);
+  };
+
+  const electron = getElectron();
+
+  const handleUpload = file => {
+    inputLabel = `uploaded: ${file.shortName}`;
+    inputFile = file.filePath;
+  };
+
+  onMount(() => {
+    setUploadListener(handleUpload);
+
+    return () => {
+      setUploadListener(null);
+    };
+  });
+
+  const handleAddUrl = () =>
+    showModal(ChangeDownloadUrlModal, {
+      onConfirm: url => {
+        inputLabel = url;
+        inputFile = url;
+      },
+    });
+</script>
+
+<FormProvider>
+  <ModalBase {...$$restProps}>
+    <svelte:fragment slot="header">Import database dump</svelte:fragment>
+
+    <div class="m-3">Source: {inputLabel}</div>
+    {#if electron}
+      <ElectronFilesInput />
+    {:else}
+      <UploadButton />
+    {/if}
+
+    <FormStyledButton value="Add web URL" on:click={handleAddUrl} />
+
+    <svelte:fragment slot="footer">
+      <FormSubmit value="OK" on:click={e => handleSubmit(e.detail)} />
+      <FormStyledButton type="button" value="Cancel" on:click={closeCurrentModal} />
+    </svelte:fragment>
+  </ModalBase>
+</FormProvider>

packages/web/src/utility/exportFileTools.ts

@@ -6,6 +6,57 @@ import { apiCall, apiOff, apiOn } from './api';
 import { normalizeExportColumnMap } from '../impexp/createImpExpScript';
 import { getCurrentConfig } from '../stores';
 
+export async function importSqlDump(inputFile, connection) {
+  const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
+
+  script.importDatabase({
+    inputFile,
+    connection,
+  });
+
+  await runImportExportScript({
+    script: script.getScript(),
+    runningMessage: 'Importing database',
+    canceledMessage: 'Database import canceled',
+    finishedMessage: 'Database import finished',
+  });
+}
+
+async function runImportExportScript({ script, runningMessage, canceledMessage, finishedMessage, afterFinish = null }) {
+  const electron = getElectron();
+
+  const resp = await apiCall('runners/start', { script });
+  const runid = resp.runid;
+  let isCanceled = false;
+
+  const snackId = showSnackbar({
+    message: runningMessage,
+    icon: 'icon loading',
+    buttons: [
+      {
+        label: 'Cancel',
+        onClick: () => {
+          isCanceled = true;
+          apiCall('runners/cancel', { runid });
+        },
+      },
+    ],
+  });
+
+  function handleRunnerDone() {
+    closeSnackbar(snackId);
+    apiOff(`runner-done-${runid}`, handleRunnerDone);
+    if (isCanceled) {
+      showSnackbarError(canceledMessage);
+    } else {
+      showSnackbarInfo(finishedMessage);
+      if (afterFinish) afterFinish();
+    }
+  }
+
+  apiOn(`runner-done-${runid}`, handleRunnerDone);
+}
+
 export async function saveExportedFile(filters, defaultPath, extension, dataName, getScript: (filaPath: string) => {}) {
   const electron = getElectron();
 
@@ -27,36 +78,17 @@ export async function saveExportedFile(filters, defaultPath, extension, dataName
 
   const script = getScript(filePath);
 
-  const resp = await apiCall('runners/start', { script });
+  runImportExportScript({
-  const runid = resp.runid;
+    script,
-  let isCanceled = false;
+    runningMessage: `Exporting ${dataName}`,
-
+    canceledMessage: `Export ${dataName} canceled`,
-  const snackId = showSnackbar({
+    finishedMessage: `Export ${dataName} finished`,
-    message: `Exporting ${dataName}`,
+    afterFinish: () => {
-    icon: 'icon loading',
+      if (!electron) {
-    buttons: [
+        window.open(`${resolveApi()}/uploads/get?file=${pureFileName}`, '_blank');
-      {
+      }
-        label: 'Cancel',
+    },
-        onClick: () => {
-          isCanceled = true;
-          apiCall('runners/cancel', { runid });
-        },
-      },
-    ],
   });
-
-  function handleRunnerDone() {
-    closeSnackbar(snackId);
-    apiOff(`runner-done-${runid}`, handleRunnerDone);
-    if (isCanceled) showSnackbarError(`Export ${dataName} canceled`);
-    else showSnackbarInfo(`Export ${dataName} finished`);
-
-    if (!electron) {
-      window.open(`${resolveApi()}/uploads/get?file=${pureFileName}`, '_blank');
-    }
-  }
-
-  apiOn(`runner-done-${runid}`, handleRunnerDone);
 }
 
 export async function exportQuickExportFile(dataName, reader, format, columnMap = null) {