mysql dumper POC

packages/api/src/shell/dumpDatabase.js

@@ -0,0 +1,38 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+function doDump(dumper) {
+  return new Promise((resolve, reject) => {
+    dumper.once('end', () => {
+      resolve(true);
+    });
+    dumper.once('error', err => {
+      reject(err);
+    });
+    dumper.run();
+  });
+}
+
+async function dumpDatabase({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  outputFile,
+  databaseName,
+  schemaName,
+}) {
+  console.log(`Dumping database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  console.log(`Connected.`);
+
+  const dumper = await driver.createBackupDumper(pool, {
+    outputFile,
+    databaseName,
+    schemaName,
+  });
+  await doDump(dumper);
+}
+
+module.exports = dumpDatabase;

packages/api/src/shell/index.js

@@ -21,6 +21,7 @@ const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
 const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
+const dumpDatabase = require('./dumpDatabase');
 
 const dbgateApi = {
   queryReader,
@@ -45,6 +46,7 @@ const dbgateApi = {
   loadFile,
   deployDb,
   initializeApiEnvironment,
+  dumpDatabase,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/utility/connectUtility.js

@@ -39,7 +39,7 @@ async function loadConnection(driver, storedConnection, connectionMode) {
   return storedConnection;
 }
 
-async function connectUtility(driver, storedConnection, connectionMode) {
+async function connectUtility(driver, storedConnection, connectionMode, additionalOptions = null) {
   const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
 
   const connection = {
@@ -93,7 +93,7 @@ async function connectUtility(driver, storedConnection, connectionMode) {
     }
   }
 
-  const conn = await driver.connect(connection);
+  const conn = await driver.connect({ ...connection, ...additionalOptions });
   return conn;
 }
 

packages/tools/src/ScriptWriter.ts

@@ -49,6 +49,10 @@ export class ScriptWriter {
     }
   }
 
+  dumpDatabase(options) {
+    this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`);
+  }
+
   comment(s) {
     this._put(`// ${s}`);
   }
@@ -121,6 +125,13 @@ export class ScriptWriterJson {
     });
   }
 
+  dumpDatabase(options) {
+    this.commands.push({
+      type: 'dumpDatabase',
+      options,
+    });
+  }
+
   getScript(schedule = null) {
     return {
       type: 'json',
@@ -158,6 +169,9 @@ export function jsonScriptToJavascript(json) {
       case 'comment':
         script.comment(cmd.text);
         break;
+      case 'dumpDatabase':
+        script.dumpDatabase(cmd.options);
+        break;
     }
   }
 

packages/types/engines.d.ts

@@ -51,6 +51,10 @@ export interface SupportedDbKeyType {
   showItemList?: boolean;
 }
 
+export interface SqlBackupDumper {
+  run();
+}
+
 export interface EngineDriver {
   engine: string;
   title: string;
@@ -60,6 +64,7 @@ export interface EngineDriver {
   readOnlySessions: boolean;
   supportedKeyTypes: SupportedDbKeyType[];
   supportsDatabaseUrl?: boolean;
+  supportsDatabaseDump?: boolean;
   isElectronOnly?: boolean;
   supportedCreateDatabase?: boolean;
   showConnectionField?: (field: string, values: any) => boolean;
@@ -99,6 +104,7 @@ export interface EngineDriver {
   dialect: SqlDialect;
   dialectByVersion(version): SqlDialect;
   createDumper(options = null): SqlDumper;
+  createBackupDumper(pool: any, options): Promise<SqlBackupDumper>;
   getAuthTypes(): EngineAuthType[];
   readCollection(pool: any, options: ReadCollectionOptions): Promise<any>;
   updateCollection(pool: any, changeSet: any): Promise<any>;

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -86,6 +86,10 @@
       });
     };
 
+    const handleSqlDump = () => {
+      exportSqlDump(connection, name);
+    };
+
     const handleShowDiagram = async () => {
       const db = await getDatabaseInfo({
         conid: connection._id,
@@ -200,8 +204,10 @@
       driver?.databaseEngineTypes?.includes('sql') && { onClick: handleNewTable, text: 'New table' },
       driver?.databaseEngineTypes?.includes('document') && { onClick: handleNewCollection, text: 'New collection' },
       { divider: true },
-      isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import' },
-      isSqlOrDoc && { onClick: handleExport, text: 'Export' },
+      isSqlOrDoc && !connection.isReadOnly && { onClick: handleImport, text: 'Import wizard' },
+      isSqlOrDoc && { onClick: handleExport, text: 'Export wizard' },
+      driver?.supportsDatabaseDump && { onClick: handleSqlDump, text: 'Backup/export SQL dump' },
+      { divider: true },
       isSqlOrDoc && { onClick: handleShowDiagram, text: 'Show diagram' },
       isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' },
       isSqlOrDoc && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
@@ -267,6 +273,7 @@
   import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
   import { filterAppsForDatabase } from '../utility/appTools';
   import newQuery from '../query/newQuery';
+  import { exportSqlDump } from '../utility/exportFileTools';
 
   export let data;
   export let passProps;

packages/web/src/utility/exportFileTools.ts

@@ -6,46 +6,28 @@ import { apiCall, apiOff, apiOn } from './api';
 import { normalizeExportColumnMap } from '../impexp/createImpExpScript';
 import { getCurrentConfig } from '../stores';
 
-export async function exportQuickExportFile(dataName, reader, format, columnMap = null) {
+export async function saveExportedFile(filters, defaultPath, extension, dataName, getScript: (filaPath: string) => {}) {
   const electron = getElectron();
 
   let filePath;
   let pureFileName;
   if (electron) {
-    const filters = [{ name: format.label, extensions: [format.extension] }];
     filePath = await electron.showSaveDialog({
       filters,
-      defaultPath: `${dataName}.${format.extension}`,
+      defaultPath,
       properties: ['showOverwriteConfirmation'],
     });
   } else {
-    const resp = await apiCall('files/generate-uploads-file', { extension: format.extension });
+    const resp = await apiCall('files/generate-uploads-file', { extension });
     filePath = resp.filePath;
     pureFileName = resp.fileName;
   }
 
   if (!filePath) return;
 
-  const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
+  const script = getScript(filePath);
 
-  const sourceVar = script.allocVariable();
-  script.assign(sourceVar, reader.functionName, reader.props);
-
-  const targetVar = script.allocVariable();
-  const writer = format.createWriter(filePath, dataName);
-  script.assign(targetVar, writer.functionName, writer.props);
-
-  const colmap = normalizeExportColumnMap(columnMap);
-  let colmapVar = null;
-  if (colmap) {
-    colmapVar = script.allocVariable();
-    script.assignValue(colmapVar, colmap);
-  }
-
-  script.copyStream(sourceVar, targetVar, colmapVar);
-  script.endLine();
-
-  const resp = await apiCall('runners/start', { script: script.getScript() });
+  const resp = await apiCall('runners/start', { script });
   const runid = resp.runid;
   let isCanceled = false;
 
@@ -77,6 +59,57 @@ export async function exportQuickExportFile(dataName, reader, format, columnMap
   apiOn(`runner-done-${runid}`, handleRunnerDone);
 }
 
+export async function exportQuickExportFile(dataName, reader, format, columnMap = null) {
+  await saveExportedFile(
+    [{ name: format.label, extensions: [format.extension] }],
+    `${dataName}.${format.extension}`,
+    format.extension,
+    dataName,
+    filePath => {
+      const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
+
+      const sourceVar = script.allocVariable();
+      script.assign(sourceVar, reader.functionName, reader.props);
+
+      const targetVar = script.allocVariable();
+      const writer = format.createWriter(filePath, dataName);
+      script.assign(targetVar, writer.functionName, writer.props);
+
+      const colmap = normalizeExportColumnMap(columnMap);
+      let colmapVar = null;
+      if (colmap) {
+        colmapVar = script.allocVariable();
+        script.assignValue(colmapVar, colmap);
+      }
+
+      script.copyStream(sourceVar, targetVar, colmapVar);
+      script.endLine();
+
+      return script.getScript();
+    }
+  );
+}
+
+export async function exportSqlDump(connection, databaseName) {
+  await saveExportedFile(
+    [{ name: 'SQL files', extensions: ['sql'] }],
+    `${databaseName}.sql`,
+    'sql',
+    `${databaseName}-dump`,
+    filePath => {
+      const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
+
+      script.dumpDatabase({
+        connection,
+        databaseName,
+        outputFile: filePath,
+      });
+
+      return script.getScript();
+    }
+  );
+}
+
 export async function saveFileToDisk(
   filePathFunc,
   options: any = { formatLabel: 'HTML page', formatExtension: 'html' }

plugins/dbgate-plugin-mysql/package.json

@@ -31,11 +31,12 @@
     "prepublishOnly": "yarn build"
   },
   "devDependencies": {
+    "antares-mysql-dumper": "link:/Users/jena/jenasoft/antares-mysql-dumper",
     "dbgate-plugin-tools": "^1.0.7",
     "dbgate-query-splitter": "^4.8.3",
-    "webpack": "^4.42.0",
-    "webpack-cli": "^3.3.11",
     "dbgate-tools": "^4.1.1",
-    "mysql2": "^2.2.5"
+    "mysql2": "^2.2.5",
+    "webpack": "^4.42.0",
+    "webpack-cli": "^3.3.11"
   }
-}
+}

plugins/dbgate-plugin-mysql/src/backend/drivers.js

@@ -4,6 +4,7 @@ const driverBases = require('../frontend/drivers');
 const Analyser = require('./Analyser');
 const mysql2 = require('mysql2');
 const { createBulkInsertStreamBase, makeUniqueColumnNames } = require('dbgate-tools');
+const { MySqlDumper } = require('antares-mysql-dumper');
 
 function extractColumns(fields) {
   if (fields) {
@@ -28,7 +29,7 @@ const drivers = driverBases.map(driverBase => ({
   ...driverBase,
   analyserClass: Analyser,
 
-  async connect({ server, port, user, password, database, ssl, isReadOnly }) {
+  async connect({ server, port, user, password, database, ssl, isReadOnly, forceRowsAsObjects }) {
     const connection = mysql2.createConnection({
       host: server,
       port,
@@ -36,7 +37,7 @@ const drivers = driverBases.map(driverBase => ({
       password,
       database,
       ssl,
-      rowsAsArray: true,
+      rowsAsArray: forceRowsAsObjects ? false : true,
       supportBigNumbers: true,
       bigNumberStrings: true,
       dateStrings: true,
@@ -172,6 +173,15 @@ const drivers = driverBases.map(driverBase => ({
     // @ts-ignore
     return createBulkInsertStreamBase(this, stream, pool, name, options);
   },
+  async createBackupDumper(pool, options) {
+    const { outputFile, databaseName, schemaName } = options;
+    const res = new MySqlDumper({
+      connection: pool,
+      schema: databaseName || schemaName,
+      outputFile,
+    });
+    return res;
+  },
 }));
 
 module.exports = drivers;

plugins/dbgate-plugin-mysql/src/frontend/drivers.js

@@ -47,6 +47,7 @@ const mysqlDriverBase = {
   defaultPort: 3306,
   getQuerySplitterOptions: () => mysqlSplitterOptions,
   readOnlySessions: true,
+  supportsDatabaseDump: true,
 
   getNewObjectTemplates() {
     return [

yarn.lock

@@ -1024,6 +1024,11 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/svelte/-/svelte-1.0.10.tgz#30ec7feeee0bdf38b12a50f0686f8a2e7b6b9dc0"
   integrity sha512-EBrpH2iXXfaf/9z81koiDYkp2mlwW2XzFcAqn6qh7VKyP8zBvHHAQzNhY+W9vH5arAjmGAm5g8ElWq6YmXm3ig==
 
+"@turf/helpers@^6.5.0":
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e"
+  integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw==
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14":
   version "7.1.14"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402"
@@ -1668,6 +1673,10 @@ ansi-styles@^5.0.0:
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b"
   integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
 
+"antares-mysql-dumper@link:../antares-mysql-dumper":
+  version "0.0.0"
+  uid ""
+
 anymatch@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb"
@@ -7389,6 +7398,11 @@ moment@^2.24.0:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
   integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==
 
+moment@^2.29.2:
+  version "2.29.2"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4"
+  integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==
+
 mongodb-client-encryption@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/mongodb-client-encryption/-/mongodb-client-encryption-1.2.3.tgz#0078f2cf385762e052b0c12d9be256eb1ef9a347"