From 2063005d5cafc94cde0bd0ca262d579465b798ba Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 11 Mar 2021 20:37:05 +0100 Subject: [PATCH] import export --- packages/web/src/datagrid/DataGridCore.svelte | 19 ++ .../web/src/datagrid/SqlDataGridCore.svelte | 12 + packages/web/src/elements/LargeButton.svelte | 54 ++++ packages/web/src/forms/LargeFormButton.svelte | 17 ++ .../src/impexp/FormConnectionSelect.svelte | 17 ++ .../web/src/impexp/FormDatabaseSelect.svelte | 21 ++ .../impexp/ImportExportConfigurator.svelte | 41 +++ packages/web/src/impexp/ScriptWriter.ts | 50 ++++ .../web/src/impexp/SourceTargetConfig.svelte | 79 ++++++ packages/web/src/impexp/createImpExpScript.ts | 240 ++++++++++++++++++ .../web/src/modals/ImportExportModal.svelte | 135 ++++++++++ packages/web/src/modals/ModalBase.svelte | 44 +++- 12 files changed, 716 insertions(+), 13 deletions(-) create mode 100644 packages/web/src/elements/LargeButton.svelte create mode 100644 packages/web/src/forms/LargeFormButton.svelte create mode 100644 packages/web/src/impexp/FormConnectionSelect.svelte create mode 100644 packages/web/src/impexp/FormDatabaseSelect.svelte create mode 100644 packages/web/src/impexp/ImportExportConfigurator.svelte create mode 100644 packages/web/src/impexp/ScriptWriter.ts create mode 100644 packages/web/src/impexp/SourceTargetConfig.svelte create mode 100644 packages/web/src/impexp/createImpExpScript.ts create mode 100644 packages/web/src/modals/ImportExportModal.svelte diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index 876c73bd..52e95ead 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -109,6 +109,15 @@ onClick: () => get(currentDataGrid).copyToClipboard(), }); + registerCommand({ + id: 'dataGrid.export', + category: 'Data grid', + name: 'Export', + keyText: 'Ctrl+E', + enabledStore: derived(currentDataGrid, grid => grid != null && grid.exportEnabled()), + onClick: () => get(currentDataGrid).exportGrid(), + }); + function getRowCountInfo(selectedCells, grider, realColumnUniqueNames, selectedRowData, allRowCount) { if (selectedCells.length > 1 && selectedCells.every(x => _.isNumber(x[0]) && _.isNumber(x[1]))) { let sum = _.sumBy(selectedCells, cell => { @@ -180,6 +189,7 @@ export let onReferenceClick = undefined; export let onSave; export let focusOnVisible = false; + export let onExportGrid = null; export let isLoadedAll; export let loadedTime; @@ -231,6 +241,14 @@ return display; } + export function exportGrid() { + if (onExportGrid) onExportGrid(); + } + + export function exportEnabled() { + return !!onExportGrid; + } + export function revertRowChanges() { grider.beginUpdate(); for (const index of getSelectedRowIndexes()) { @@ -813,6 +831,7 @@ return [ { command: 'dataGrid.refresh' }, { command: 'dataGrid.copyToClipboard' }, + { command: 'dataGrid.export' }, { divider: true }, { command: 'dataGrid.save' }, { command: 'dataGrid.revertRowChanges' }, diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index 09aeb32a..e57d3554 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -48,6 +48,7 @@ import { scriptToSql } from 'dbgate-sqltree'; import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; + import ImportExportModal from '../modals/ImportExportModal.svelte'; import { showModal } from '../modals/modalTools'; import axiosInstance from '../utility/axiosInstance'; @@ -98,6 +99,16 @@ engine: display.engine, }); } + + function exportGrid() { + const initialValues: any = {}; + initialValues.sourceStorageType = 'query'; + initialValues.sourceConnectionId = conid; + initialValues.sourceDatabaseName = database; + initialValues.sourceSql = display.getExportQuery(); + initialValues.sourceList = display.baseTable ? [display.baseTable.pureName] : []; + showModal(ImportExportModal, { initialValues }); + } + import FontIcon from '../icons/FontIcon.svelte'; + import { createEventDispatcher } from 'svelte'; + + export let icon; + export let disabled = false; + + const dispatch = createEventDispatcher(); + + function handleClick() { + if (!disabled) dispatch('click'); + } + + +
+
+ +
+
+ +
+
+ + diff --git a/packages/web/src/forms/LargeFormButton.svelte b/packages/web/src/forms/LargeFormButton.svelte new file mode 100644 index 00000000..4369b510 --- /dev/null +++ b/packages/web/src/forms/LargeFormButton.svelte @@ -0,0 +1,17 @@ + + + + + diff --git a/packages/web/src/impexp/FormConnectionSelect.svelte b/packages/web/src/impexp/FormConnectionSelect.svelte new file mode 100644 index 00000000..36cb0201 --- /dev/null +++ b/packages/web/src/impexp/FormConnectionSelect.svelte @@ -0,0 +1,17 @@ + + +{#if connectionOptions.length == 0} +
Not available
+{:else} + +{/if} diff --git a/packages/web/src/impexp/FormDatabaseSelect.svelte b/packages/web/src/impexp/FormDatabaseSelect.svelte new file mode 100644 index 00000000..4d02f455 --- /dev/null +++ b/packages/web/src/impexp/FormDatabaseSelect.svelte @@ -0,0 +1,21 @@ + + +{#if databaseOptions.length == 0} +
Not available
+{:else} + +{/if} diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte new file mode 100644 index 00000000..fa9dbac8 --- /dev/null +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -0,0 +1,41 @@ + + +
+
+ +
+ +
+ +
+
+ + diff --git a/packages/web/src/impexp/ScriptWriter.ts b/packages/web/src/impexp/ScriptWriter.ts new file mode 100644 index 00000000..95c72957 --- /dev/null +++ b/packages/web/src/impexp/ScriptWriter.ts @@ -0,0 +1,50 @@ +import _ from 'lodash'; +import { extractShellApiFunctionName, extractShellApiPlugins } from 'dbgate-tools'; + +export default class ScriptWriter { + s = ''; + packageNames: string[] = []; + varCount = 0; + + constructor(varCount = '0') { + this.varCount = parseInt(varCount) || 0; + } + + allocVariable(prefix = 'var') { + this.varCount += 1; + return `${prefix}${this.varCount}`; + } + + put(s = '') { + this.s += s; + this.s += '\n'; + } + + assign(variableName, functionName, props) { + this.put(`const ${variableName} = await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});`); + this.packageNames.push(...extractShellApiPlugins(functionName, props)); + } + + requirePackage(packageName) { + this.packageNames.push(packageName); + } + + copyStream(sourceVar, targetVar) { + this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar});`); + } + + comment(s) { + this.put(`// ${s}`); + } + + getScript(schedule = null) { + const packageNames = this.packageNames; + let prefix = _.uniq(packageNames) + .map(packageName => `// @require ${packageName}\n`) + .join(''); + if (schedule) prefix += `// @schedule ${schedule}`; + if (prefix) prefix += '\n'; + + return prefix + this.s; + } +} diff --git a/packages/web/src/impexp/SourceTargetConfig.svelte b/packages/web/src/impexp/SourceTargetConfig.svelte new file mode 100644 index 00000000..fc4a4922 --- /dev/null +++ b/packages/web/src/impexp/SourceTargetConfig.svelte @@ -0,0 +1,79 @@ + + +
+ {#if direction == 'source'} +
+ Source configuration +
+ {/if} + {#if direction == 'target'} +
+ Target configuration +
+ {/if} + + x.directions.includes(direction))} + name={storageTypeField} + label="Storage type" + /> + + {#if storageType == 'database' || storageType == 'query'} + + + {/if} +
+ + diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts new file mode 100644 index 00000000..015fa737 --- /dev/null +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -0,0 +1,240 @@ +import _ from 'lodash'; +import ScriptWriter from './ScriptWriter'; +import getAsArray from '../utility/getAsArray'; +import { getConnectionInfo } from '../utility/metadataLoaders'; +import { findEngineDriver, findObjectLike } from 'dbgate-tools'; +import { findFileFormat } from '../plugins/fileformats'; + +export function getTargetName(extensions, source, values) { + const key = `targetName_${source}`; + if (values[key]) return values[key]; + const format = findFileFormat(extensions, values.targetStorageType); + if (format) { + const res = format.getDefaultOutputName ? format.getDefaultOutputName(source, values) : null; + if (res) return res; + return `${source}.${format.extension}`; + } + return source; +} + +function extractApiParameters(values, direction, format) { + const pairs = (format.args || []) + .filter(arg => arg.apiName) + .map(arg => [arg.apiName, values[`${direction}_${format.storageType}_${arg.name}`]]) + .filter(x => x[1] != null); + return _.fromPairs(pairs); +} + +async function getConnection(extensions, storageType, conid, database) { + if (storageType == 'database' || storageType == 'query') { + const conn = await getConnectionInfo({ conid }); + const driver = findEngineDriver(conn, extensions); + return [ + { + ..._.omit(conn, ['_id', 'displayName']), + database, + }, + driver, + ]; + } + return [null, null]; +} + +function getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver) { + const { sourceStorageType } = values; + if (sourceStorageType == 'database') { + const fullName = { schemaName: values.sourceSchemaName, pureName: sourceName }; + return [ + 'tableReader', + { + connection: sourceConnection, + ...fullName, + }, + ]; + } + if (sourceStorageType == 'query') { + return [ + 'queryReader', + { + connection: sourceConnection, + sql: values.sourceSql, + }, + ]; + } + if (findFileFormat(extensions, sourceStorageType)) { + const sourceFile = values[`sourceFile_${sourceName}`]; + const format = findFileFormat(extensions, sourceStorageType); + if (format && format.readerFunc) { + return [ + format.readerFunc, + { + ..._.omit(sourceFile, ['isDownload']), + ...extractApiParameters(values, 'source', format), + }, + ]; + } + } + if (sourceStorageType == 'jsldata') { + return ['jslDataReader', { jslid: values.sourceJslId }]; + } + if (sourceStorageType == 'archive') { + return [ + 'archiveReader', + { + folderName: values.sourceArchiveFolder, + fileName: sourceName, + }, + ]; + } + throw new Error(`Unknown source storage type: ${sourceStorageType}`); +} + +function getFlagsFroAction(action) { + switch (action) { + case 'dropCreateTable': + return { + createIfNotExists: true, + dropIfExists: true, + }; + case 'truncate': + return { + createIfNotExists: true, + truncate: true, + }; + } + + return { + createIfNotExists: true, + }; +} + +function getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver) { + const { targetStorageType } = values; + const format = findFileFormat(extensions, targetStorageType); + if (format && format.writerFunc) { + const outputParams = format.getOutputParams && format.getOutputParams(sourceName, values); + return [ + format.writerFunc, + { + ...(outputParams + ? outputParams + : { + fileName: getTargetName(extensions, sourceName, values), + }), + ...extractApiParameters(values, 'target', format), + }, + ]; + } + if (targetStorageType == 'database') { + return [ + 'tableWriter', + { + connection: targetConnection, + schemaName: values.targetSchemaName, + pureName: getTargetName(extensions, sourceName, values), + ...getFlagsFroAction(values[`actionType_${sourceName}`]), + }, + ]; + } + if (targetStorageType == 'archive') { + return [ + 'archiveWriter', + { + folderName: values.targetArchiveFolder, + fileName: getTargetName(extensions, sourceName, values), + }, + ]; + } + + throw new Error(`Unknown target storage type: ${targetStorageType}`); +} + +export default async function createImpExpScript(extensions, values, addEditorInfo = true) { + const script = new ScriptWriter(values.startVariableIndex || 0); + + const [sourceConnection, sourceDriver] = await getConnection( + extensions, + values.sourceStorageType, + values.sourceConnectionId, + values.sourceDatabaseName + ); + const [targetConnection, targetDriver] = await getConnection( + extensions, + values.targetStorageType, + values.targetConnectionId, + values.targetDatabaseName + ); + + const sourceList = getAsArray(values.sourceList); + for (const sourceName of sourceList) { + const sourceVar = script.allocVariable(); + // @ts-ignore + script.assign(sourceVar, ...getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver)); + + const targetVar = script.allocVariable(); + // @ts-ignore + script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver)); + + script.copyStream(sourceVar, targetVar); + script.put(); + } + if (addEditorInfo) { + script.comment('@ImportExportConfigurator'); + script.comment(JSON.stringify(values)); + } + return script.getScript(values.schedule); +} + +export function getActionOptions(extensions, source, values, targetDbinfo) { + const res = []; + const targetName = getTargetName(extensions, source, values); + if (values.targetStorageType == 'database') { + let existing = findObjectLike( + { schemaName: values.targetSchemaName, pureName: targetName }, + targetDbinfo, + 'tables' + ); + if (existing) { + res.push({ + label: 'Append data', + value: 'appendData', + }); + res.push({ + label: 'Truncate and import', + value: 'truncate', + }); + res.push({ + label: 'Drop and create table', + value: 'dropCreateTable', + }); + } else { + res.push({ + label: 'Create table', + value: 'createTable', + }); + } + } else { + res.push({ + label: 'Create file', + value: 'createFile', + }); + } + return res; +} + +export async function createPreviewReader(extensions, values, sourceName) { + const [sourceConnection, sourceDriver] = await getConnection( + extensions, + values.sourceStorageType, + values.sourceConnectionId, + values.sourceDatabaseName + ); + const [functionName, props] = getSourceExpr(extensions, sourceName, values, sourceConnection, sourceDriver); + return { + functionName, + props: { + ...props, + limitRows: 100, + }, + }; +} diff --git a/packages/web/src/modals/ImportExportModal.svelte b/packages/web/src/modals/ImportExportModal.svelte new file mode 100644 index 00000000..ea911ccd --- /dev/null +++ b/packages/web/src/modals/ImportExportModal.svelte @@ -0,0 +1,135 @@ + + + + + + Import/Export + {#if busy} + + {/if} + + +
+ + + + + + + + + + + + + + {#if previewReader} + + + + {/if} + + + + + + + +
+ + +
+
+ + diff --git a/packages/web/src/modals/ModalBase.svelte b/packages/web/src/modals/ModalBase.svelte index caac593e..ef2370af 100644 --- a/packages/web/src/modals/ModalBase.svelte +++ b/packages/web/src/modals/ModalBase.svelte @@ -8,6 +8,8 @@ export let fullScreen = false; export let noPadding = false; export let modalId; + export let skipBody = false; + export let skipFooter = false; function handleCloseModal() { closeModal(modalId); @@ -31,18 +33,26 @@
-
-
-
- + {#if $$slots.header} +
+
+
+ +
-
-
+ {/if} + {#if !skipBody} +
+ +
+ {:else} -
- + {/if} + {#if !skipFooter} + + {/if}
@@ -63,16 +73,24 @@ .window { background-color: var(--theme-bg-0); - margin: auto; - margin-top: 15vh; border: 1px solid var(--theme-border); - width: 50%; overflow: auto; outline: none; } .window:not(.fullScreen) { border-radius: 10px; + margin: auto; + margin-top: 15vh; + width: 50%; + } + + .window.fullScreen { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; } .close {