From e5135b1a9da75394e6a8a297edc95c47df4a3848 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 10:23:31 +0200 Subject: [PATCH 01/25] run tests on feature branch --- .github/workflows/run-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index ff0a610c..d05e07f2 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -4,6 +4,7 @@ on: branches: - master - develop + - 'feature\/.+' jobs: test-runner: From f8e39a2a5df5882fcc115cdf9fef3310c7291213 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 10:25:30 +0200 Subject: [PATCH 02/25] runtests on feature branch --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d05e07f2..78548007 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -4,7 +4,7 @@ on: branches: - master - develop - - 'feature\/.+' + - 'feature\**' jobs: test-runner: From ce431e6e219db9c04ea8a5da36b13b28d1abe993 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 10:25:52 +0200 Subject: [PATCH 03/25] fix --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 78548007..b8ceda79 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -4,7 +4,7 @@ on: branches: - master - develop - - 'feature\**' + - 'feature/**' jobs: test-runner: From a7846b4adf5b81c0ac562ca5374573636d435954 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 12:15:43 +0200 Subject: [PATCH 04/25] import export tab working --- .../src/appobj/DatabaseObjectAppObject.svelte | 48 ++++---- packages/web/src/tabs/ImportExportTab.svelte | 106 +++++++++++------- 2 files changed, 87 insertions(+), 67 deletions(-) diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 4dbd0d08..03595a08 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -764,31 +764,31 @@ }, { onClick: () => { - // openNewTab( - // { - // tabComponent: 'ImportExportTab', - // title: 'Import/Export', - // icon: 'img export', - // }, - // { - // editor: { - // sourceStorageType: 'database', - // sourceConnectionId: data.conid, - // sourceDatabaseName: data.database, - // sourceSchemaName: data.schemaName, - // sourceList: [data.pureName], - // }, - // } - // ); - showModal(ImportExportModal, { - initialValues: { - sourceStorageType: 'database', - sourceConnectionId: data.conid, - sourceDatabaseName: data.database, - sourceSchemaName: data.schemaName, - sourceList: [data.pureName], + openNewTab( + { + tabComponent: 'ImportExportTab', + title: 'Import/Export', + icon: 'img export', }, - }); + { + editor: { + sourceStorageType: 'database', + sourceConnectionId: data.conid, + sourceDatabaseName: data.database, + sourceSchemaName: data.schemaName, + sourceList: [data.pureName], + }, + } + ); + // showModal(ImportExportModal, { + // initialValues: { + // sourceStorageType: 'database', + // sourceConnectionId: data.conid, + // sourceDatabaseName: data.database, + // sourceSchemaName: data.schemaName, + // sourceList: [data.pureName], + // }, + // }); }, } ); diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 2e065d33..286b2714 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -24,6 +24,9 @@ import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte'; import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte'; import useEditorData from '../query/useEditorData'; + import ToolStripContainer from '../buttons/ToolStripContainer.svelte'; + import ToolStripButton from '../buttons/ToolStripButton.svelte'; + import FormProviderCore from '../forms/FormProviderCore.svelte'; let busy = false; let executeNumber = 0; @@ -39,10 +42,24 @@ const refreshArchiveFolderRef = createRef(null); + const formValues = writable({ + sourceStorageType: 'database', + targetStorageType: getDefaultFileFormat($extensions).storageType, + targetArchiveFolder: $currentArchive, + sourceArchiveFolder: $currentArchive, + ...detectCurrentTarget(), + ...initialValues, + }); + const { editorState, editorValue, setEditorData } = useEditorData({ tabid, + onInitialData: value => { + $formValues = value; + }, }); + $: setEditorData($formValues); + function detectCurrentTarget() { if (!importToCurrentTarget) return {}; @@ -94,7 +111,8 @@ }; const handleGenerateScript = async e => { - const code = await createImpExpScript($extensions, e.detail, undefined, true); + const values = $formValues; + const code = await createImpExpScript($extensions, values, undefined, true); openNewTab( { title: 'Shell #', @@ -107,7 +125,7 @@ const handleExecute = async e => { if (busy) return; - const values = e.detail; + const values = $formValues; busy = true; const script = await createImpExpScript($extensions, values); executeNumber += 1; @@ -130,49 +148,41 @@ }; - - -
- + + + +
+ - {#if busy} - - {/if} -
+ {#if busy} + + {/if} +
- - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - + + + {#if busy} + Stop + {:else} + Run + {/if} + Generate script + + From d25527336816160c9c2d14e97e330bf733b766d6 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 12:47:13 +0200 Subject: [PATCH 05/25] open import/export tab function --- .../src/appobj/ArchiveFileAppObject.svelte | 21 ++++++--- .../web/src/appobj/DatabaseAppObject.svelte | 44 ++++++++++++------- .../src/appobj/DatabaseObjectAppObject.svelte | 24 ++++------ packages/web/src/utility/exportFileTools.ts | 17 +++---- packages/web/src/utility/importExportTools.ts | 14 ++++++ 5 files changed, 74 insertions(+), 46 deletions(-) create mode 100644 packages/web/src/utility/importExportTools.ts diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 20457d54..71b84c8e 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -71,12 +71,13 @@ import { getExtensions } from '../stores'; import createQuickExportMenu from '../utility/createQuickExportMenu'; - import { exportQuickExportFile } from '../utility/exportFileTools'; + import { exportQuickExportFile, } from '../utility/exportFileTools'; import openNewTab from '../utility/openNewTab'; import AppObjectCore from './AppObjectCore.svelte'; import InputTextModal from '../modals/InputTextModal.svelte'; import ConfirmModal from '../modals/ConfirmModal.svelte'; import { apiCall } from '../utility/api'; + import { openImportExportTab } from '../utility/importExportTools'; export let data; @@ -156,13 +157,19 @@ { text: 'Export', onClick: () => { - showModal(ImportExportModal, { - initialValues: { - sourceStorageType: 'archive', - sourceArchiveFolder: data.folderName, - sourceList: [data.fileName], - }, + openImportExportTab({ + sourceStorageType: 'archive', + sourceArchiveFolder: data.folderName, + sourceList: [data.fileName], }); + + // showModal(ImportExportModal, { + // initialValues: { + // sourceStorageType: 'archive', + // sourceArchiveFolder: data.folderName, + // sourceList: [data.fileName], + // }, + // }); }, } ), diff --git a/packages/web/src/appobj/DatabaseAppObject.svelte b/packages/web/src/appobj/DatabaseAppObject.svelte index 4673664f..822df297 100644 --- a/packages/web/src/appobj/DatabaseAppObject.svelte +++ b/packages/web/src/appobj/DatabaseAppObject.svelte @@ -98,25 +98,39 @@ }; const handleImport = () => { - showModal(ImportExportModal, { - initialValues: { - sourceStorageType: getDefaultFileFormat($extensions).storageType, - targetStorageType: 'database', - targetConnectionId: connection._id, - targetDatabaseName: name, - }, + openImportExportTab({ + sourceStorageType: getDefaultFileFormat($extensions).storageType, + targetStorageType: 'database', + targetConnectionId: connection._id, + targetDatabaseName: name, }); + + // showModal(ImportExportModal, { + // initialValues: { + // sourceStorageType: getDefaultFileFormat($extensions).storageType, + // targetStorageType: 'database', + // targetConnectionId: connection._id, + // targetDatabaseName: name, + // }, + // }); }; const handleExport = () => { - showModal(ImportExportModal, { - initialValues: { - targetStorageType: getDefaultFileFormat($extensions).storageType, - sourceStorageType: 'database', - sourceConnectionId: connection._id, - sourceDatabaseName: name, - }, + openImportExportTab({ + targetStorageType: getDefaultFileFormat($extensions).storageType, + sourceStorageType: 'database', + sourceConnectionId: connection._id, + sourceDatabaseName: name, }); + + // showModal(ImportExportModal, { + // initialValues: { + // targetStorageType: getDefaultFileFormat($extensions).storageType, + // sourceStorageType: 'database', + // sourceConnectionId: connection._id, + // sourceDatabaseName: name, + // }, + // }); }; const handleSqlGenerator = () => { @@ -390,13 +404,13 @@ import ConfirmSqlModal, { runOperationOnDatabase, saveScriptToDatabase } from '../modals/ConfirmSqlModal.svelte'; import { filterAppsForDatabase } from '../utility/appTools'; import newQuery from '../query/newQuery'; - import { exportSqlDump } from '../utility/exportFileTools'; import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte'; import ExportDatabaseDumpModal from '../modals/ExportDatabaseDumpModal.svelte'; import ConfirmModal from '../modals/ConfirmModal.svelte'; import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte'; import NewCollectionModal from '../modals/NewCollectionModal.svelte'; import hasPermission from '../utility/hasPermission'; + import { openImportExportTab } from '../utility/importExportTools'; export let data; export let passProps; diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 03595a08..e0e451cc 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -764,22 +764,13 @@ }, { onClick: () => { - openNewTab( - { - tabComponent: 'ImportExportTab', - title: 'Import/Export', - icon: 'img export', - }, - { - editor: { - sourceStorageType: 'database', - sourceConnectionId: data.conid, - sourceDatabaseName: data.database, - sourceSchemaName: data.schemaName, - sourceList: [data.pureName], - }, - } - ); + openImportExportTab({ + sourceStorageType: 'database', + sourceConnectionId: data.conid, + sourceDatabaseName: data.database, + sourceSchemaName: data.schemaName, + sourceList: [data.pureName], + }); // showModal(ImportExportModal, { // initialValues: { // sourceStorageType: 'database', @@ -848,6 +839,7 @@ import { format as dateFormat } from 'date-fns'; import { getDefaultFileFormat } from '../plugins/fileformats'; import hasPermission from '../utility/hasPermission'; + import { openImportExportTab } from '../utility/importExportTools'; export let data; export let passProps; diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index 39ce5d67..4704aefe 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -42,8 +42,8 @@ export async function exportSqlDump(outputFile, connection, databaseName, pureFi onOpenResult: pureFileName && !getElectron() ? () => { - downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql'); - } + downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql'); + } : null, openResultLabel: 'Download SQL file', }); @@ -226,17 +226,18 @@ export async function downloadFromApi(route: string, donloadName: string) { method: 'GET', headers: resolveApiHeaders(), }) - .then((res) => res.blob()) - .then((blob) => { + .then(res => res.blob()) + .then(blob => { const objUrl = URL.createObjectURL(blob); - const a = document.createElement("a"); + const a = document.createElement('a'); document.body.appendChild(a); a.download = donloadName; a.href = objUrl; a.click(); a.remove(); setTimeout(() => { - URL.revokeObjectURL(objUrl) - }) - }) + URL.revokeObjectURL(objUrl); + }); + }); } + diff --git a/packages/web/src/utility/importExportTools.ts b/packages/web/src/utility/importExportTools.ts new file mode 100644 index 00000000..597fcd89 --- /dev/null +++ b/packages/web/src/utility/importExportTools.ts @@ -0,0 +1,14 @@ +import openNewTab from './openNewTab'; + +export function openImportExportTab(editorProps) { + openNewTab( + { + tabComponent: 'ImportExportTab', + title: 'Import/Export', + icon: 'img export', + }, + { + editor: editorProps, + } + ); +} From a9a5a3491eae41e8794fd8ac3ebd916bdc8c65c6 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Mon, 16 Sep 2024 13:03:49 +0200 Subject: [PATCH 06/25] showModal(ImportExportModal => openImportExportTab --- .../src/appobj/DatabaseObjectAppObject.svelte | 23 ++++++++----- packages/web/src/commands/stdCommands.ts | 19 +++++++---- .../datagrid/CollectionDataGridCore.svelte | 4 ++- .../web/src/datagrid/JslDataGridCore.svelte | 4 ++- .../web/src/datagrid/SqlDataGridCore.svelte | 4 ++- packages/web/src/tabs/ShellTab.svelte | 4 ++- packages/web/src/utility/importExportTools.ts | 3 +- packages/web/src/utility/openElectronFile.ts | 34 +++++++++++++------ packages/web/src/utility/uploadFiles.ts | 21 +++++++++--- 9 files changed, 82 insertions(+), 34 deletions(-) diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index e0e451cc..2e34f923 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -618,15 +618,22 @@ }); } else if (menu.isImport) { const { conid, database } = data; - showModal(ImportExportModal, { - initialValues: { - sourceStorageType: getDefaultFileFormat(getExtensions()).storageType, - targetStorageType: 'database', - targetConnectionId: conid, - targetDatabaseName: database, - fixedTargetPureName: data.pureName, - }, + openImportExportTab({ + sourceStorageType: getDefaultFileFormat(getExtensions()).storageType, + targetStorageType: 'database', + targetConnectionId: conid, + targetDatabaseName: database, + fixedTargetPureName: data.pureName, }); + // showModal(ImportExportModal, { + // initialValues: { + // sourceStorageType: getDefaultFileFormat(getExtensions()).storageType, + // targetStorageType: 'database', + // targetConnectionId: conid, + // targetDatabaseName: database, + // fixedTargetPureName: data.pureName, + // }, + // }); } else { openDatabaseObjectDetail( menu.tab, diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index 34f2f7be..eeb689a3 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -44,6 +44,7 @@ import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import NewCollectionModal from '../modals/NewCollectionModal.svelte'; import ConfirmModal from '../modals/ConfirmModal.svelte'; import localforage from 'localforage'; +import { openImportExportTab } from '../utility/importExportTools'; // function themeCommand(theme: ThemeDefinition) { // return { @@ -483,10 +484,18 @@ registerCommand({ toolbar: true, icon: 'icon import', onClick: () => - showModal(ImportExportModal, { - importToCurrentTarget: true, - initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType }, - }), + openImportExportTab( + { + sourceStorageType: getDefaultFileFormat(get(extensions)).storageType, + }, + { + importToCurrentTarget: true, + } + ), + // showModal(ImportExportModal, { + // importToCurrentTarget: true, + // initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType }, + // }), }); registerCommand({ @@ -595,8 +604,6 @@ registerCommand({ onClick: () => getElectron().send('check-for-updates'), }); - - export function registerFileCommands({ idPrefix, category, diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index b8fbf7e7..8151d67c 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -136,6 +136,7 @@ import LoadingDataGridCore from './LoadingDataGridCore.svelte'; import { mongoFilterBehaviour, standardFilterBehaviours } from 'dbgate-tools'; + import { openImportExportTab } from '../utility/importExportTools'; export let conid; export let display; @@ -207,7 +208,8 @@ initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native'; initialValues.sourceList = [pureName]; initialValues[`columns_${pureName}`] = display.getExportColumnMap(); - showModal(ImportExportModal, { initialValues }); + openImportExportTab(initialValues); + // showModal(ImportExportModal, { initialValues }); } export function openQuery() { diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index 18d0cfb3..83f81762 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -59,6 +59,7 @@ import LoadingDataGridCore from './LoadingDataGridCore.svelte'; import RowsArrayGrider from './RowsArrayGrider'; + import { openImportExportTab } from '../utility/importExportTools'; export let jslid; export let display; @@ -152,7 +153,8 @@ initialValues.sourceList = ['query-data']; initialValues[`columns_query-data`] = display.getExportColumnMap(); } - showModal(ImportExportModal, { initialValues }); + openImportExportTab(initialValues); + // showModal(ImportExportModal, { initialValues }); } const quickExportHandler = fmt => async () => { diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index df8157d8..64cbd9af 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -84,6 +84,7 @@ import LoadingDataGridCore from './LoadingDataGridCore.svelte'; import hasPermission from '../utility/hasPermission'; + import { openImportExportTab } from '../utility/importExportTools'; export let conid; export let display; @@ -145,7 +146,8 @@ initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native'; initialValues.sourceList = display.baseTableOrSimilar ? [display.baseTableOrSimilar.pureName] : []; initialValues[`columns_${pureName}`] = display.getExportColumnMap(); - showModal(ImportExportModal, { initialValues }); + openImportExportTab(initialValues); + // showModal(ImportExportModal, { initialValues }); } export function openQuery() { diff --git a/packages/web/src/tabs/ShellTab.svelte b/packages/web/src/tabs/ShellTab.svelte index 05a4eea7..1e865a47 100644 --- a/packages/web/src/tabs/ShellTab.svelte +++ b/packages/web/src/tabs/ShellTab.svelte @@ -60,6 +60,7 @@ import { showSnackbarError } from '../utility/snackbar'; import useEffect from '../utility/useEffect'; import useTimerLabel from '../utility/useTimerLabel'; + import { openImportExportTab } from '../utility/importExportTools'; export let tabid; @@ -156,7 +157,8 @@ export function openWizard() { const jsonTextMatch = ($editorValue || '').match(configRegex); if (jsonTextMatch) { - showModal(ImportExportModal, { initialValues: JSON.parse(jsonTextMatch[1]) }); + openImportExportTab(JSON.parse(jsonTextMatch[1])); + // showModal(ImportExportModal, { initialValues: JSON.parse(jsonTextMatch[1]) }); } else { showSnackbarError('No wizard info found'); } diff --git a/packages/web/src/utility/importExportTools.ts b/packages/web/src/utility/importExportTools.ts index 597fcd89..c7fa89da 100644 --- a/packages/web/src/utility/importExportTools.ts +++ b/packages/web/src/utility/importExportTools.ts @@ -1,11 +1,12 @@ import openNewTab from './openNewTab'; -export function openImportExportTab(editorProps) { +export function openImportExportTab(editorProps, additionalProps = {}) { openNewTab( { tabComponent: 'ImportExportTab', title: 'Import/Export', icon: 'img export', + ...additionalProps, }, { editor: editorProps, diff --git a/packages/web/src/utility/openElectronFile.ts b/packages/web/src/utility/openElectronFile.ts index a7f1e4fe..a07782e7 100644 --- a/packages/web/src/utility/openElectronFile.ts +++ b/packages/web/src/utility/openElectronFile.ts @@ -5,13 +5,14 @@ import ImportExportModal from '../modals/ImportExportModal.svelte'; import getElectron from './getElectron'; import { currentDatabase, extensions, getCurrentDatabase } from '../stores'; import { getUploadListener } from './uploadFiles'; -import {getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools'; +import { getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools'; import { apiCall } from './api'; import openNewTab from './openNewTab'; import { openJsonDocument } from '../tabs/JsonTab.svelte'; import { SAVED_FILE_HANDLERS } from '../appobj/SavedFileAppObject.svelte'; import _ from 'lodash'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; +import { openImportExportTab } from './importExportTools'; export function canOpenByElectron(file, extensions) { if (!file) return false; @@ -178,17 +179,30 @@ export function openElectronFileCore(filePath, extensions) { shortName: parsed.name, }); } else { - showModal(ImportExportModal, { - openedFile: { - filePath, - storageType: format.storageType, - shortName: parsed.name, - }, - importToCurrentTarget: true, - initialValues: { + openImportExportTab( + { sourceStorageType: format.storageType, }, - }); + { + openedFile: { + filePath, + storageType: format.storageType, + shortName: parsed.name, + }, + } + ); + + // showModal(ImportExportModal, { + // openedFile: { + // filePath, + // storageType: format.storageType, + // shortName: parsed.name, + // }, + // importToCurrentTarget: true, + // initialValues: { + // sourceStorageType: format.storageType, + // }, + // }); } } } diff --git a/packages/web/src/utility/uploadFiles.ts b/packages/web/src/utility/uploadFiles.ts index 88d6a8d8..7fe245df 100644 --- a/packages/web/src/utility/uploadFiles.ts +++ b/packages/web/src/utility/uploadFiles.ts @@ -8,6 +8,7 @@ import { showModal } from '../modals/modalTools'; import ImportExportModal from '../modals/ImportExportModal.svelte'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import openNewTab from './openNewTab'; +import { openImportExportTab } from './importExportTools'; let uploadListener; @@ -79,13 +80,23 @@ export default function uploadFiles(files) { uploadListener(fileData); } else { if (findFileFormat(ext, fileData.storageType)) { - showModal(ImportExportModal, { - uploadedFile: fileData, - importToCurrentTarget: true, - initialValues: { + openImportExportTab( + { sourceStorageType: fileData.storageType, }, - }); + { + uploadedFile: fileData, + importToCurrentTarget: true, + } + ); + + // showModal(ImportExportModal, { + // uploadedFile: fileData, + // importToCurrentTarget: true, + // initialValues: { + // sourceStorageType: fileData.storageType, + // }, + // }); } } From 88d7e07bead9fd1adb6f3892030849cdbffb1896 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 16 Sep 2024 17:16:54 +0200 Subject: [PATCH 07/25] fixed upload file --- .../impexp/ImportExportConfigurator.svelte | 46 +++++++++---------- packages/web/src/tabs/ImportExportTab.svelte | 44 +++++++++++++----- packages/web/src/utility/importExportTools.ts | 2 +- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte index daa58a33..2c161072 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.svelte +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -68,8 +68,8 @@ import SourceTargetConfig from './SourceTargetConfig.svelte'; - export let uploadedFile = undefined; - export let openedFile = undefined; + // export let uploadedFile = undefined; + // export let openedFile = undefined; export let previewReaderStore; const { values, setFieldValue } = getFormContext(); @@ -98,7 +98,7 @@ } }; - const handleUpload = file => { + export function addUploadedFile(file) { addFilesToSourceList( $extensions, [ @@ -116,26 +116,26 @@ }; onMount(() => { - setUploadListener(handleUpload); - if (uploadedFile) { - handleUpload(uploadedFile); - } - if (openedFile) { - handleUpload(openedFile); - // addFilesToSourceList( - // $extensions, - // [ - // { - // fileName: openedFile.filePath, - // shortName: openedFile.shortName, - // }, - // ], - // $values, - // values, - // !sourceList || sourceList.length == 0 ? openedFile.storageType : null, - // previewSource.set - // ); - } + setUploadListener(addUploadedFile); + // if (uploadedFile) { + // handleUpload(uploadedFile); + // } + // if (openedFile) { + // handleUpload(openedFile); + // // addFilesToSourceList( + // // $extensions, + // // [ + // // { + // // fileName: openedFile.filePath, + // // shortName: openedFile.shortName, + // // }, + // // ], + // // $values, + // // values, + // // !sourceList || sourceList.length == 0 ? openedFile.storageType : null, + // // previewSource.set + // // ); + // } return () => { setUploadListener(null); diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 286b2714..9bccc6be 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -42,22 +42,42 @@ const refreshArchiveFolderRef = createRef(null); - const formValues = writable({ - sourceStorageType: 'database', - targetStorageType: getDefaultFileFormat($extensions).storageType, - targetArchiveFolder: $currentArchive, - sourceArchiveFolder: $currentArchive, - ...detectCurrentTarget(), - ...initialValues, - }); + const formValues = writable({}); + + let domConfigurator; + + // const formValues = writable({ + // sourceStorageType: 'database', + // targetStorageType: getDefaultFileFormat($extensions).storageType, + // targetArchiveFolder: $currentArchive, + // sourceArchiveFolder: $currentArchive, + // ...detectCurrentTarget(), + // ...initialValues, + // }); const { editorState, editorValue, setEditorData } = useEditorData({ tabid, onInitialData: value => { - $formValues = value; + $formValues = { + sourceStorageType: 'database', + targetStorageType: getDefaultFileFormat($extensions).storageType, + targetArchiveFolder: $currentArchive, + sourceArchiveFolder: $currentArchive, + ...detectCurrentTarget(), + ...value, + }; + + if (uploadedFile) { + domConfigurator.addUploadedFile(uploadedFile); + } + if (openedFile) { + domConfigurator.addUploadedFile(openedFile); + } }, }); + // $: console.log('formValues', $formValues); + $: setEditorData($formValues); function detectCurrentTarget() { @@ -111,7 +131,7 @@ }; const handleGenerateScript = async e => { - const values = $formValues; + const values = $formValues as any; const code = await createImpExpScript($extensions, values, undefined, true); openNewTab( { @@ -125,7 +145,7 @@ const handleExecute = async e => { if (busy) return; - const values = $formValues; + const values = $formValues as any; busy = true; const script = await createImpExpScript($extensions, values); executeNumber += 1; @@ -152,7 +172,7 @@
- + {#if busy} diff --git a/packages/web/src/utility/importExportTools.ts b/packages/web/src/utility/importExportTools.ts index c7fa89da..bed2b20d 100644 --- a/packages/web/src/utility/importExportTools.ts +++ b/packages/web/src/utility/importExportTools.ts @@ -6,7 +6,7 @@ export function openImportExportTab(editorProps, additionalProps = {}) { tabComponent: 'ImportExportTab', title: 'Import/Export', icon: 'img export', - ...additionalProps, + props: additionalProps, }, { editor: editorProps, From 26c01f43f9d02461efab9cda18abdd520ac5977d Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 16 Sep 2024 17:28:30 +0200 Subject: [PATCH 08/25] drag & drop file to export/import tab --- .../impexp/ImportExportConfigurator.svelte | 42 +++++++------------ packages/web/src/tabs/ImportExportTab.svelte | 19 +++++++-- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte index 2c161072..a155a895 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.svelte +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -67,10 +67,12 @@ import SourceName from './SourceName.svelte'; import SourceTargetConfig from './SourceTargetConfig.svelte'; + import useEffect from '../utility/useEffect'; // export let uploadedFile = undefined; // export let openedFile = undefined; export let previewReaderStore; + export let isTabActive; const { values, setFieldValue } = getFormContext(); @@ -113,36 +115,20 @@ previewSource.set ); // setFieldValue('sourceList', [...(sourceList || []), file.originalName]); - }; + } - onMount(() => { - setUploadListener(addUploadedFile); - // if (uploadedFile) { - // handleUpload(uploadedFile); - // } - // if (openedFile) { - // handleUpload(openedFile); - // // addFilesToSourceList( - // // $extensions, - // // [ - // // { - // // fileName: openedFile.filePath, - // // shortName: openedFile.shortName, - // // }, - // // ], - // // $values, - // // values, - // // !sourceList || sourceList.length == 0 ? openedFile.storageType : null, - // // previewSource.set - // // ); - // } - - return () => { - setUploadListener(null); - }; + $: effectActiveTab = useEffect(() => { + if (isTabActive) { + setUploadListener(addUploadedFile); + return () => { + setUploadListener(null); + }; + } else { + return () => {}; + } }); - // engine={sourceEngine} - // {setPreviewSource} + + $effectActiveTab;
diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 9bccc6be..5c37f58c 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -16,7 +16,14 @@ import { getDefaultFileFormat } from '../plugins/fileformats'; import RunnerOutputFiles from '../query/RunnerOutputFiles.svelte'; import SocketMessageView from '../query/SocketMessageView.svelte'; - import { currentArchive, currentDatabase, extensions, visibleWidgetSideBar, selectedWidget } from '../stores'; + import { + currentArchive, + currentDatabase, + extensions, + visibleWidgetSideBar, + selectedWidget, + activeTabId, + } from '../stores'; import { apiCall, apiOff, apiOn } from '../utility/api'; import createRef from '../utility/createRef'; import openNewTab from '../utility/openNewTab'; @@ -104,7 +111,7 @@ } } - $: effect = useEffect(() => registerRunnerDone(runnerId)); + $: effectRunner = useEffect(() => registerRunnerDone(runnerId)); function registerRunnerDone(rid) { if (rid) { @@ -117,7 +124,7 @@ } } - $: $effect; + $: $effectRunner; const handleRunnerDone = () => { busy = false; @@ -172,7 +179,11 @@
- + {#if busy} From 319a7fd003d5121477def285f5f7e12554c1047e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 16 Sep 2024 18:50:14 +0200 Subject: [PATCH 09/25] csv import test (failing) --- .../__tests__/import-formats.spec.js | 43 +++++++++++++++++++ integration-tests/package.json | 3 +- integration-tests/setupTests.js | 8 ++++ integration-tests/tools.js | 5 --- package.json | 1 + yarn.lock | 5 +++ 6 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 integration-tests/__tests__/import-formats.spec.js diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js new file mode 100644 index 00000000..4631531d --- /dev/null +++ b/integration-tests/__tests__/import-formats.spec.js @@ -0,0 +1,43 @@ +const dbgateApi = require('dbgate-api/src/shell'); +// const jsonLinesWriter = require('dbgate-api/src/shell/jsonLinesWriter'); +const tmp = require('tmp'); +// const dbgatePluginCsv = require('dbgate-plugin-csv/src/backend'); +const fs = require('fs'); +const requirePlugin = require('dbgate-api/src/shell/requirePlugin'); + +const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Updated At; Reporter; Assignee +801; "Does it 'burst' the database on startup or first lUI load ? "; https://github.com/dbgate/dbgate/issues/801; ""; open; 05/23/2024; 05/23/2024; rgarrigue; +799; "BUG: latest AppImage crashes on opening in Fedora 39"; https://github.com/dbgate/dbgate/issues/799; ""; open; 05/21/2024; 05/24/2024; BenGraham-Git; +798; "MongoDB write operations fail"; https://github.com/dbgate/dbgate/issues/798; "bug,solved"; open; 05/21/2024; 05/24/2024; mahmed0715; +797; "BUG: Unable to open SQL files"; https://github.com/dbgate/dbgate/issues/797; "bug"; open; 05/20/2024; 05/21/2024; cesarValdivia; +795; "BUG: MS SQL Server connection error (KEY_USAGE_BIT_INCORRECT)"; https://github.com/dbgate/dbgate/issues/795; ""; open; 05/20/2024; 05/20/2024; keskinonur; +794; "GLIBC_2.29' not found and i have 2.31"; https://github.com/dbgate/dbgate/issues/794; ""; closed; 05/20/2024; 05/21/2024; MFdanGM; +793; "BUG: PostgresSQL doesn't show tables when connected"; https://github.com/dbgate/dbgate/issues/793; ""; open; 05/20/2024; 05/22/2024; stomper013; +792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo; +`; + +test('csv import test', async () => { + expect(1).toEqual(1); + const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv'); + + const csvFileName = tmp.tmpNameSync(); + const jsonlFileName = tmp.tmpNameSync(); + + fs.writeFileSync(csvFileName, CSV_DATA); + + const reader = await dbgatePluginCsv.shellApi.reader({ + fileName: csvFileName, + }); + + const writer = await dbgateApi.jsonLinesWriter({ + fileName: jsonlFileName, + }); + await dbgateApi.copyStream(reader, writer); + + const jsonData = fs.readFileSync(jsonlFileName, 'utf-8'); + const rows = jsonData + .split('\n') + .filter(x => x.trim() !== '') + .map(x => JSON.parse(x)); + expect(rows.length).toEqual(9); +}); diff --git a/integration-tests/package.json b/integration-tests/package.json index dcb1e67e..6a0dea08 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -22,6 +22,7 @@ "devDependencies": { "cross-env": "^7.0.3", "jest": "^27.0.1", - "pino-pretty": "^11.2.2" + "pino-pretty": "^11.2.2", + "tmp": "^0.2.3" } } diff --git a/integration-tests/setupTests.js b/integration-tests/setupTests.js index b52c4066..e0f76378 100644 --- a/integration-tests/setupTests.js +++ b/integration-tests/setupTests.js @@ -1,4 +1,10 @@ +global.DBGATE_PACKAGES = { + 'dbgate-tools': require('dbgate-tools'), + 'dbgate-sqltree': require('dbgate-sqltree'), +}; + const { prettyFactory } = require('pino-pretty'); +const tmp = require('tmp'); const pretty = prettyFactory({ colorize: true, @@ -20,3 +26,5 @@ global.console = { process.stdout.write(messages.join(' ') + '\n'); }, }; + +tmp.setGracefulCleanup(); diff --git a/integration-tests/tools.js b/integration-tests/tools.js index 846c2765..4ab8c20d 100644 --- a/integration-tests/tools.js +++ b/integration-tests/tools.js @@ -1,8 +1,3 @@ -global.DBGATE_PACKAGES = { - 'dbgate-tools': require('dbgate-tools'), - 'dbgate-sqltree': require('dbgate-sqltree'), -}; - const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver'); const crypto = require('crypto'); diff --git a/package.json b/package.json index ec6297e8..ac6acbc0 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build:api": "yarn workspace dbgate-api build", "build:web:docker": "yarn workspace dbgate-web build", "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend", + "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend", "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch", "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs", "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist", diff --git a/yarn.lock b/yarn.lock index 97bc9f2e..acc569e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10182,6 +10182,11 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" +tmp@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae" + integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w== + tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" From 4065e0501376131de53038010999b6723a59739c Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 09:59:47 +0200 Subject: [PATCH 10/25] CSV import fixed --- .../__tests__/import-formats.spec.js | 23 ++++++++ plugins/dbgate-plugin-csv/package.json | 2 +- .../dbgate-plugin-csv/src/backend/reader.js | 1 + yarn.lock | 55 ++++++++----------- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index 4631531d..9dfdbd54 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -27,6 +27,7 @@ test('csv import test', async () => { const reader = await dbgatePluginCsv.shellApi.reader({ fileName: csvFileName, + delimiter: ';', }); const writer = await dbgateApi.jsonLinesWriter({ @@ -39,5 +40,27 @@ test('csv import test', async () => { .split('\n') .filter(x => x.trim() !== '') .map(x => JSON.parse(x)); + expect(rows[0].columns).toEqual([ + { columnName: 'Issue Number' }, + { columnName: 'Title' }, + { columnName: 'Github URL' }, + { columnName: 'Labels' }, + { columnName: 'State' }, + { columnName: 'Created At' }, + { columnName: 'Updated At' }, + { columnName: 'Reporter' }, + { columnName: 'Assignee' }, + ]); expect(rows.length).toEqual(9); + expect(rows[1]).toEqual({ + 'Issue Number': '801', + Title: "Does it 'burst' the database on startup or first lUI load ? ", + 'Github URL': 'https://github.com/dbgate/dbgate/issues/801', + Labels: '', + State: 'open', + 'Created At': '05/23/2024', + 'Updated At': '05/23/2024', + Reporter: 'rgarrigue', + Assignee: '', + }); }); diff --git a/plugins/dbgate-plugin-csv/package.json b/plugins/dbgate-plugin-csv/package.json index 08fefed6..0519f5be 100644 --- a/plugins/dbgate-plugin-csv/package.json +++ b/plugins/dbgate-plugin-csv/package.json @@ -32,7 +32,7 @@ "prepublishOnly": "yarn build" }, "devDependencies": { - "csv": "^5.3.2", + "csv": "^6.3.10", "dbgate-plugin-tools": "^1.0.7", "lodash": "^4.17.21", "webpack": "^5.91.0", diff --git a/plugins/dbgate-plugin-csv/src/backend/reader.js b/plugins/dbgate-plugin-csv/src/backend/reader.js index 6c07518f..0429b8a6 100644 --- a/plugins/dbgate-plugin-csv/src/backend/reader.js +++ b/plugins/dbgate-plugin-csv/src/backend/reader.js @@ -51,6 +51,7 @@ async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, delimiter, skip_lines_with_error: true, to_line: limitRows ? limitRows + 1 : undefined, + ltrim: true, }); const downloadedFile = await dbgateApi.download(fileName); const fileStream = fs.createReadStream(downloadedFile, encoding); diff --git a/yarn.lock b/yarn.lock index acc569e5..82413127 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3116,30 +3116,30 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csv-generate@^3.4.3: - version "3.4.3" - resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.4.3.tgz#bc42d943b45aea52afa896874291da4b9108ffff" - integrity sha512-w/T+rqR0vwvHqWs/1ZyMDWtHHSJaN06klRqJXBEpDJaM/+dZkso0OKh1VcuuYvK3XM53KysVNq8Ko/epCK8wOw== +csv-generate@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-4.4.1.tgz#729781ace8d1b92f6bfb407d1ab9548728c55681" + integrity sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg== -csv-parse@^4.16.3: - version "4.16.3" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.16.3.tgz#7ca624d517212ebc520a36873c3478fa66efbaf7" - integrity sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg== +csv-parse@^5.5.6: + version "5.5.6" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a" + integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A== -csv-stringify@^5.6.5: - version "5.6.5" - resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.6.5.tgz#c6d74badda4b49a79bf4e72f91cce1e33b94de00" - integrity sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A== +csv-stringify@^6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.1.tgz#a31837dd35e34787e3c248159c982a21af964d94" + integrity sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ== -csv@^5.3.2: - version "5.5.3" - resolved "https://registry.yarnpkg.com/csv/-/csv-5.5.3.tgz#cd26c1e45eae00ce6a9b7b27dcb94955ec95207d" - integrity sha512-QTaY0XjjhTQOdguARF0lGKm5/mEq9PD9/VhZZegHDIBq2tQwgNpHc3dneD4mGo2iJs+fTKv5Bp0fZ+BRuY3Z0g== +csv@^6.3.10: + version "6.3.10" + resolved "https://registry.yarnpkg.com/csv/-/csv-6.3.10.tgz#960a3a9cef08573ecca2d80ddb71152aca383088" + integrity sha512-5NYZG4AN2ZUthmNxIudgBEdMPUnbQHu9V4QTzBPqQzUP3KQsFiJo+8HQ0+oVxj1PomIT1/f67VI1QH/hsrZLKA== dependencies: - csv-generate "^3.4.3" - csv-parse "^4.16.3" - csv-stringify "^5.6.5" - stream-transform "^2.1.3" + csv-generate "^4.4.1" + csv-parse "^5.5.6" + csv-stringify "^6.5.1" + stream-transform "^3.3.2" dashdash@^1.12.0: version "1.14.1" @@ -7456,11 +7456,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mixme@^0.5.1: - version "0.5.10" - resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.5.10.tgz#d653b2984b75d9018828f1ea333e51717ead5f51" - integrity sha512-5H76ANWinB1H3twpJ6JY8uvAtpmFvHNArpilJAjXRKXSDDLPIMoZArw5SH0q9z+lLs8IrMw7Q2VWpWimFKFT1Q== - mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -9750,12 +9745,10 @@ stoppable@^1.1.0: resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== -stream-transform@^2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.1.3.tgz#a1c3ecd72ddbf500aa8d342b0b9df38f5aa598e3" - integrity sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ== - dependencies: - mixme "^0.5.1" +stream-transform@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-3.3.2.tgz#398c67b2f3b6ed5d04ceadde9e412bda8416c8ab" + integrity sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ== streamsearch@^1.1.0: version "1.1.0" From 0c2b25f79a9497d16a6b1ec784fbce1cf9b76575 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 10:28:58 +0200 Subject: [PATCH 11/25] auto-detect CSV delimiter --- packages/api/src/controllers/jsldata.js | 9 ++-- plugins/dbgate-plugin-csv/package.json | 3 +- .../dbgate-plugin-csv/src/backend/reader.js | 41 ++++++++++++++++++- .../dbgate-plugin-csv/src/frontend/index.js | 1 + 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 0cf04a97..448ba5d3 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -18,11 +18,14 @@ function readFirstLine(file) { } if (reader.hasNextLine()) { reader.nextLine((err, line) => { - if (err) reject(err); - resolve(line); + if (err) { + reader.close(() => reject(err)); // Ensure reader is closed on error + return; + } + reader.close(() => resolve(line)); // Ensure reader is closed after reading }); } else { - resolve(null); + reader.close(() => resolve(null)); // Properly close if no lines are present } }); }); diff --git a/plugins/dbgate-plugin-csv/package.json b/plugins/dbgate-plugin-csv/package.json index 0519f5be..72bc027a 100644 --- a/plugins/dbgate-plugin-csv/package.json +++ b/plugins/dbgate-plugin-csv/package.json @@ -34,8 +34,9 @@ "devDependencies": { "csv": "^6.3.10", "dbgate-plugin-tools": "^1.0.7", + "line-reader": "^0.4.0", "lodash": "^4.17.21", "webpack": "^5.91.0", "webpack-cli": "^5.1.4" } -} \ No newline at end of file +} diff --git a/plugins/dbgate-plugin-csv/src/backend/reader.js b/plugins/dbgate-plugin-csv/src/backend/reader.js index 0429b8a6..53d0ad33 100644 --- a/plugins/dbgate-plugin-csv/src/backend/reader.js +++ b/plugins/dbgate-plugin-csv/src/backend/reader.js @@ -2,8 +2,32 @@ const zipObject = require('lodash/zipObject'); const csv = require('csv'); const fs = require('fs'); const stream = require('stream'); +const lineReader = require('line-reader'); let dbgateApi; + +function readFirstLine(file) { + return new Promise((resolve, reject) => { + lineReader.open(file, (err, reader) => { + if (err) { + reject(err); + return; + } + if (reader.hasNextLine()) { + reader.nextLine((err, line) => { + if (err) { + reader.close(() => reject(err)); // Ensure reader is closed on error + return; + } + reader.close(() => resolve(line)); // Ensure reader is closed after reading + }); + } else { + reader.close(() => resolve(null)); // Properly close if no lines are present + } + }); + }); +} + class CsvPrepareStream extends stream.Transform { constructor({ header }) { super({ objectMode: true }); @@ -46,6 +70,22 @@ class CsvPrepareStream extends stream.Transform { async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, limitRows = undefined }) { console.log(`Reading file ${fileName}`); + const downloadedFile = await dbgateApi.download(fileName); + + if (!delimiter) { + // auto detect delimiter + // read first line from downloadedFile + const firstLine = await readFirstLine(downloadedFile); + if (firstLine) { + const delimiterCounts = { + ',': firstLine.replace(/[^,]/g, '').length, + ';': firstLine.replace(/[^;]/g, '').length, + '|': firstLine.replace(/[^|]/g, '').length, + }; + + delimiter = Object.keys(delimiterCounts).reduce((a, b) => (delimiterCounts[a] > delimiterCounts[b] ? a : b), ','); + } + } const csvStream = csv.parse({ // @ts-ignore delimiter, @@ -53,7 +93,6 @@ async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, to_line: limitRows ? limitRows + 1 : undefined, ltrim: true, }); - const downloadedFile = await dbgateApi.download(fileName); const fileStream = fs.createReadStream(downloadedFile, encoding); const csvPrepare = new CsvPrepareStream({ header }); fileStream.pipe(csvStream); diff --git a/plugins/dbgate-plugin-csv/src/frontend/index.js b/plugins/dbgate-plugin-csv/src/frontend/index.js index 54ae0411..ef43d8de 100644 --- a/plugins/dbgate-plugin-csv/src/frontend/index.js +++ b/plugins/dbgate-plugin-csv/src/frontend/index.js @@ -17,6 +17,7 @@ const fileFormat = { name: 'delimiter', label: 'Delimiter', options: [ + { name: 'Auto-detect', value: '' }, { name: 'Comma (,)', value: ',' }, { name: 'Semicolon (;)', value: ';' }, { name: 'Tab', value: '\t' }, From 77d60ccfa56c9808b2960634b769f370e2a12e49 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 10:29:49 +0200 Subject: [PATCH 12/25] auto-detect CSV delimiter in test --- integration-tests/__tests__/import-formats.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index 9dfdbd54..ea55ebd4 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -27,7 +27,6 @@ test('csv import test', async () => { const reader = await dbgatePluginCsv.shellApi.reader({ fileName: csvFileName, - delimiter: ';', }); const writer = await dbgateApi.jsonLinesWriter({ From 74fceeec7837f4b9341c115f233d0560bd7c666a Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 10:45:41 +0200 Subject: [PATCH 13/25] fix --- packages/web/src/impexp/SourceTargetConfig.svelte | 1 + packages/web/src/tabs/ImportExportTab.svelte | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/web/src/impexp/SourceTargetConfig.svelte b/packages/web/src/impexp/SourceTargetConfig.svelte index 241c02ec..d25e5512 100644 --- a/packages/web/src/impexp/SourceTargetConfig.svelte +++ b/packages/web/src/impexp/SourceTargetConfig.svelte @@ -196,6 +196,7 @@ width: 20vw; margin-left: var(--dim-large-form-margin); margin-bottom: var(--dim-large-form-margin); + border: 1px solid var(--theme-border); } .label { diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 5c37f58c..69e219d5 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -34,6 +34,8 @@ import ToolStripContainer from '../buttons/ToolStripContainer.svelte'; import ToolStripButton from '../buttons/ToolStripButton.svelte'; import FormProviderCore from '../forms/FormProviderCore.svelte'; + import { changeTab } from '../utility/common'; + import _ from 'lodash'; let busy = false; let executeNumber = 0; @@ -42,7 +44,6 @@ const previewReaderStore = writable(null); export let tabid; - export let initialValues; export let uploadedFile = undefined; export let openedFile = undefined; export let importToCurrentTarget = false; @@ -80,6 +81,11 @@ if (openedFile) { domConfigurator.addUploadedFile(openedFile); } + + changeTab(tabid, tab => ({ + ...tab, + props: _.omit(tab.props, ['uploadedFile', 'openedFile', 'importToCurrentTarget']), + })); }, }); From 665ce227412055f8e56a84248a8a26dd0222d169 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 12:16:59 +0200 Subject: [PATCH 14/25] JSON import --- packages/api/package.json | 1 + packages/api/src/shell/jsonReader.js | 58 +++++++++++++++++++++++++ packages/web/src/plugins/fileformats.ts | 1 + yarn.lock | 12 +++++ 4 files changed, 72 insertions(+) create mode 100644 packages/api/src/shell/jsonReader.js diff --git a/packages/api/package.json b/packages/api/package.json index 4812aefd..c2813e3e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -57,6 +57,7 @@ "rimraf": "^3.0.0", "simple-encryptor": "^4.0.0", "ssh2": "^1.11.0", + "stream-json": "^1.8.0", "tar": "^6.0.5" }, "scripts": { diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js new file mode 100644 index 00000000..e2918773 --- /dev/null +++ b/packages/api/src/shell/jsonReader.js @@ -0,0 +1,58 @@ +const fs = require('fs'); +const stream = require('stream'); +const byline = require('byline'); +const { getLogger } = require('dbgate-tools'); +const logger = getLogger('jsonReader'); +const { parser } = require('stream-json'); +const { pick } = require('stream-json/filters/Pick'); +const { streamArray } = require('stream-json/streamers/StreamArray'); + +class ParseStream extends stream.Transform { + constructor({ limitRows }) { + super({ objectMode: true }); + this.wasHeader = false; + this.limitRows = limitRows; + this.rowsWritten = 0; + } + _transform(chunk, encoding, done) { + const obj = JSON.parse(chunk); + if (!this.wasHeader) { + this.push({ + __isStreamHeader: true, + __isDynamicStructure: true, + }); + + this.wasHeader = true; + } + if (!this.limitRows || this.rowsWritten < this.limitRows) { + this.push(obj.value); + this.rowsWritten += 1; + } + done(); + } +} + +async function jsonReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { + logger.info(`Reading file ${fileName}`); + + const fileStream = fs.createReadStream( + fileName, + // @ts-ignore + encoding + ); + const parseJsonStream = parser(); + fileStream.pipe(parseJsonStream); + + const { streamArray } = require('stream-json/streamers/StreamArray'); + const streamArrayStream = streamArray(); + + parseJsonStream.pipe(streamArrayStream); + + const parseStream = new ParseStream({ limitRows }); + + streamArrayStream.pipe(parseStream); + + return parseStream; +} + +module.exports = jsonReader; diff --git a/packages/web/src/plugins/fileformats.ts b/packages/web/src/plugins/fileformats.ts index a65aef9d..700bba3b 100644 --- a/packages/web/src/plugins/fileformats.ts +++ b/packages/web/src/plugins/fileformats.ts @@ -13,6 +13,7 @@ const jsonFormat = { storageType: 'json', extension: 'json', name: 'JSON', + readerFunc: 'jsonReader', writerFunc: 'jsonArrayWriter', }; diff --git a/yarn.lock b/yarn.lock index 82413127..66c1a205 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9745,6 +9745,18 @@ stoppable@^1.1.0: resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== +stream-chain@^2.2.5: + version "2.2.5" + resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09" + integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA== + +stream-json@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c" + integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw== + dependencies: + stream-chain "^2.2.5" + stream-transform@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-3.3.2.tgz#398c67b2f3b6ed5d04ceadde9e412bda8416c8ab" From 880912806ae07ca4e08935e3241a00037305e774 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 12:32:03 +0200 Subject: [PATCH 15/25] JSON import --- .../__tests__/import-formats.spec.js | 34 ++++++++++++++++++- packages/api/src/shell/index.js | 2 ++ packages/api/src/shell/jsonReader.js | 3 +- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index ea55ebd4..5cd7871c 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -17,7 +17,6 @@ const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Up `; test('csv import test', async () => { - expect(1).toEqual(1); const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv'); const csvFileName = tmp.tmpNameSync(); @@ -63,3 +62,36 @@ test('csv import test', async () => { Assignee: '', }); }); + +test('JSON array import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + const jsonLinesFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify([ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + ]) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + }); + + const writer = await dbgateApi.jsonLinesWriter({ + fileName: jsonLinesFileName, + }); + await dbgateApi.copyStream(reader, writer); + + const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8'); + const rows = jsonData + .split('\n') + .filter(x => x.trim() !== '') + .map(x => JSON.parse(x)); + expect(rows.length).toEqual(2); + expect(rows[0]).toEqual({ + id: 1, + val: 'v1', + }); +}); diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 253bb053..82513088 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -29,6 +29,7 @@ const modifyJsonLinesReader = require('./modifyJsonLinesReader'); const dataDuplicator = require('./dataDuplicator'); const dbModelToJson = require('./dbModelToJson'); const jsonToDbModel = require('./jsonToDbModel'); +const jsonReader = require('./jsonReader'); const dbgateApi = { queryReader, @@ -61,6 +62,7 @@ const dbgateApi = { dataDuplicator, dbModelToJson, jsonToDbModel, + jsonReader, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js index e2918773..b54b9471 100644 --- a/packages/api/src/shell/jsonReader.js +++ b/packages/api/src/shell/jsonReader.js @@ -15,7 +15,6 @@ class ParseStream extends stream.Transform { this.rowsWritten = 0; } _transform(chunk, encoding, done) { - const obj = JSON.parse(chunk); if (!this.wasHeader) { this.push({ __isStreamHeader: true, @@ -25,7 +24,7 @@ class ParseStream extends stream.Transform { this.wasHeader = true; } if (!this.limitRows || this.rowsWritten < this.limitRows) { - this.push(obj.value); + this.push(chunk.value); this.rowsWritten += 1; } done(); From 503e09ddd18cd64184fef005c22561490b366292 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 12:40:41 +0200 Subject: [PATCH 16/25] import test small refactor --- .../__tests__/import-formats.spec.js | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index 5cd7871c..d78b824b 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -16,11 +16,27 @@ const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Up 792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo; `; +async function getReaderRows(reader) { + const jsonLinesFileName = tmp.tmpNameSync(); + + const writer = await dbgateApi.jsonLinesWriter({ + fileName: jsonLinesFileName, + }); + await dbgateApi.copyStream(reader, writer); + + const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8'); + const rows = jsonData + .split('\n') + .filter(x => x.trim() !== '') + .map(x => JSON.parse(x)); + + return rows; +} + test('csv import test', async () => { const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv'); const csvFileName = tmp.tmpNameSync(); - const jsonlFileName = tmp.tmpNameSync(); fs.writeFileSync(csvFileName, CSV_DATA); @@ -28,16 +44,8 @@ test('csv import test', async () => { fileName: csvFileName, }); - const writer = await dbgateApi.jsonLinesWriter({ - fileName: jsonlFileName, - }); - await dbgateApi.copyStream(reader, writer); + const rows = await getReaderRows(reader); - const jsonData = fs.readFileSync(jsonlFileName, 'utf-8'); - const rows = jsonData - .split('\n') - .filter(x => x.trim() !== '') - .map(x => JSON.parse(x)); expect(rows[0].columns).toEqual([ { columnName: 'Issue Number' }, { columnName: 'Title' }, @@ -65,7 +73,6 @@ test('csv import test', async () => { test('JSON array import test', async () => { const jsonFileName = tmp.tmpNameSync(); - const jsonLinesFileName = tmp.tmpNameSync(); fs.writeFileSync( jsonFileName, @@ -79,16 +86,8 @@ test('JSON array import test', async () => { fileName: jsonFileName, }); - const writer = await dbgateApi.jsonLinesWriter({ - fileName: jsonLinesFileName, - }); - await dbgateApi.copyStream(reader, writer); + const rows = await getReaderRows(reader); - const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8'); - const rows = jsonData - .split('\n') - .filter(x => x.trim() !== '') - .map(x => JSON.parse(x)); expect(rows.length).toEqual(2); expect(rows[0]).toEqual({ id: 1, From fd8a28831e53d1aac1b5c9467aa5472c78fd679f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 12:52:53 +0200 Subject: [PATCH 17/25] JSON object import --- .../__tests__/import-formats.spec.js | 34 ++++++++++++++++--- packages/api/src/shell/jsonReader.js | 34 +++++++++++++------ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index d78b824b..7bfcc905 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -89,8 +89,34 @@ test('JSON array import test', async () => { const rows = await getReaderRows(reader); expect(rows.length).toEqual(2); - expect(rows[0]).toEqual({ - id: 1, - val: 'v1', - }); + expect(rows).toEqual([ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + ]); +}); + +test('JSON object import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify({ + k1: { id: 1, val: 'v1' }, + k2: { id: 2, val: 'v2' }, + }) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + jsonStyle: 'object', + keyField: 'mykey', + }); + + const rows = await getReaderRows(reader); + + expect(rows.length).toEqual(2); + expect(rows).toEqual([ + { mykey: 'k1', id: 1, val: 'v1' }, + { mykey: 'k2', id: 2, val: 'v2' }, + ]); }); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js index b54b9471..74a76cee 100644 --- a/packages/api/src/shell/jsonReader.js +++ b/packages/api/src/shell/jsonReader.js @@ -6,12 +6,15 @@ const logger = getLogger('jsonReader'); const { parser } = require('stream-json'); const { pick } = require('stream-json/filters/Pick'); const { streamArray } = require('stream-json/streamers/StreamArray'); +const { streamObject } = require('stream-json/streamers/StreamObject'); class ParseStream extends stream.Transform { - constructor({ limitRows }) { + constructor({ limitRows, jsonStyle, keyField }) { super({ objectMode: true }); this.wasHeader = false; this.limitRows = limitRows; + this.jsonStyle = jsonStyle; + this.keyField = keyField; this.rowsWritten = 0; } _transform(chunk, encoding, done) { @@ -24,14 +27,22 @@ class ParseStream extends stream.Transform { this.wasHeader = true; } if (!this.limitRows || this.rowsWritten < this.limitRows) { - this.push(chunk.value); + if (this.jsonStyle === 'object') { + this.push({ + ...chunk.value, + [this.keyField]: chunk.key, + }); + } else { + this.push(chunk.value); + } + this.rowsWritten += 1; } done(); } } -async function jsonReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { +async function jsonReader({ fileName, jsonStyle, keyField = '_key', encoding = 'utf-8', limitRows = undefined }) { logger.info(`Reading file ${fileName}`); const fileStream = fs.createReadStream( @@ -42,14 +53,17 @@ async function jsonReader({ fileName, encoding = 'utf-8', limitRows = undefined const parseJsonStream = parser(); fileStream.pipe(parseJsonStream); - const { streamArray } = require('stream-json/streamers/StreamArray'); - const streamArrayStream = streamArray(); + const parseStream = new ParseStream({ limitRows, jsonStyle, keyField }); - parseJsonStream.pipe(streamArrayStream); - - const parseStream = new ParseStream({ limitRows }); - - streamArrayStream.pipe(parseStream); + if (jsonStyle === 'object') { + const tramsformer = streamObject(); + parseJsonStream.pipe(tramsformer); + tramsformer.pipe(parseStream); + } else { + const tramsformer = streamArray(); + parseJsonStream.pipe(tramsformer); + tramsformer.pipe(parseStream); + } return parseStream; } From 56f015ffd5981862b2904d793497a4aac650dbea Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 13:23:51 +0200 Subject: [PATCH 18/25] JSON import rootField support --- .../__tests__/import-formats.spec.js | 29 +++++++++++++++++++ packages/api/src/shell/jsonReader.js | 26 ++++++++++++----- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index 7bfcc905..21dc38e2 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -120,3 +120,32 @@ test('JSON object import test', async () => { { mykey: 'k2', id: 2, val: 'v2' }, ]); }); + +test('JSON filtered object import test', async () => { + const jsonFileName = tmp.tmpNameSync(); + + fs.writeFileSync( + jsonFileName, + JSON.stringify({ + filtered: { + k1: { id: 1, val: 'v1' }, + k2: { id: 2, val: 'v2' }, + }, + }) + ); + + const reader = await dbgateApi.jsonReader({ + fileName: jsonFileName, + jsonStyle: 'object', + keyField: 'mykey', + rootField: 'filtered', + }); + + const rows = await getReaderRows(reader); + + expect(rows.length).toEqual(2); + expect(rows).toEqual([ + { mykey: 'k1', id: 1, val: 'v1' }, + { mykey: 'k2', id: 2, val: 'v2' }, + ]); +}); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js index 74a76cee..1347595e 100644 --- a/packages/api/src/shell/jsonReader.js +++ b/packages/api/src/shell/jsonReader.js @@ -2,12 +2,13 @@ const fs = require('fs'); const stream = require('stream'); const byline = require('byline'); const { getLogger } = require('dbgate-tools'); -const logger = getLogger('jsonReader'); const { parser } = require('stream-json'); const { pick } = require('stream-json/filters/Pick'); const { streamArray } = require('stream-json/streamers/StreamArray'); const { streamObject } = require('stream-json/streamers/StreamObject'); +const logger = getLogger('jsonReader'); + class ParseStream extends stream.Transform { constructor({ limitRows, jsonStyle, keyField }) { super({ objectMode: true }); @@ -42,7 +43,14 @@ class ParseStream extends stream.Transform { } } -async function jsonReader({ fileName, jsonStyle, keyField = '_key', encoding = 'utf-8', limitRows = undefined }) { +async function jsonReader({ + fileName, + jsonStyle, + keyField = '_key', + rootField = null, + encoding = 'utf-8', + limitRows = undefined, +}) { logger.info(`Reading file ${fileName}`); const fileStream = fs.createReadStream( @@ -55,16 +63,18 @@ async function jsonReader({ fileName, jsonStyle, keyField = '_key', encoding = ' const parseStream = new ParseStream({ limitRows, jsonStyle, keyField }); - if (jsonStyle === 'object') { - const tramsformer = streamObject(); - parseJsonStream.pipe(tramsformer); - tramsformer.pipe(parseStream); + const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray(); + + if (rootField) { + const filterStream = pick({ filter: rootField }); + parseJsonStream.pipe(filterStream); + filterStream.pipe(tramsformer); } else { - const tramsformer = streamArray(); parseJsonStream.pipe(tramsformer); - tramsformer.pipe(parseStream); } + tramsformer.pipe(parseStream); + return parseStream; } From f080b18d3f43afd63d42e68e8f107220cbb397ce Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 13:47:28 +0200 Subject: [PATCH 19/25] refactor --- packages/api/src/shell/index.js | 6 ++-- .../{jsonArrayWriter.js => jsonWriter.js} | 4 +-- packages/web/src/plugins/fileformats.ts | 29 +++++++++++++++++-- 3 files changed, 32 insertions(+), 7 deletions(-) rename packages/api/src/shell/{jsonArrayWriter.js => jsonWriter.js} (91%) diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index 82513088..1a2bef62 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -6,7 +6,7 @@ const copyStream = require('./copyStream'); const fakeObjectReader = require('./fakeObjectReader'); const consoleObjectWriter = require('./consoleObjectWriter'); const jsonLinesWriter = require('./jsonLinesWriter'); -const jsonArrayWriter = require('./jsonArrayWriter'); +const jsonWriter = require('./jsonWriter'); const jsonLinesReader = require('./jsonLinesReader'); const sqlDataWriter = require('./sqlDataWriter'); const jslDataReader = require('./jslDataReader'); @@ -38,8 +38,9 @@ const dbgateApi = { tableReader, copyStream, jsonLinesWriter, - jsonArrayWriter, jsonLinesReader, + jsonReader, + jsonWriter, sqlDataWriter, fakeObjectReader, consoleObjectWriter, @@ -62,7 +63,6 @@ const dbgateApi = { dataDuplicator, dbModelToJson, jsonToDbModel, - jsonReader, }; requirePlugin.initializeDbgateApi(dbgateApi); diff --git a/packages/api/src/shell/jsonArrayWriter.js b/packages/api/src/shell/jsonWriter.js similarity index 91% rename from packages/api/src/shell/jsonArrayWriter.js rename to packages/api/src/shell/jsonWriter.js index ed18b86f..d80f343b 100644 --- a/packages/api/src/shell/jsonArrayWriter.js +++ b/packages/api/src/shell/jsonWriter.js @@ -40,7 +40,7 @@ class StringifyStream extends stream.Transform { } } -async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) { +async function jsonWriter({ fileName, encoding = 'utf-8' }) { logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream(); const fileStream = fs.createWriteStream(fileName, encoding); @@ -49,4 +49,4 @@ async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) { return stringify; } -module.exports = jsonArrayWriter; +module.exports = jsonWriter; diff --git a/packages/web/src/plugins/fileformats.ts b/packages/web/src/plugins/fileformats.ts index 700bba3b..34b26ae6 100644 --- a/packages/web/src/plugins/fileformats.ts +++ b/packages/web/src/plugins/fileformats.ts @@ -14,7 +14,32 @@ const jsonFormat = { extension: 'json', name: 'JSON', readerFunc: 'jsonReader', - writerFunc: 'jsonArrayWriter', + writerFunc: 'jsonWriter', + + args: [ + { + type: 'select', + name: 'jsonStyle', + label: 'JSON style', + options: [ + { name: 'Array', value: 'array' }, + { name: 'Object', value: 'object' }, + ], + apiName: 'jsonStyle', + }, + { + type: 'textbox', + name: 'keyField', + label: 'Key field', + apiName: 'keyField', + }, + { + type: 'textbox', + name: 'rootField', + label: 'Root field', + apiName: 'rootField', + }, + ], }; const sqlFormat = { @@ -39,7 +64,7 @@ const jsonQuickExport = { label: 'JSON', extension: 'json', createWriter: fileName => ({ - functionName: 'jsonArrayWriter', + functionName: 'jsonWriter', props: { fileName, }, From 260b2e4b12cb71cb4cd04be85071f4627225f700 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 14:28:31 +0200 Subject: [PATCH 20/25] JSON export - support for object style, key field, root field --- packages/api/src/shell/jsonWriter.js | 60 ++++++++++++++++++++++--- packages/web/src/plugins/fileformats.ts | 8 ++-- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/packages/api/src/shell/jsonWriter.js b/packages/api/src/shell/jsonWriter.js index d80f343b..3dcb4fc6 100644 --- a/packages/api/src/shell/jsonWriter.js +++ b/packages/api/src/shell/jsonWriter.js @@ -1,17 +1,22 @@ const { getLogger } = require('dbgate-tools'); const fs = require('fs'); const stream = require('stream'); +const _ = require('lodash'); const logger = getLogger('jsonArrayWriter'); class StringifyStream extends stream.Transform { - constructor() { + constructor({ jsonStyle, keyField, rootField }) { super({ objectMode: true }); this.wasHeader = false; this.wasRecord = false; + this.jsonStyle = jsonStyle; + this.keyField = keyField; + this.rootField = rootField; } _transform(chunk, encoding, done) { let skip = false; + const keyField = this.keyField || '_key'; if (!this.wasHeader) { skip = chunk.__isStreamHeader; @@ -19,30 +24,71 @@ class StringifyStream extends stream.Transform { } if (!skip) { if (!this.wasRecord) { - this.push('[\n'); + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push(`{"${this.rootField}": {\n`); + } else { + this.push(`{"${this.rootField}": [\n`); + } + } else { + if (this.jsonStyle === 'object') { + this.push('{\n'); + } else { + this.push('[\n'); + } + } } else { this.push(',\n'); } this.wasRecord = true; - this.push(JSON.stringify(chunk)); + if (this.jsonStyle === 'object') { + const key = chunk[keyField] ?? chunk[Object.keys(chunk)[0]]; + this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [keyField]))}`); + } else { + this.push(JSON.stringify(chunk)); + } } done(); } _flush(done) { if (!this.wasRecord) { - this.push('[]\n'); + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push(`{"${this.rootField}": {}}\n`); + } else { + this.push(`{"${this.rootField}": []}\n`); + } + } else { + if (this.jsonStyle === 'object') { + this.push('{}\n'); + } else { + this.push('[]\n'); + } + } } else { - this.push('\n]\n'); + if (this.rootField) { + if (this.jsonStyle === 'object') { + this.push('\n}}\n'); + } else { + this.push('\n]}\n'); + } + } else { + if (this.jsonStyle === 'object') { + this.push('\n}\n'); + } else { + this.push('\n]\n'); + } + } } done(); } } -async function jsonWriter({ fileName, encoding = 'utf-8' }) { +async function jsonWriter({ fileName, jsonStyle, keyField, rootField, encoding = 'utf-8' }) { logger.info(`Writing file ${fileName}`); - const stringify = new StringifyStream(); + const stringify = new StringifyStream({ jsonStyle, keyField, rootField }); const fileStream = fs.createWriteStream(fileName, encoding); stringify.pipe(fileStream); stringify['finisher'] = fileStream; diff --git a/packages/web/src/plugins/fileformats.ts b/packages/web/src/plugins/fileformats.ts index 34b26ae6..94b49411 100644 --- a/packages/web/src/plugins/fileformats.ts +++ b/packages/web/src/plugins/fileformats.ts @@ -22,19 +22,19 @@ const jsonFormat = { name: 'jsonStyle', label: 'JSON style', options: [ - { name: 'Array', value: 'array' }, + { name: 'Array', value: '' }, { name: 'Object', value: 'object' }, ], apiName: 'jsonStyle', }, { - type: 'textbox', + type: 'text', name: 'keyField', - label: 'Key field', + label: 'Key field (only for "Object" style)', apiName: 'keyField', }, { - type: 'textbox', + type: 'text', name: 'rootField', label: 'Root field', apiName: 'rootField', From fc6a43b4fed282ce0049656813da3dfa483a4fc1 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 15:06:54 +0200 Subject: [PATCH 21/25] download fileat first in imports --- packages/api/src/shell/jsonLinesReader.js | 5 ++++- packages/api/src/shell/jsonReader.js | 7 +++++-- packages/api/src/shell/jsonWriter.js | 9 ++++----- packages/web/src/impexp/FilesInput.svelte | 2 +- packages/web/src/impexp/ImportExportConfigurator.svelte | 1 - packages/web/src/modals/ChangeDownloadUrlModal.svelte | 3 ++- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/api/src/shell/jsonLinesReader.js b/packages/api/src/shell/jsonLinesReader.js index b226e487..db36d966 100644 --- a/packages/api/src/shell/jsonLinesReader.js +++ b/packages/api/src/shell/jsonLinesReader.js @@ -2,6 +2,7 @@ const fs = require('fs'); const stream = require('stream'); const byline = require('byline'); const { getLogger } = require('dbgate-tools'); +const download = require('./download'); const logger = getLogger('jsonLinesReader'); class ParseStream extends stream.Transform { @@ -35,8 +36,10 @@ class ParseStream extends stream.Transform { async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) { logger.info(`Reading file ${fileName}`); + const downloadedFile = await download(fileName); + const fileStream = fs.createReadStream( - fileName, + downloadedFile, // @ts-ignore encoding ); diff --git a/packages/api/src/shell/jsonReader.js b/packages/api/src/shell/jsonReader.js index 1347595e..48423f3c 100644 --- a/packages/api/src/shell/jsonReader.js +++ b/packages/api/src/shell/jsonReader.js @@ -6,16 +6,18 @@ const { parser } = require('stream-json'); const { pick } = require('stream-json/filters/Pick'); const { streamArray } = require('stream-json/streamers/StreamArray'); const { streamObject } = require('stream-json/streamers/StreamObject'); +const download = require('./download'); const logger = getLogger('jsonReader'); + class ParseStream extends stream.Transform { constructor({ limitRows, jsonStyle, keyField }) { super({ objectMode: true }); this.wasHeader = false; this.limitRows = limitRows; this.jsonStyle = jsonStyle; - this.keyField = keyField; + this.keyField = keyField || '_key'; this.rowsWritten = 0; } _transform(chunk, encoding, done) { @@ -53,8 +55,9 @@ async function jsonReader({ }) { logger.info(`Reading file ${fileName}`); + const downloadedFile = await download(fileName); const fileStream = fs.createReadStream( - fileName, + downloadedFile, // @ts-ignore encoding ); diff --git a/packages/api/src/shell/jsonWriter.js b/packages/api/src/shell/jsonWriter.js index 3dcb4fc6..a0ce19f2 100644 --- a/packages/api/src/shell/jsonWriter.js +++ b/packages/api/src/shell/jsonWriter.js @@ -11,12 +11,11 @@ class StringifyStream extends stream.Transform { this.wasHeader = false; this.wasRecord = false; this.jsonStyle = jsonStyle; - this.keyField = keyField; + this.keyField = keyField || '_key'; this.rootField = rootField; } _transform(chunk, encoding, done) { let skip = false; - const keyField = this.keyField || '_key'; if (!this.wasHeader) { skip = chunk.__isStreamHeader; @@ -43,8 +42,8 @@ class StringifyStream extends stream.Transform { this.wasRecord = true; if (this.jsonStyle === 'object') { - const key = chunk[keyField] ?? chunk[Object.keys(chunk)[0]]; - this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [keyField]))}`); + const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]]; + this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`); } else { this.push(JSON.stringify(chunk)); } @@ -86,7 +85,7 @@ class StringifyStream extends stream.Transform { } } -async function jsonWriter({ fileName, jsonStyle, keyField, rootField, encoding = 'utf-8' }) { +async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) { logger.info(`Writing file ${fileName}`); const stringify = new StringifyStream({ jsonStyle, keyField, rootField }); const fileStream = fs.createWriteStream(fileName, encoding); diff --git a/packages/web/src/impexp/FilesInput.svelte b/packages/web/src/impexp/FilesInput.svelte index 81a4e4c6..37e4d719 100644 --- a/packages/web/src/impexp/FilesInput.svelte +++ b/packages/web/src/impexp/FilesInput.svelte @@ -1,6 +1,6 @@ - + Download imported file from web From 3357295d986267796c6eb3d596653e803f51e092 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 15:19:45 +0200 Subject: [PATCH 22/25] removed ImportExportModal --- .../src/appobj/ArchiveFileAppObject.svelte | 1 - .../web/src/appobj/DatabaseAppObject.svelte | 1 - .../src/appobj/DatabaseObjectAppObject.svelte | 1 - packages/web/src/commands/stdCommands.ts | 1 - .../datagrid/CollectionDataGridCore.svelte | 2 - .../web/src/datagrid/JslDataGridCore.svelte | 4 - .../web/src/datagrid/SqlDataGridCore.svelte | 3 - .../web/src/modals/ImportExportModal.svelte | 203 ------------------ packages/web/src/tabs/ShellTab.svelte | 2 - packages/web/src/utility/openElectronFile.ts | 1 - packages/web/src/utility/uploadFiles.ts | 1 - 11 files changed, 220 deletions(-) delete mode 100644 packages/web/src/modals/ImportExportModal.svelte diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 71b84c8e..97874510 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -65,7 +65,6 @@ - - - - - Import/Export - {#if busy} - - {/if} - - - -
- - - {#if busy} - - {/if} -
- - - - - - - - - - - - - - - - - - - - - -
- {#if busy} - Stop - {:else} - Run - {/if} - Generate script - - Close -
- - - - - diff --git a/packages/web/src/tabs/ShellTab.svelte b/packages/web/src/tabs/ShellTab.svelte index 1e865a47..e8bba0bc 100644 --- a/packages/web/src/tabs/ShellTab.svelte +++ b/packages/web/src/tabs/ShellTab.svelte @@ -47,8 +47,6 @@ import { registerFileCommands } from '../commands/stdCommands'; import VerticalSplitter from '../elements/VerticalSplitter.svelte'; - import ImportExportModal from '../modals/ImportExportModal.svelte'; - import { showModal } from '../modals/modalTools'; import AceEditor from '../query/AceEditor.svelte'; import RunnerOutputPane from '../query/RunnerOutputPane.svelte'; import useEditorData from '../query/useEditorData'; diff --git a/packages/web/src/utility/openElectronFile.ts b/packages/web/src/utility/openElectronFile.ts index a07782e7..c4e0b09d 100644 --- a/packages/web/src/utility/openElectronFile.ts +++ b/packages/web/src/utility/openElectronFile.ts @@ -1,7 +1,6 @@ import { showModal } from '../modals/modalTools'; import { get } from 'svelte/store'; import newQuery from '../query/newQuery'; -import ImportExportModal from '../modals/ImportExportModal.svelte'; import getElectron from './getElectron'; import { currentDatabase, extensions, getCurrentDatabase } from '../stores'; import { getUploadListener } from './uploadFiles'; diff --git a/packages/web/src/utility/uploadFiles.ts b/packages/web/src/utility/uploadFiles.ts index 7fe245df..ddbcfa50 100644 --- a/packages/web/src/utility/uploadFiles.ts +++ b/packages/web/src/utility/uploadFiles.ts @@ -5,7 +5,6 @@ import getElectron from './getElectron'; import resolveApi, { resolveApiHeaders } from './resolveApi'; import { findFileFormat } from '../plugins/fileformats'; import { showModal } from '../modals/modalTools'; -import ImportExportModal from '../modals/ImportExportModal.svelte'; import ErrorMessageModal from '../modals/ErrorMessageModal.svelte'; import openNewTab from './openNewTab'; import { openImportExportTab } from './importExportTools'; From a0527d78e9d918706775d0696c4ab31005d1dc5d Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 15:47:40 +0200 Subject: [PATCH 23/25] save import/export jobs --- .../web/src/appobj/SavedFileAppObject.svelte | 9 ++++ packages/web/src/tabs/ImportExportTab.svelte | 44 +++++++++++-------- .../web/src/widgets/SavedFilesList.svelte | 4 +- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/packages/web/src/appobj/SavedFileAppObject.svelte b/packages/web/src/appobj/SavedFileAppObject.svelte index 97496c30..89c468d6 100644 --- a/packages/web/src/appobj/SavedFileAppObject.svelte +++ b/packages/web/src/appobj/SavedFileAppObject.svelte @@ -65,6 +65,14 @@ currentConnection: true, }; + const jobs: FileTypeHandler = { + icon: 'img export', + format: 'json', + tabComponent: 'ImportExportTab', + folder: 'jobs', + currentConnection: false, + }; + const perspectives: FileTypeHandler = { icon: 'img perspective', format: 'json', @@ -82,6 +90,7 @@ sqlite, diagrams, perspectives, + jobs, }; export const extractKey = data => data.file; diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte index 69e219d5..b7d0a664 100644 --- a/packages/web/src/tabs/ImportExportTab.svelte +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -1,15 +1,25 @@ + + @@ -218,19 +238,6 @@ - - {#if busy} @@ -239,6 +246,7 @@ Run {/if} Generate script + diff --git a/packages/web/src/widgets/SavedFilesList.svelte b/packages/web/src/widgets/SavedFilesList.svelte index 9d798903..44d8da92 100644 --- a/packages/web/src/widgets/SavedFilesList.svelte +++ b/packages/web/src/widgets/SavedFilesList.svelte @@ -20,6 +20,7 @@ const queryFiles = useFiles({ folder: 'query' }); const sqliteFiles = useFiles({ folder: 'sqlite' }); const diagramFiles = useFiles({ folder: 'diagrams' }); + const jobFiles = useFiles({ folder: 'jobs' }); const perspectiveFiles = useFiles({ folder: 'perspectives' }); $: files = [ @@ -31,11 +32,12 @@ ...($sqliteFiles || []), ...($diagramFiles || []), ...($perspectiveFiles || []), + ...($jobFiles || []), ]; function handleRefreshFiles() { apiCall('files/refresh', { - folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives'], + folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives', 'jobs'], }); } From 226512a4ca9ed5cd6172699a09967dc3fc8c5b8c Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 15:50:01 +0200 Subject: [PATCH 24/25] removed open wizard from shell function --- packages/web/src/impexp/createImpExpScript.ts | 6 +---- packages/web/src/tabs/ShellTab.svelte | 27 +------------------ 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index 5ea6c3b4..cb001994 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -192,7 +192,7 @@ export function normalizeExportColumnMap(colmap) { return null; } -export default async function createImpExpScript(extensions, values, addEditorInfo = true, forceScript = false) { +export default async function createImpExpScript(extensions, values, forceScript = false) { const config = getCurrentConfig(); const script = config.allowShellScripting || forceScript @@ -233,10 +233,6 @@ export default async function createImpExpScript(extensions, values, addEditorIn script.copyStream(sourceVar, targetVar, colmapVar); script.endLine(); } - if (addEditorInfo) { - script.comment('@ImportExportConfigurator'); - script.comment(JSON.stringify(values)); - } return script.getScript(values.schedule); } diff --git a/packages/web/src/tabs/ShellTab.svelte b/packages/web/src/tabs/ShellTab.svelte index e8bba0bc..6a810006 100644 --- a/packages/web/src/tabs/ShellTab.svelte +++ b/packages/web/src/tabs/ShellTab.svelte @@ -23,15 +23,6 @@ onClick: () => getCurrentEditor().copyNodeScript(), }); - registerCommand({ - id: 'shell.openWizard', - category: 'Shell', - name: 'Open wizard', - // testEnabled: () => getCurrentEditor()?.openWizardEnabled(), - onClick: () => getCurrentEditor().openWizard(), - }); - - const configRegex = /\s*\/\/\s*@ImportExportConfigurator\s*\n\s*\/\/\s*(\{[^\n]+\})\n/; const requireRegex = /\s*(\/\/\s*@require\s+[^\n]+)\n/g; const initRegex = /([^\n]+\/\/\s*@init)/g; @@ -58,8 +49,7 @@ import { showSnackbarError } from '../utility/snackbar'; import useEffect from '../utility/useEffect'; import useTimerLabel from '../utility/useTimerLabel'; - import { openImportExportTab } from '../utility/importExportTools'; - + export let tabid; const tabVisible: any = getContext('tabVisible'); @@ -148,20 +138,6 @@ copyTextToClipboard(resp); } - // export function openWizardEnabled() { - // return ($editorValue || '').match(configRegex); - // } - - export function openWizard() { - const jsonTextMatch = ($editorValue || '').match(configRegex); - if (jsonTextMatch) { - openImportExportTab(JSON.parse(jsonTextMatch[1])); - // showModal(ImportExportModal, { initialValues: JSON.parse(jsonTextMatch[1]) }); - } else { - showSnackbarError('No wizard info found'); - } - } - function getActiveScript() { const selectedText = domEditor.getEditor().getSelectedText(); const editorText = $editorValue; @@ -208,7 +184,6 @@ return [ { command: 'shell.execute' }, { command: 'shell.kill' }, - { command: 'shell.openWizard' }, { divider: true }, { command: 'shell.toggleComment' }, { divider: true }, From b346a458a617a4a029db34cd3e08260d42479b18 Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Tue, 17 Sep 2024 16:00:59 +0200 Subject: [PATCH 25/25] fix --- packages/api/src/shell/modifyJsonLinesReader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index 7aa00736..dec137ba 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -66,7 +66,7 @@ class ParseStream extends stream.Transform { ...obj, ...update.fields, }, - (v, k) => v.$$undefined$$ + (v, k) => v?.$$undefined$$ ); } }