mirrors/dbgate
1
0
Fork 0
mirror of https://github.com/dbgate/dbgate synced 2024-09-19 21:02:03 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Merge branch 'feature/import-export'

Browse Source
This commit is contained in:
SPRINX0\prochazka 2024-09-17 16:17:46 +02:00
commit 5c58c35a64
40 changed files with 796 additions and 533 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/run-tests.yaml vendored
View File

@ -4,6 +4,7 @@ on:
branches:
- master
- develop
- 'feature/**'
jobs:
test-runner:

151
integration-tests/__tests__/import-formats.spec.js Normal file
View File

@ -0,0 +1,151 @@
const dbgateApi = require('dbgate-api/src/shell');
// const jsonLinesWriter = require('dbgate-api/src/shell/jsonLinesWriter');
const tmp = require('tmp');
// const dbgatePluginCsv = require('dbgate-plugin-csv/src/backend');
const fs = require('fs');
const requirePlugin = require('dbgate-api/src/shell/requirePlugin');
const CSV_DATA = `Issue Number; Title; Github URL; Labels; State; Created At; Updated At; Reporter; Assignee
801; "Does it 'burst' the database on startup or first lUI load ? "; https://github.com/dbgate/dbgate/issues/801; ""; open; 05/23/2024; 05/23/2024; rgarrigue;
799; "BUG: latest AppImage crashes on opening in Fedora 39"; https://github.com/dbgate/dbgate/issues/799; ""; open; 05/21/2024; 05/24/2024; BenGraham-Git;
798; "MongoDB write operations fail"; https://github.com/dbgate/dbgate/issues/798; "bug,solved"; open; 05/21/2024; 05/24/2024; mahmed0715;
797; "BUG: Unable to open SQL files"; https://github.com/dbgate/dbgate/issues/797; "bug"; open; 05/20/2024; 05/21/2024; cesarValdivia;
795; "BUG: MS SQL Server connection error (KEY_USAGE_BIT_INCORRECT)"; https://github.com/dbgate/dbgate/issues/795; ""; open; 05/20/2024; 05/20/2024; keskinonur;
794; "GLIBC_2.29' not found and i have 2.31"; https://github.com/dbgate/dbgate/issues/794; ""; closed; 05/20/2024; 05/21/2024; MFdanGM;
793; "BUG: PostgresSQL doesn't show tables when connected"; https://github.com/dbgate/dbgate/issues/793; ""; open; 05/20/2024; 05/22/2024; stomper013;
792; "FEAT: Wayland support"; https://github.com/dbgate/dbgate/issues/792; ""; closed; 05/19/2024; 05/21/2024; VosaXalo;
`;
async function getReaderRows(reader) {
const jsonLinesFileName = tmp.tmpNameSync();
const writer = await dbgateApi.jsonLinesWriter({
fileName: jsonLinesFileName,
});
await dbgateApi.copyStream(reader, writer);
const jsonData = fs.readFileSync(jsonLinesFileName, 'utf-8');
const rows = jsonData
.split('\n')
.filter(x => x.trim() !== '')
.map(x => JSON.parse(x));
return rows;
}
test('csv import test', async () => {
const dbgatePluginCsv = requirePlugin('dbgate-plugin-csv');
const csvFileName = tmp.tmpNameSync();
fs.writeFileSync(csvFileName, CSV_DATA);
const reader = await dbgatePluginCsv.shellApi.reader({
fileName: csvFileName,
});
const rows = await getReaderRows(reader);
expect(rows[0].columns).toEqual([
{ columnName: 'Issue Number' },
{ columnName: 'Title' },
{ columnName: 'Github URL' },
{ columnName: 'Labels' },
{ columnName: 'State' },
{ columnName: 'Created At' },
{ columnName: 'Updated At' },
{ columnName: 'Reporter' },
{ columnName: 'Assignee' },
]);
expect(rows.length).toEqual(9);
expect(rows[1]).toEqual({
'Issue Number': '801',
Title: "Does it 'burst' the database on startup or first lUI load ? ",
'Github URL': 'https://github.com/dbgate/dbgate/issues/801',
Labels: '',
State: 'open',
'Created At': '05/23/2024',
'Updated At': '05/23/2024',
Reporter: 'rgarrigue',
Assignee: '',
});
});
test('JSON array import test', async () => {
const jsonFileName = tmp.tmpNameSync();
fs.writeFileSync(
jsonFileName,
JSON.stringify([
{ id: 1, val: 'v1' },
{ id: 2, val: 'v2' },
])
);
const reader = await dbgateApi.jsonReader({
fileName: jsonFileName,
});
const rows = await getReaderRows(reader);
expect(rows.length).toEqual(2);
expect(rows).toEqual([
{ id: 1, val: 'v1' },
{ id: 2, val: 'v2' },
]);
});
test('JSON object import test', async () => {
const jsonFileName = tmp.tmpNameSync();
fs.writeFileSync(
jsonFileName,
JSON.stringify({
k1: { id: 1, val: 'v1' },
k2: { id: 2, val: 'v2' },
})
);
const reader = await dbgateApi.jsonReader({
fileName: jsonFileName,
jsonStyle: 'object',
keyField: 'mykey',
});
const rows = await getReaderRows(reader);
expect(rows.length).toEqual(2);
expect(rows).toEqual([
{ mykey: 'k1', id: 1, val: 'v1' },
{ mykey: 'k2', id: 2, val: 'v2' },
]);
});
test('JSON filtered object import test', async () => {
const jsonFileName = tmp.tmpNameSync();
fs.writeFileSync(
jsonFileName,
JSON.stringify({
filtered: {
k1: { id: 1, val: 'v1' },
k2: { id: 2, val: 'v2' },
},
})
);
const reader = await dbgateApi.jsonReader({
fileName: jsonFileName,
jsonStyle: 'object',
keyField: 'mykey',
rootField: 'filtered',
});
const rows = await getReaderRows(reader);
expect(rows.length).toEqual(2);
expect(rows).toEqual([
{ mykey: 'k1', id: 1, val: 'v1' },
{ mykey: 'k2', id: 2, val: 'v2' },
]);
});

3
integration-tests/package.json
View File

@ -22,6 +22,7 @@
"devDependencies": {
"cross-env": "^7.0.3",
"jest": "^27.0.1",
"pino-pretty": "^11.2.2"
"pino-pretty": "^11.2.2",
"tmp": "^0.2.3"
}
}

8
integration-tests/setupTests.js
View File

@ -1,4 +1,10 @@
global.DBGATE_PACKAGES = {
'dbgate-tools': require('dbgate-tools'),
'dbgate-sqltree': require('dbgate-sqltree'),
};
const { prettyFactory } = require('pino-pretty');
const tmp = require('tmp');
const pretty = prettyFactory({
colorize: true,
@ -20,3 +26,5 @@ global.console = {
process.stdout.write(messages.join(' ') + '\n');
},
};
tmp.setGracefulCleanup();

5
integration-tests/tools.js
View File

@ -1,8 +1,3 @@
global.DBGATE_PACKAGES = {
'dbgate-tools': require('dbgate-tools'),
'dbgate-sqltree': require('dbgate-sqltree'),
};
const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
const crypto = require('crypto');

1
package.json
View File

@ -36,6 +36,7 @@
"build:api": "yarn workspace dbgate-api build",
"build:web:docker": "yarn workspace dbgate-web build",
"build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
"build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
"build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
"storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
"plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",

1
packages/api/package.json
View File

@ -57,6 +57,7 @@
"rimraf": "^3.0.0",
"simple-encryptor": "^4.0.0",
"ssh2": "^1.11.0",
"stream-json": "^1.8.0",
"tar": "^6.0.5"
},
"scripts": {

9
packages/api/src/controllers/jsldata.js
View File

@ -18,11 +18,14 @@ function readFirstLine(file) {
}
if (reader.hasNextLine()) {
reader.nextLine((err, line) => {
if (err) reject(err);
resolve(line);
if (err) {
reader.close(() => reject(err)); // Ensure reader is closed on error
return;
}
reader.close(() => resolve(line)); // Ensure reader is closed after reading
});
} else {
resolve(null);
reader.close(() => resolve(null)); // Properly close if no lines are present
}
});
});

6
packages/api/src/shell/index.js
View File

@ -6,7 +6,7 @@ const copyStream = require('./copyStream');
const fakeObjectReader = require('./fakeObjectReader');
const consoleObjectWriter = require('./consoleObjectWriter');
const jsonLinesWriter = require('./jsonLinesWriter');
const jsonArrayWriter = require('./jsonArrayWriter');
const jsonWriter = require('./jsonWriter');
const jsonLinesReader = require('./jsonLinesReader');
const sqlDataWriter = require('./sqlDataWriter');
const jslDataReader = require('./jslDataReader');
@ -29,6 +29,7 @@ const modifyJsonLinesReader = require('./modifyJsonLinesReader');
const dataDuplicator = require('./dataDuplicator');
const dbModelToJson = require('./dbModelToJson');
const jsonToDbModel = require('./jsonToDbModel');
const jsonReader = require('./jsonReader');
const dbgateApi = {
queryReader,
@ -37,8 +38,9 @@ const dbgateApi = {
tableReader,
copyStream,
jsonLinesWriter,
jsonArrayWriter,
jsonLinesReader,
jsonReader,
jsonWriter,
sqlDataWriter,
fakeObjectReader,
consoleObjectWriter,

52
packages/api/src/shell/jsonArrayWriter.js
View File

@ -1,52 +0,0 @@
const { getLogger } = require('dbgate-tools');
const fs = require('fs');
const stream = require('stream');
const logger = getLogger('jsonArrayWriter');
class StringifyStream extends stream.Transform {
constructor() {
super({ objectMode: true });
this.wasHeader = false;
this.wasRecord = false;
}
_transform(chunk, encoding, done) {
let skip = false;
if (!this.wasHeader) {
skip = chunk.__isStreamHeader;
this.wasHeader = true;
}
if (!skip) {
if (!this.wasRecord) {
this.push('[\n');
} else {
this.push(',\n');
}
this.wasRecord = true;
this.push(JSON.stringify(chunk));
}
done();
}
_flush(done) {
if (!this.wasRecord) {
this.push('[]\n');
} else {
this.push('\n]\n');
}
done();
}
}
async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
logger.info(`Writing file ${fileName}`);
const stringify = new StringifyStream();
const fileStream = fs.createWriteStream(fileName, encoding);
stringify.pipe(fileStream);
stringify['finisher'] = fileStream;
return stringify;
}
module.exports = jsonArrayWriter;

5
packages/api/src/shell/jsonLinesReader.js
View File

@ -2,6 +2,7 @@ const fs = require('fs');
const stream = require('stream');
const byline = require('byline');
const { getLogger } = require('dbgate-tools');
const download = require('./download');
const logger = getLogger('jsonLinesReader');
class ParseStream extends stream.Transform {
@ -35,8 +36,10 @@ class ParseStream extends stream.Transform {
async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
logger.info(`Reading file ${fileName}`);
const downloadedFile = await download(fileName);
const fileStream = fs.createReadStream(
fileName,
downloadedFile,
// @ts-ignore
encoding
);

84
packages/api/src/shell/jsonReader.js Normal file
View File

@ -0,0 +1,84 @@
const fs = require('fs');
const stream = require('stream');
const byline = require('byline');
const { getLogger } = require('dbgate-tools');
const { parser } = require('stream-json');
const { pick } = require('stream-json/filters/Pick');
const { streamArray } = require('stream-json/streamers/StreamArray');
const { streamObject } = require('stream-json/streamers/StreamObject');
const download = require('./download');
const logger = getLogger('jsonReader');
class ParseStream extends stream.Transform {
constructor({ limitRows, jsonStyle, keyField }) {
super({ objectMode: true });
this.wasHeader = false;
this.limitRows = limitRows;
this.jsonStyle = jsonStyle;
this.keyField = keyField || '_key';
this.rowsWritten = 0;
}
_transform(chunk, encoding, done) {
if (!this.wasHeader) {
this.push({
__isStreamHeader: true,
__isDynamicStructure: true,
});
this.wasHeader = true;
}
if (!this.limitRows || this.rowsWritten < this.limitRows) {
if (this.jsonStyle === 'object') {
this.push({
...chunk.value,
[this.keyField]: chunk.key,
});
} else {
this.push(chunk.value);
}
this.rowsWritten += 1;
}
done();
}
}
async function jsonReader({
fileName,
jsonStyle,
keyField = '_key',
rootField = null,
encoding = 'utf-8',
limitRows = undefined,
}) {
logger.info(`Reading file ${fileName}`);
const downloadedFile = await download(fileName);
const fileStream = fs.createReadStream(
downloadedFile,
// @ts-ignore
encoding
);
const parseJsonStream = parser();
fileStream.pipe(parseJsonStream);
const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray();
if (rootField) {
const filterStream = pick({ filter: rootField });
parseJsonStream.pipe(filterStream);
filterStream.pipe(tramsformer);
} else {
parseJsonStream.pipe(tramsformer);
}
tramsformer.pipe(parseStream);
return parseStream;
}
module.exports = jsonReader;

97
packages/api/src/shell/jsonWriter.js Normal file
View File

@ -0,0 +1,97 @@
const { getLogger } = require('dbgate-tools');
const fs = require('fs');
const stream = require('stream');
const _ = require('lodash');
const logger = getLogger('jsonArrayWriter');
class StringifyStream extends stream.Transform {
constructor({ jsonStyle, keyField, rootField }) {
super({ objectMode: true });
this.wasHeader = false;
this.wasRecord = false;
this.jsonStyle = jsonStyle;
this.keyField = keyField || '_key';
this.rootField = rootField;
}
_transform(chunk, encoding, done) {
let skip = false;
if (!this.wasHeader) {
skip = chunk.__isStreamHeader;
this.wasHeader = true;
}
if (!skip) {
if (!this.wasRecord) {
if (this.rootField) {
if (this.jsonStyle === 'object') {
this.push(`{"${this.rootField}": {\n`);
} else {
this.push(`{"${this.rootField}": [\n`);
}
} else {
if (this.jsonStyle === 'object') {
this.push('{\n');
} else {
this.push('[\n');
}
}
} else {
this.push(',\n');
}
this.wasRecord = true;
if (this.jsonStyle === 'object') {
const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
} else {
this.push(JSON.stringify(chunk));
}
}
done();
}
_flush(done) {
if (!this.wasRecord) {
if (this.rootField) {
if (this.jsonStyle === 'object') {
this.push(`{"${this.rootField}": {}}\n`);
} else {
this.push(`{"${this.rootField}": []}\n`);
}
} else {
if (this.jsonStyle === 'object') {
this.push('{}\n');
} else {
this.push('[]\n');
}
}
} else {
if (this.rootField) {
if (this.jsonStyle === 'object') {
this.push('\n}}\n');
} else {
this.push('\n]}\n');
}
} else {
if (this.jsonStyle === 'object') {
this.push('\n}\n');
} else {
this.push('\n]\n');
}
}
}
done();
}
}
async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
logger.info(`Writing file ${fileName}`);
const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
const fileStream = fs.createWriteStream(fileName, encoding);
stringify.pipe(fileStream);
stringify['finisher'] = fileStream;
return stringify;
}
module.exports = jsonWriter;

2
packages/api/src/shell/modifyJsonLinesReader.js
View File

@ -66,7 +66,7 @@ class ParseStream extends stream.Transform {
...obj,
...update.fields,
},
(v, k) => v.$$undefined$$
(v, k) => v?.$$undefined$$
);
}
}

22
packages/web/src/appobj/ArchiveFileAppObject.svelte
View File

@ -65,18 +65,18 @@
<script lang="ts">
import _ from 'lodash';
import { filterName } from 'dbgate-tools';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { getExtensions } from '../stores';
import createQuickExportMenu from '../utility/createQuickExportMenu';
import { exportQuickExportFile } from '../utility/exportFileTools';
import { exportQuickExportFile, } from '../utility/exportFileTools';
import openNewTab from '../utility/openNewTab';
import AppObjectCore from './AppObjectCore.svelte';
import InputTextModal from '../modals/InputTextModal.svelte';
import ConfirmModal from '../modals/ConfirmModal.svelte';
import { apiCall } from '../utility/api';
import { openImportExportTab } from '../utility/importExportTools';
export let data;
@ -156,13 +156,19 @@
{
text: 'Export',
onClick: () => {
showModal(ImportExportModal, {
initialValues: {
sourceStorageType: 'archive',
sourceArchiveFolder: data.folderName,
sourceList: [data.fileName],
},
openImportExportTab({
sourceStorageType: 'archive',
sourceArchiveFolder: data.folderName,
sourceList: [data.fileName],
});
// showModal(ImportExportModal, {
// initialValues: {
// sourceStorageType: 'archive',
// sourceArchiveFolder: data.folderName,
// sourceList: [data.fileName],
// },
// });
},
}
),

45
packages/web/src/appobj/DatabaseAppObject.svelte
View File

@ -98,25 +98,39 @@
};
const handleImport = () => {
showModal(ImportExportModal, {
initialValues: {
sourceStorageType: getDefaultFileFormat($extensions).storageType,
targetStorageType: 'database',
targetConnectionId: connection._id,
targetDatabaseName: name,
},
openImportExportTab({
sourceStorageType: getDefaultFileFormat($extensions).storageType,
targetStorageType: 'database',
targetConnectionId: connection._id,
targetDatabaseName: name,
});
// showModal(ImportExportModal, {
// initialValues: {
// sourceStorageType: getDefaultFileFormat($extensions).storageType,
// targetStorageType: 'database',
// targetConnectionId: connection._id,
// targetDatabaseName: name,
// },
// });
};
const handleExport = () => {
showModal(ImportExportModal, {
initialValues: {
targetStorageType: getDefaultFileFormat($extensions).storageType,
sourceStorageType: 'database',
sourceConnectionId: connection._id,
sourceDatabaseName: name,
},
openImportExportTab({
targetStorageType: getDefaultFileFormat($extensions).storageType,
sourceStorageType: 'database',
sourceConnectionId: connection._id,
sourceDatabaseName: name,
});
// showModal(ImportExportModal, {
// initialValues: {
// targetStorageType: getDefaultFileFormat($extensions).storageType,
// sourceStorageType: 'database',
// sourceConnectionId: connection._id,
// sourceDatabaseName: name,
// },
// });
};
const handleSqlGenerator = () => {
@ -360,7 +374,6 @@
import uuidv1 from 'uuid/v1';
import _, { find } from 'lodash';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
import { getDefaultFileFormat } from '../plugins/fileformats';
@ -390,13 +403,13 @@
import ConfirmSqlModal, { runOperationOnDatabase, saveScriptToDatabase } from '../modals/ConfirmSqlModal.svelte';
import { filterAppsForDatabase } from '../utility/appTools';
import newQuery from '../query/newQuery';
import { exportSqlDump } from '../utility/exportFileTools';
import ImportDatabaseDumpModal from '../modals/ImportDatabaseDumpModal.svelte';
import ExportDatabaseDumpModal from '../modals/ExportDatabaseDumpModal.svelte';
import ConfirmModal from '../modals/ConfirmModal.svelte';
import { closeMultipleTabs } from '../tabpanel/TabsPanel.svelte';
import NewCollectionModal from '../modals/NewCollectionModal.svelte';
import hasPermission from '../utility/hasPermission';
import { openImportExportTab } from '../utility/importExportTools';
export let data;
export let passProps;

64
packages/web/src/appobj/DatabaseObjectAppObject.svelte
View File

@ -618,15 +618,22 @@
});
} else if (menu.isImport) {
const { conid, database } = data;
showModal(ImportExportModal, {
initialValues: {
sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
targetStorageType: 'database',
targetConnectionId: conid,
targetDatabaseName: database,
fixedTargetPureName: data.pureName,
},
openImportExportTab({
sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
targetStorageType: 'database',
targetConnectionId: conid,
targetDatabaseName: database,
fixedTargetPureName: data.pureName,
});
// showModal(ImportExportModal, {
// initialValues: {
// sourceStorageType: getDefaultFileFormat(getExtensions()).storageType,
// targetStorageType: 'database',
// targetConnectionId: conid,
// targetDatabaseName: database,
// fixedTargetPureName: data.pureName,
// },
// });
} else {
openDatabaseObjectDetail(
menu.tab,
@ -764,31 +771,22 @@
},
{
onClick: () => {
// openNewTab(
// {
// tabComponent: 'ImportExportTab',
// title: 'Import/Export',
// icon: 'img export',
// },
// {
// editor: {
// sourceStorageType: 'database',
// sourceConnectionId: data.conid,
// sourceDatabaseName: data.database,
// sourceSchemaName: data.schemaName,
// sourceList: [data.pureName],
// },
// }
// );
showModal(ImportExportModal, {
initialValues: {
sourceStorageType: 'database',
sourceConnectionId: data.conid,
sourceDatabaseName: data.database,
sourceSchemaName: data.schemaName,
sourceList: [data.pureName],
},
openImportExportTab({
sourceStorageType: 'database',
sourceConnectionId: data.conid,
sourceDatabaseName: data.database,
sourceSchemaName: data.schemaName,
sourceList: [data.pureName],
});
// showModal(ImportExportModal, {
// initialValues: {
// sourceStorageType: 'database',
// sourceConnectionId: data.conid,
// sourceDatabaseName: data.database,
// sourceSchemaName: data.schemaName,
// sourceList: [data.pureName],
// },
// });
},
}
);
@ -832,7 +830,6 @@
import { filterName, generateDbPairingId, getAlterDatabaseScript, getConnectionLabel } from 'dbgate-tools';
import { getConnectionInfo, getDatabaseInfo } from '../utility/metadataLoaders';
import fullDisplayName from '../utility/fullDisplayName';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { findEngineDriver } from 'dbgate-tools';
import uuidv1 from 'uuid/v1';
@ -848,6 +845,7 @@
import { format as dateFormat } from 'date-fns';
import { getDefaultFileFormat } from '../plugins/fileformats';
import hasPermission from '../utility/hasPermission';
import { openImportExportTab } from '../utility/importExportTools';
export let data;
export let passProps;

9
packages/web/src/appobj/SavedFileAppObject.svelte
View File

@ -65,6 +65,14 @@
currentConnection: true,
};
const jobs: FileTypeHandler = {
icon: 'img export',
format: 'json',
tabComponent: 'ImportExportTab',
folder: 'jobs',
currentConnection: false,
};
const perspectives: FileTypeHandler = {
icon: 'img perspective',
format: 'json',
@ -82,6 +90,7 @@
sqlite,
diagrams,
perspectives,
jobs,
};
export const extractKey = data => data.file;

20
packages/web/src/commands/stdCommands.ts
View File

@ -13,7 +13,6 @@ import registerCommand from './registerCommand';
import { get } from 'svelte/store';
import AboutModal from '../modals/AboutModal.svelte';
import SettingsModal from '../settings/SettingsModal.svelte';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
import { showModal } from '../modals/modalTools';
import newQuery, { newDiagram, newPerspective, newQueryDesign } from '../query/newQuery';
@ -44,6 +43,7 @@ import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
import NewCollectionModal from '../modals/NewCollectionModal.svelte';
import ConfirmModal from '../modals/ConfirmModal.svelte';
import localforage from 'localforage';
import { openImportExportTab } from '../utility/importExportTools';
// function themeCommand(theme: ThemeDefinition) {
// return {
@ -483,10 +483,18 @@ registerCommand({
toolbar: true,
icon: 'icon import',
onClick: () =>
showModal(ImportExportModal, {
importToCurrentTarget: true,
initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType },
}),
openImportExportTab(
{
sourceStorageType: getDefaultFileFormat(get(extensions)).storageType,
},
{
importToCurrentTarget: true,
}
),
// showModal(ImportExportModal, {
// importToCurrentTarget: true,
// initialValues: { sourceStorageType: getDefaultFileFormat(get(extensions)).storageType },
// }),
});
registerCommand({
@ -595,8 +603,6 @@ registerCommand({
onClick: () => getElectron().send('check-for-updates'),
});
export function registerFileCommands({
idPrefix,
category,

6
packages/web/src/datagrid/CollectionDataGridCore.svelte
View File

@ -122,8 +122,6 @@
import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
import registerCommand from '../commands/registerCommand';
import { extractShellConnection } from '../impexp/createImpExpScript';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { apiCall } from '../utility/api';
import { registerMenu } from '../utility/contextMenu';
@ -136,6 +134,7 @@
import LoadingDataGridCore from './LoadingDataGridCore.svelte';
import { mongoFilterBehaviour, standardFilterBehaviours } from 'dbgate-tools';
import { openImportExportTab } from '../utility/importExportTools';
export let conid;
export let display;
@ -207,7 +206,8 @@
initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native';
initialValues.sourceList = [pureName];
initialValues[`columns_${pureName}`] = display.getExportColumnMap();
showModal(ImportExportModal, { initialValues });
openImportExportTab(initialValues);
// showModal(ImportExportModal, { initialValues });
}
export function openQuery() {

8
packages/web/src/datagrid/JslDataGridCore.svelte
View File

@ -45,9 +45,6 @@
import _ from 'lodash';
import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
import registerCommand from '../commands/registerCommand';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { extensions } from '../stores';
import { apiCall, apiOff, apiOn } from '../utility/api';
import { registerMenu } from '../utility/contextMenu';
@ -58,7 +55,7 @@
import ChangeSetGrider from './ChangeSetGrider';
import LoadingDataGridCore from './LoadingDataGridCore.svelte';
import RowsArrayGrider from './RowsArrayGrider';
import { openImportExportTab } from '../utility/importExportTools';
export let jslid;
export let display;
@ -152,7 +149,8 @@
initialValues.sourceList = ['query-data'];
initialValues[`columns_query-data`] = display.getExportColumnMap();
}
showModal(ImportExportModal, { initialValues });
openImportExportTab(initialValues);
// showModal(ImportExportModal, { initialValues });
}
const quickExportHandler = fmt => async () => {

7
packages/web/src/datagrid/SqlDataGridCore.svelte
View File

@ -65,13 +65,10 @@
<script lang="ts">
import _ from 'lodash';
import { getContext } from 'svelte';
import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
import registerCommand from '../commands/registerCommand';
import { extractShellConnection } from '../impexp/createImpExpScript';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { apiCall } from '../utility/api';
import { registerMenu } from '../utility/contextMenu';
@ -84,6 +81,7 @@
import LoadingDataGridCore from './LoadingDataGridCore.svelte';
import hasPermission from '../utility/hasPermission';
import { openImportExportTab } from '../utility/importExportTools';
export let conid;
export let display;
@ -145,7 +143,8 @@
initialValues.sourceQueryType = coninfo.isReadOnly ? 'json' : 'native';
initialValues.sourceList = display.baseTableOrSimilar ? [display.baseTableOrSimilar.pureName] : [];
initialValues[`columns_${pureName}`] = display.getExportColumnMap();
showModal(ImportExportModal, { initialValues });
openImportExportTab(initialValues);
// showModal(ImportExportModal, { initialValues });
}
export function openQuery() {

2
packages/web/src/impexp/FilesInput.svelte
View File

@ -1,6 +1,6 @@
<script lang="ts" context="module">
function extractUrlName(url, values) {
const match = url.match(/\/([^/]+)($|\?)/);
const match = url.match(/\/([^/\?]+)($|\?)/);
if (match) {
const res = match[1];
if (res.includes('.')) {

47
packages/web/src/impexp/ImportExportConfigurator.svelte
View File

@ -46,7 +46,6 @@
</script>
<script lang="ts">
import { onMount } from 'svelte';
import { writable } from 'svelte/store';
import Link from '../elements/Link.svelte';
import TableControl from '../elements/TableControl.svelte';
@ -67,10 +66,12 @@
import SourceName from './SourceName.svelte';
import SourceTargetConfig from './SourceTargetConfig.svelte';
import useEffect from '../utility/useEffect';
export let uploadedFile = undefined;
export let openedFile = undefined;
// export let uploadedFile = undefined;
// export let openedFile = undefined;
export let previewReaderStore;
export let isTabActive;
const { values, setFieldValue } = getFormContext();
@ -98,7 +99,7 @@
}
};
const handleUpload = file => {
export function addUploadedFile(file) {
addFilesToSourceList(
$extensions,
[
@ -113,36 +114,20 @@
previewSource.set
);
// setFieldValue('sourceList', [...(sourceList || []), file.originalName]);
};
}
onMount(() => {
setUploadListener(handleUpload);
if (uploadedFile) {
handleUpload(uploadedFile);
$: effectActiveTab = useEffect(() => {
if (isTabActive) {
setUploadListener(addUploadedFile);
return () => {
setUploadListener(null);
};
} else {
return () => {};
}
if (openedFile) {
handleUpload(openedFile);
// addFilesToSourceList(
// $extensions,
// [
// {
// fileName: openedFile.filePath,
// shortName: openedFile.shortName,
// },
// ],
// $values,
// values,
// !sourceList || sourceList.length == 0 ? openedFile.storageType : null,
// previewSource.set
// );
}
return () => {
setUploadListener(null);
};
});
// engine={sourceEngine}
// {setPreviewSource}
$effectActiveTab;
</script>
<div class="flex1">

1
packages/web/src/impexp/SourceTargetConfig.svelte
View File

@ -196,6 +196,7 @@
width: 20vw;
margin-left: var(--dim-large-form-margin);
margin-bottom: var(--dim-large-form-margin);
border: 1px solid var(--theme-border);
}
.label {

6
packages/web/src/impexp/createImpExpScript.ts
View File

@ -192,7 +192,7 @@ export function normalizeExportColumnMap(colmap) {
return null;
}
export default async function createImpExpScript(extensions, values, addEditorInfo = true, forceScript = false) {
export default async function createImpExpScript(extensions, values, forceScript = false) {
const config = getCurrentConfig();
const script =
config.allowShellScripting || forceScript
@ -233,10 +233,6 @@ export default async function createImpExpScript(extensions, values, addEditorIn
script.copyStream(sourceVar, targetVar, colmapVar);
script.endLine();
}
if (addEditorInfo) {
script.comment('@ImportExportConfigurator');
script.comment(JSON.stringify(values));
}
return script.getScript(values.schedule);
}

3
packages/web/src/modals/ChangeDownloadUrlModal.svelte
View File

@ -8,6 +8,7 @@
import { closeCurrentModal } from './modalTools';
export let onConfirm;
export let url;
const handleSubmit = e => {
onConfirm(e.detail.url);
@ -15,7 +16,7 @@
};
</script>
<FormProvider>
<FormProvider initialValues={{ url }}>
<ModalBase {...$$restProps}>
<svelte:fragment slot="header">Download imported file from web</svelte:fragment>

203
packages/web/src/modals/ImportExportModal.svelte
View File

@ -1,203 +0,0 @@
<script lang="ts">
import moment from 'moment';
import { writable } from 'svelte/store';
import HorizontalSplitter from '../elements/HorizontalSplitter.svelte';
import LargeButton from '../buttons/LargeButton.svelte';
import LoadingInfo from '../elements/LoadingInfo.svelte';
import VerticalSplitter from '../elements/VerticalSplitter.svelte';
import FormProvider from '../forms/FormProvider.svelte';
import FormTextField from '../forms/FormTextField.svelte';
import LargeFormButton from '../forms/LargeFormButton.svelte';
import FontIcon from '../icons/FontIcon.svelte';
import createImpExpScript from '../impexp/createImpExpScript';
import ImportExportConfigurator from '../impexp/ImportExportConfigurator.svelte';
import PreviewDataGrid from '../impexp/PreviewDataGrid.svelte';
import { getDefaultFileFormat } from '../plugins/fileformats';
import RunnerOutputFiles from '../query/RunnerOutputFiles.svelte';
import SocketMessageView from '../query/SocketMessageView.svelte';
import { currentArchive, currentDatabase, extensions, visibleWidgetSideBar, selectedWidget } from '../stores';
import { apiCall, apiOff, apiOn } from '../utility/api';
import createRef from '../utility/createRef';
import openNewTab from '../utility/openNewTab';
import useEffect from '../utility/useEffect';
import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte';
import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte';
import ModalBase from './ModalBase.svelte';
import { closeCurrentModal } from './modalTools';
let busy = false;
let executeNumber = 0;
let runnerId = null;
const previewReaderStore = writable(null);
export let initialValues;
export let uploadedFile = undefined;
export let openedFile = undefined;
export let importToCurrentTarget = false;
const refreshArchiveFolderRef = createRef(null);
function detectCurrentTarget() {
if (!importToCurrentTarget) return {};
if ($currentDatabase && $selectedWidget != 'archive') {
return {
targetStorageType: 'database',
targetConnectionId: $currentDatabase?.connection?._id,
targetDatabaseName: $currentDatabase?.name,
};
}
if ($currentArchive == 'default') {
return {
targetStorageType: 'archive',
targetArchiveFolder: `import-${moment().format('YYYY-MM-DD-hh-mm-ss')}`,
};
} else {
return {
targetStorageType: 'archive',
targetArchiveFolder: $currentArchive,
};
}
}
$: effect = useEffect(() => registerRunnerDone(runnerId));
function registerRunnerDone(rid) {
if (rid) {
apiOn(`runner-done-${rid}`, handleRunnerDone);
return () => {
apiOff(`runner-done-${rid}`, handleRunnerDone);
};
} else {
return () => {};
}
}
$: $effect;
const handleRunnerDone = () => {
busy = false;
if (refreshArchiveFolderRef.get()) {
apiCall('archive/refresh-folders', {});
apiCall('archive/refresh-files', { folder: refreshArchiveFolderRef.get() });
$currentArchive = refreshArchiveFolderRef.get();
$selectedWidget = 'archive';
$visibleWidgetSideBar = true;
}
};
const handleGenerateScript = async e => {
closeCurrentModal();
const code = await createImpExpScript($extensions, e.detail, undefined, true);
openNewTab(
{
title: 'Shell #',
icon: 'img shell',
tabComponent: 'ShellTab',
},
{ editor: code }
);
};
const handleExecute = async e => {
if (busy) return;
const values = e.detail;
busy = true;
const script = await createImpExpScript($extensions, values);
executeNumber += 1;
let runid = runnerId;
const resp = await apiCall('runners/start', { script });
runid = resp.runid;
runnerId = runid;
if (values.targetStorageType == 'archive') {
refreshArchiveFolderRef.set(values.targetArchiveFolder);
} else {
refreshArchiveFolderRef.set(null);
}
};
const handleCancel = () => {
apiCall('runners/cancel', {
runid: runnerId,
});
};
</script>
<FormProvider
initialValues={{
sourceStorageType: 'database',
targetStorageType: getDefaultFileFormat($extensions).storageType,
targetArchiveFolder: $currentArchive,
sourceArchiveFolder: $currentArchive,
...detectCurrentTarget(),
...initialValues,
}}
>
<ModalBase {...$$restProps} fullScreen>
<svelte:fragment slot="header">
Import/Export
{#if busy}
<FontIcon icon="icon loading" />
{/if}
</svelte:fragment>
<HorizontalSplitter initialValue="70%">
<div class="content" slot="1">
<ImportExportConfigurator {uploadedFile} {openedFile} {previewReaderStore} />
{#if busy}
<LoadingInfo wrapper message="Processing import/export ..." />
{/if}
</div>
<svelte:fragment slot="2">
<WidgetColumnBar>
<WidgetColumnBarItem title="Output files" name="output" height="20%">
<RunnerOutputFiles {runnerId} {executeNumber} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Messages" name="messages">
<SocketMessageView
eventName={runnerId ? `runner-info-${runnerId}` : null}
{executeNumber}
showNoMessagesAlert
/>
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Preview" name="preview" skip={!$previewReaderStore}>
<PreviewDataGrid reader={$previewReaderStore} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Advanced configuration" name="config" collapsed>
<FormTextField label="Schedule" name="schedule" />
<FormTextField label="Start variable index" name="startVariableIndex" />
</WidgetColumnBarItem>
</WidgetColumnBar>
</svelte:fragment>
</HorizontalSplitter>
<svelte:fragment slot="footer">
<div class="flex m-2">
{#if busy}
<LargeButton icon="icon stop" on:click={handleCancel}>Stop</LargeButton>
{:else}
<LargeFormButton on:click={handleExecute} icon="icon run">Run</LargeFormButton>
{/if}
<LargeFormButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</LargeFormButton>
<LargeButton on:click={closeCurrentModal} icon="icon close">Close</LargeButton>
</div>
</svelte:fragment>
</ModalBase>
</FormProvider>
<style>
.content {
flex: 1;
display: flex;
flex-direction: column;
overflow-y: auto;
overflow-x: hidden;
}
</style>

30
packages/web/src/plugins/fileformats.ts
View File

@ -13,7 +13,33 @@ const jsonFormat = {
storageType: 'json',
extension: 'json',
name: 'JSON',
writerFunc: 'jsonArrayWriter',
readerFunc: 'jsonReader',
writerFunc: 'jsonWriter',
args: [
{
type: 'select',
name: 'jsonStyle',
label: 'JSON style',
options: [
{ name: 'Array', value: '' },
{ name: 'Object', value: 'object' },
],
apiName: 'jsonStyle',
},
{
type: 'text',
name: 'keyField',
label: 'Key field (only for "Object" style)',
apiName: 'keyField',
},
{
type: 'text',
name: 'rootField',
label: 'Root field',
apiName: 'rootField',
},
],
};
const sqlFormat = {
@ -38,7 +64,7 @@ const jsonQuickExport = {
label: 'JSON',
extension: 'json',
createWriter: fileName => ({
functionName: 'jsonArrayWriter',
functionName: 'jsonWriter',
props: {
fileName,
},

193
packages/web/src/tabs/ImportExportTab.svelte
View File

@ -1,22 +1,39 @@
<script lang="ts" context="module">
const getCurrentEditor = () => getActiveComponent('ImportExportTab');
registerFileCommands({
idPrefix: 'job',
category: 'Job',
getCurrentEditor,
folder: 'jobs',
format: 'json',
fileExtension: 'job',
// undoRedo: true,
});
</script>
<script lang="ts">
import moment from 'moment';
import { writable } from 'svelte/store';
import HorizontalSplitter from '../elements/HorizontalSplitter.svelte';
import LargeButton from '../buttons/LargeButton.svelte';
import LoadingInfo from '../elements/LoadingInfo.svelte';
import VerticalSplitter from '../elements/VerticalSplitter.svelte';
import FormProvider from '../forms/FormProvider.svelte';
import FormTextField from '../forms/FormTextField.svelte';
import LargeFormButton from '../forms/LargeFormButton.svelte';
import FontIcon from '../icons/FontIcon.svelte';
import createImpExpScript from '../impexp/createImpExpScript';
import ImportExportConfigurator from '../impexp/ImportExportConfigurator.svelte';
import PreviewDataGrid from '../impexp/PreviewDataGrid.svelte';
import { getDefaultFileFormat } from '../plugins/fileformats';
import RunnerOutputFiles from '../query/RunnerOutputFiles.svelte';
import SocketMessageView from '../query/SocketMessageView.svelte';
import { currentArchive, currentDatabase, extensions, visibleWidgetSideBar, selectedWidget } from '../stores';
import {
currentArchive,
currentDatabase,
extensions,
visibleWidgetSideBar,
selectedWidget,
activeTabId,
} from '../stores';
import { apiCall, apiOff, apiOn } from '../utility/api';
import createRef from '../utility/createRef';
import openNewTab from '../utility/openNewTab';
@ -24,6 +41,15 @@
import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte';
import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte';
import useEditorData from '../query/useEditorData';
import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
import ToolStripButton from '../buttons/ToolStripButton.svelte';
import FormProviderCore from '../forms/FormProviderCore.svelte';
import { changeTab } from '../utility/common';
import _ from 'lodash';
import createActivator, { getActiveComponent } from '../utility/createActivator';
import { registerFileCommands } from '../commands/stdCommands';
import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
import ToolStripSaveButton from '../buttons/ToolStripSaveButton.svelte';
let busy = false;
let executeNumber = 0;
@ -32,17 +58,57 @@
const previewReaderStore = writable(null);
export let tabid;
export let initialValues;
export let uploadedFile = undefined;
export let openedFile = undefined;
export let importToCurrentTarget = false;
const refreshArchiveFolderRef = createRef(null);
const formValues = writable({});
let domConfigurator;
export const activator = createActivator('ImportExportTab', true);
// const formValues = writable({
// sourceStorageType: 'database',
// targetStorageType: getDefaultFileFormat($extensions).storageType,
// targetArchiveFolder: $currentArchive,
// sourceArchiveFolder: $currentArchive,
// ...detectCurrentTarget(),
// ...initialValues,
// });
const { editorState, editorValue, setEditorData } = useEditorData({
tabid,
onInitialData: value => {
$formValues = {
sourceStorageType: 'database',
targetStorageType: getDefaultFileFormat($extensions).storageType,
targetArchiveFolder: $currentArchive,
sourceArchiveFolder: $currentArchive,
...detectCurrentTarget(),
...value,
};
if (uploadedFile) {
domConfigurator.addUploadedFile(uploadedFile);
}
if (openedFile) {
domConfigurator.addUploadedFile(openedFile);
}
changeTab(tabid, tab => ({
...tab,
props: _.omit(tab.props, ['uploadedFile', 'openedFile', 'importToCurrentTarget']),
}));
},
});
// $: console.log('formValues', $formValues);
$: setEditorData($formValues);
function detectCurrentTarget() {
if (!importToCurrentTarget) return {};
@ -67,7 +133,7 @@
}
}
$: effect = useEffect(() => registerRunnerDone(runnerId));
$: effectRunner = useEffect(() => registerRunnerDone(runnerId));
function registerRunnerDone(rid) {
if (rid) {
@ -80,7 +146,7 @@
}
}
$: $effect;
$: $effectRunner;
const handleRunnerDone = () => {
busy = false;
@ -94,7 +160,8 @@
};
const handleGenerateScript = async e => {
const code = await createImpExpScript($extensions, e.detail, undefined, true);
const values = $formValues as any;
const code = await createImpExpScript($extensions, values, undefined, true);
openNewTab(
{
title: 'Shell #',
@ -107,7 +174,7 @@
const handleExecute = async e => {
if (busy) return;
const values = e.detail;
const values = $formValues as any;
busy = true;
const script = await createImpExpScript($extensions, values);
executeNumber += 1;
@ -128,63 +195,60 @@
runid: runnerId,
});
};
export function getData() {
return $editorState.value || '';
}
</script>
<FormProvider
initialValues={{
sourceStorageType: 'database',
targetStorageType: getDefaultFileFormat($extensions).storageType,
targetArchiveFolder: $currentArchive,
sourceArchiveFolder: $currentArchive,
...detectCurrentTarget(),
...initialValues,
}}
>
<HorizontalSplitter initialValue="70%">
<div class="content" slot="1">
<ImportExportConfigurator {uploadedFile} {openedFile} {previewReaderStore} />
<ToolStripContainer>
<FormProviderCore values={formValues}>
<HorizontalSplitter initialValue="70%">
<div class="content" slot="1">
<ImportExportConfigurator
bind:this={domConfigurator}
{previewReaderStore}
isTabActive={tabid == $activeTabId}
/>
{#if busy}
<LoadingInfo wrapper message="Processing import/export ..." />
{/if}
</div>
{#if busy}
<LoadingInfo wrapper message="Processing import/export ..." />
{/if}
</div>
<svelte:fragment slot="2">
<WidgetColumnBar>
<WidgetColumnBarItem title="Output files" name="output" height="20%">
<RunnerOutputFiles {runnerId} {executeNumber} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Messages" name="messages">
<SocketMessageView
eventName={runnerId ? `runner-info-${runnerId}` : null}
{executeNumber}
showNoMessagesAlert
/>
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Preview" name="preview" skip={!$previewReaderStore}>
<PreviewDataGrid reader={$previewReaderStore} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Advanced configuration" name="config" collapsed>
<FormTextField label="Schedule" name="schedule" />
<FormTextField label="Start variable index" name="startVariableIndex" />
</WidgetColumnBarItem>
</WidgetColumnBar>
</svelte:fragment>
</HorizontalSplitter>
<!-- <svelte:fragment slot="footer">
<div class="flex m-2">
{#if busy}
<LargeButton icon="icon stop" on:click={handleCancel}>Stop</LargeButton>
{:else}
<LargeFormButton on:click={handleExecute} icon="icon run">Run</LargeFormButton>
{/if}
<LargeFormButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</LargeFormButton>
<LargeButton on:click={closeCurrentModal} icon="icon close">Close</LargeButton>
</div>
</svelte:fragment> -->
</FormProvider>
<svelte:fragment slot="2">
<WidgetColumnBar>
<WidgetColumnBarItem title="Output files" name="output" height="20%">
<RunnerOutputFiles {runnerId} {executeNumber} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Messages" name="messages">
<SocketMessageView
eventName={runnerId ? `runner-info-${runnerId}` : null}
{executeNumber}
showNoMessagesAlert
/>
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Preview" name="preview" skip={!$previewReaderStore}>
<PreviewDataGrid reader={$previewReaderStore} />
</WidgetColumnBarItem>
<WidgetColumnBarItem title="Advanced configuration" name="config" collapsed>
<FormTextField label="Schedule" name="schedule" />
<FormTextField label="Start variable index" name="startVariableIndex" />
</WidgetColumnBarItem>
</WidgetColumnBar>
</svelte:fragment>
</HorizontalSplitter>
</FormProviderCore>
<svelte:fragment slot="toolstrip">
{#if busy}
<ToolStripButton icon="icon stop" on:click={handleCancel}>Stop</ToolStripButton>
{:else}
<ToolStripButton on:click={handleExecute} icon="icon run">Run</ToolStripButton>
{/if}
<ToolStripButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</ToolStripButton>
<ToolStripSaveButton idPrefix="job" />
</svelte:fragment>
</ToolStripContainer>
<style>
.content {
@ -193,5 +257,6 @@
flex-direction: column;
overflow-y: auto;
overflow-x: hidden;
background-color: var(--theme-bg-0);
}
</style>

27
packages/web/src/tabs/ShellTab.svelte
View File

@ -23,15 +23,6 @@
onClick: () => getCurrentEditor().copyNodeScript(),
});
registerCommand({
id: 'shell.openWizard',
category: 'Shell',
name: 'Open wizard',
// testEnabled: () => getCurrentEditor()?.openWizardEnabled(),
onClick: () => getCurrentEditor().openWizard(),
});
const configRegex = /\s*\/\/\s*@ImportExportConfigurator\s*\n\s*\/\/\s*(\{[^\n]+\})\n/;
const requireRegex = /\s*(\/\/\s*@require\s+[^\n]+)\n/g;
const initRegex = /([^\n]+\/\/\s*@init)/g;
</script>
@ -47,8 +38,6 @@
import { registerFileCommands } from '../commands/stdCommands';
import VerticalSplitter from '../elements/VerticalSplitter.svelte';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import AceEditor from '../query/AceEditor.svelte';
import RunnerOutputPane from '../query/RunnerOutputPane.svelte';
import useEditorData from '../query/useEditorData';
@ -60,7 +49,7 @@
import { showSnackbarError } from '../utility/snackbar';
import useEffect from '../utility/useEffect';
import useTimerLabel from '../utility/useTimerLabel';
export let tabid;
const tabVisible: any = getContext('tabVisible');
@ -149,19 +138,6 @@
copyTextToClipboard(resp);
}
// export function openWizardEnabled() {
// return ($editorValue || '').match(configRegex);
// }
export function openWizard() {
const jsonTextMatch = ($editorValue || '').match(configRegex);
if (jsonTextMatch) {
showModal(ImportExportModal, { initialValues: JSON.parse(jsonTextMatch[1]) });
} else {
showSnackbarError('No wizard info found');
}
}
function getActiveScript() {
const selectedText = domEditor.getEditor().getSelectedText();
const editorText = $editorValue;
@ -208,7 +184,6 @@
return [
{ command: 'shell.execute' },
{ command: 'shell.kill' },
{ command: 'shell.openWizard' },
{ divider: true },
{ command: 'shell.toggleComment' },
{ divider: true },

17
packages/web/src/utility/exportFileTools.ts
View File

@ -42,8 +42,8 @@ export async function exportSqlDump(outputFile, connection, databaseName, pureFi
onOpenResult:
pureFileName && !getElectron()
? () => {
downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql');
}
downloadFromApi(`uploads/get?file=${pureFileName}`, 'file.sql');
}
: null,
openResultLabel: 'Download SQL file',
});
@ -226,17 +226,18 @@ export async function downloadFromApi(route: string, donloadName: string) {
method: 'GET',
headers: resolveApiHeaders(),
})
.then((res) => res.blob())
.then((blob) => {
.then(res => res.blob())
.then(blob => {
const objUrl = URL.createObjectURL(blob);
const a = document.createElement("a");
const a = document.createElement('a');
document.body.appendChild(a);
a.download = donloadName;
a.href = objUrl;
a.click();
a.remove();
setTimeout(() => {
URL.revokeObjectURL(objUrl)
})
})
URL.revokeObjectURL(objUrl);
});
});
}

15
packages/web/src/utility/importExportTools.ts Normal file
View File

@ -0,0 +1,15 @@
import openNewTab from './openNewTab';
export function openImportExportTab(editorProps, additionalProps = {}) {
openNewTab(
{
tabComponent: 'ImportExportTab',
title: 'Import/Export',
icon: 'img export',
props: additionalProps,
},
{
editor: editorProps,
}
);
}

35
packages/web/src/utility/openElectronFile.ts
View File

@ -1,17 +1,17 @@
import { showModal } from '../modals/modalTools';
import { get } from 'svelte/store';
import newQuery from '../query/newQuery';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import getElectron from './getElectron';
import { currentDatabase, extensions, getCurrentDatabase } from '../stores';
import { getUploadListener } from './uploadFiles';
import {getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools';
import { getConnectionLabel, getDatabaseFileLabel } from 'dbgate-tools';
import { apiCall } from './api';
import openNewTab from './openNewTab';
import { openJsonDocument } from '../tabs/JsonTab.svelte';
import { SAVED_FILE_HANDLERS } from '../appobj/SavedFileAppObject.svelte';
import _ from 'lodash';
import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
import { openImportExportTab } from './importExportTools';
export function canOpenByElectron(file, extensions) {
if (!file) return false;
@ -178,17 +178,30 @@ export function openElectronFileCore(filePath, extensions) {
shortName: parsed.name,
});
} else {
showModal(ImportExportModal, {
openedFile: {
filePath,
storageType: format.storageType,
shortName: parsed.name,
},
importToCurrentTarget: true,
initialValues: {
openImportExportTab(
{
sourceStorageType: format.storageType,
},
});
{
openedFile: {
filePath,
storageType: format.storageType,
shortName: parsed.name,
},
}
);
// showModal(ImportExportModal, {
// openedFile: {
// filePath,
// storageType: format.storageType,
// shortName: parsed.name,
// },
// importToCurrentTarget: true,
// initialValues: {
// sourceStorageType: format.storageType,
// },
// });
}
}
}

22
packages/web/src/utility/uploadFiles.ts
View File

@ -5,9 +5,9 @@ import getElectron from './getElectron';
import resolveApi, { resolveApiHeaders } from './resolveApi';
import { findFileFormat } from '../plugins/fileformats';
import { showModal } from '../modals/modalTools';
import ImportExportModal from '../modals/ImportExportModal.svelte';
import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
import openNewTab from './openNewTab';
import { openImportExportTab } from './importExportTools';
let uploadListener;
@ -79,13 +79,23 @@ export default function uploadFiles(files) {
uploadListener(fileData);
} else {
if (findFileFormat(ext, fileData.storageType)) {
showModal(ImportExportModal, {
uploadedFile: fileData,
importToCurrentTarget: true,
initialValues: {
openImportExportTab(
{
sourceStorageType: fileData.storageType,
},
});
{
uploadedFile: fileData,
importToCurrentTarget: true,
}
);
// showModal(ImportExportModal, {
// uploadedFile: fileData,
// importToCurrentTarget: true,
// initialValues: {
// sourceStorageType: fileData.storageType,
// },
// });
}
}

4
packages/web/src/widgets/SavedFilesList.svelte
View File

@ -20,6 +20,7 @@
const queryFiles = useFiles({ folder: 'query' });
const sqliteFiles = useFiles({ folder: 'sqlite' });
const diagramFiles = useFiles({ folder: 'diagrams' });
const jobFiles = useFiles({ folder: 'jobs' });
const perspectiveFiles = useFiles({ folder: 'perspectives' });
$: files = [
@ -31,11 +32,12 @@
...($sqliteFiles || []),
...($diagramFiles || []),
...($perspectiveFiles || []),
...($jobFiles || []),
];
function handleRefreshFiles() {
apiCall('files/refresh', {
folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives'],
folders: ['sql', 'shell', 'markdown', 'charts', 'query', 'sqlite', 'diagrams', 'perspectives', 'jobs'],
});
}
</script>

5
plugins/dbgate-plugin-csv/package.json
View File

@ -32,10 +32,11 @@
"prepublishOnly": "yarn build"
},
"devDependencies": {
"csv": "^5.3.2",
"csv": "^6.3.10",
"dbgate-plugin-tools": "^1.0.7",
"line-reader": "^0.4.0",
"lodash": "^4.17.21",
"webpack": "^5.91.0",
"webpack-cli": "^5.1.4"
}
}
}

42
plugins/dbgate-plugin-csv/src/backend/reader.js
View File

@ -2,8 +2,32 @@ const zipObject = require('lodash/zipObject');
const csv = require('csv');
const fs = require('fs');
const stream = require('stream');
const lineReader = require('line-reader');
let dbgateApi;
function readFirstLine(file) {
return new Promise((resolve, reject) => {
lineReader.open(file, (err, reader) => {
if (err) {
reject(err);
return;
}
if (reader.hasNextLine()) {
reader.nextLine((err, line) => {
if (err) {
reader.close(() => reject(err)); // Ensure reader is closed on error
return;
}
reader.close(() => resolve(line)); // Ensure reader is closed after reading
});
} else {
reader.close(() => resolve(null)); // Properly close if no lines are present
}
});
});
}
class CsvPrepareStream extends stream.Transform {
constructor({ header }) {
super({ objectMode: true });
@ -46,13 +70,29 @@ class CsvPrepareStream extends stream.Transform {
async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, limitRows = undefined }) {
console.log(`Reading file ${fileName}`);
const downloadedFile = await dbgateApi.download(fileName);
if (!delimiter) {
// auto detect delimiter
// read first line from downloadedFile
const firstLine = await readFirstLine(downloadedFile);
if (firstLine) {
const delimiterCounts = {
',': firstLine.replace(/[^,]/g, '').length,
';': firstLine.replace(/[^;]/g, '').length,
'|': firstLine.replace(/[^|]/g, '').length,
};
delimiter = Object.keys(delimiterCounts).reduce((a, b) => (delimiterCounts[a] > delimiterCounts[b] ? a : b), ',');
}
}
const csvStream = csv.parse({
// @ts-ignore
delimiter,
skip_lines_with_error: true,
to_line: limitRows ? limitRows + 1 : undefined,
ltrim: true,
});
const downloadedFile = await dbgateApi.download(fileName);
const fileStream = fs.createReadStream(downloadedFile, encoding);
const csvPrepare = new CsvPrepareStream({ header });
fileStream.pipe(csvStream);

1
plugins/dbgate-plugin-csv/src/frontend/index.js
View File

@ -17,6 +17,7 @@ const fileFormat = {
name: 'delimiter',
label: 'Delimiter',
options: [
{ name: 'Auto-detect', value: '' },
{ name: 'Comma (,)', value: ',' },
{ name: 'Semicolon (;)', value: ';' },
{ name: 'Tab', value: '\t' },

70
yarn.lock
View File

@ -3116,30 +3116,30 @@ cssstyle@^2.3.0:
dependencies:
cssom "~0.3.6"
csv-generate@^3.4.3:
version "3.4.3"
resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.4.3.tgz#bc42d943b45aea52afa896874291da4b9108ffff"
integrity sha512-w/T+rqR0vwvHqWs/1ZyMDWtHHSJaN06klRqJXBEpDJaM/+dZkso0OKh1VcuuYvK3XM53KysVNq8Ko/epCK8wOw==
csv-generate@^4.4.1:
version "4.4.1"
resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-4.4.1.tgz#729781ace8d1b92f6bfb407d1ab9548728c55681"
integrity sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg==
csv-parse@^4.16.3:
version "4.16.3"
resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.16.3.tgz#7ca624d517212ebc520a36873c3478fa66efbaf7"
integrity sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==
csv-parse@^5.5.6:
version "5.5.6"
resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a"
integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==
csv-stringify@^5.6.5:
version "5.6.5"
resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.6.5.tgz#c6d74badda4b49a79bf4e72f91cce1e33b94de00"
integrity sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==
csv-stringify@^6.5.1:
version "6.5.1"
resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.1.tgz#a31837dd35e34787e3c248159c982a21af964d94"
integrity sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ==
csv@^5.3.2:
version "5.5.3"
resolved "https://registry.yarnpkg.com/csv/-/csv-5.5.3.tgz#cd26c1e45eae00ce6a9b7b27dcb94955ec95207d"
integrity sha512-QTaY0XjjhTQOdguARF0lGKm5/mEq9PD9/VhZZegHDIBq2tQwgNpHc3dneD4mGo2iJs+fTKv5Bp0fZ+BRuY3Z0g==
csv@^6.3.10:
version "6.3.10"
resolved "https://registry.yarnpkg.com/csv/-/csv-6.3.10.tgz#960a3a9cef08573ecca2d80ddb71152aca383088"
integrity sha512-5NYZG4AN2ZUthmNxIudgBEdMPUnbQHu9V4QTzBPqQzUP3KQsFiJo+8HQ0+oVxj1PomIT1/f67VI1QH/hsrZLKA==
dependencies:
csv-generate "^3.4.3"
csv-parse "^4.16.3"
csv-stringify "^5.6.5"
stream-transform "^2.1.3"
csv-generate "^4.4.1"
csv-parse "^5.5.6"
csv-stringify "^6.5.1"
stream-transform "^3.3.2"
dashdash@^1.12.0:
version "1.14.1"
@ -7456,11 +7456,6 @@ mixin-deep@^1.2.0:
for-in "^1.0.2"
is-extendable "^1.0.1"
mixme@^0.5.1:
version "0.5.10"
resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.5.10.tgz#d653b2984b75d9018828f1ea333e51717ead5f51"
integrity sha512-5H76ANWinB1H3twpJ6JY8uvAtpmFvHNArpilJAjXRKXSDDLPIMoZArw5SH0q9z+lLs8IrMw7Q2VWpWimFKFT1Q==
mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3:
version "0.5.3"
resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
@ -9750,12 +9745,22 @@ stoppable@^1.1.0:
resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b"
integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==
stream-transform@^2.1.3:
version "2.1.3"
resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.1.3.tgz#a1c3ecd72ddbf500aa8d342b0b9df38f5aa598e3"
integrity sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ==
stream-chain@^2.2.5:
version "2.2.5"
resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09"
integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==
stream-json@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c"
integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==
dependencies:
mixme "^0.5.1"
stream-chain "^2.2.5"
stream-transform@^3.3.2:
version "3.3.2"
resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-3.3.2.tgz#398c67b2f3b6ed5d04ceadde9e412bda8416c8ab"
integrity sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ==
streamsearch@^1.1.0:
version "1.1.0"
@ -10182,6 +10187,11 @@ tmp@^0.0.33:
dependencies:
os-tmpdir "~1.0.2"
tmp@^0.2.3:
version "0.2.3"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae"
integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==
tmpl@1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc"