From 8b929f40d2fd8685e00f7ab6bb86a9b7a9a91651 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Feb 2022 22:05:41 +0100 Subject: [PATCH] fixed jsonl problems, support jsonl without header --- packages/api/src/controllers/jsldata.js | 12 ++++++- .../api/src/utility/JsonLinesDatastore.js | 33 +++++++++++++++---- packages/datalib/src/GridDisplay.ts | 9 ++--- packages/datalib/src/JslGridDisplay.ts | 5 ++- packages/web/src/datagrid/JslDataGrid.svelte | 2 +- .../web/src/tabs/JsonLinesEditorTab.svelte | 2 ++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 2ba1eca4..280e57f5 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -1,6 +1,7 @@ const fs = require('fs'); const lineReader = require('line-reader'); const _ = require('lodash'); +const { __ } = require('lodash/fp'); const DatastoreProxy = require('../utility/DatastoreProxy'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const getJslFileName = require('../utility/getJslFileName'); @@ -111,7 +112,16 @@ module.exports = { async getInfo({ jslid }) { const file = getJslFileName(jslid); const firstLine = await readFirstLine(file); - if (firstLine) return JSON.parse(firstLine); + if (firstLine) { + const parsed = JSON.parse(firstLine); + if (parsed.__isStreamHeader) { + return parsed; + } + return { + __isStreamHeader: true, + __isDynamicStructure: true, + }; + } return null; }, diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index 37a7220f..904ff9ed 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -27,6 +27,7 @@ class JsonLinesDatastore { this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; + // this.firstRowToBeReturned = null; this.notifyChangedCallback = null; this.currentFilter = null; } @@ -37,6 +38,7 @@ class JsonLinesDatastore { this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; + // this.firstRowToBeReturned = null; this.currentFilter = null; reader.close(() => {}); } @@ -61,6 +63,11 @@ class JsonLinesDatastore { } async _readLine(parse) { + // if (this.firstRowToBeReturned) { + // const res = this.firstRowToBeReturned; + // this.firstRowToBeReturned = null; + // return res; + // } for (;;) { const line = await fetchNextLineFromReader(this.reader); if (!line) { @@ -70,7 +77,11 @@ class JsonLinesDatastore { if (!this.readedSchemaRow) { this.readedSchemaRow = true; - return true; + const parsedLine = JSON.parse(line); + if (parsedLine.__isStreamHeader) { + // skip to next line + continue; + } } if (this.currentFilter) { const parsedLine = JSON.parse(line); @@ -79,6 +90,7 @@ class JsonLinesDatastore { return parse ? parsedLine : true; } } else { + console.log('NO !!!'); this.readedDataRowCount += 1; return parse ? JSON.parse(line) : true; } @@ -130,11 +142,21 @@ class JsonLinesDatastore { this.reader = reader; this.currentFilter = filter; } - if (!this.readedSchemaRow) { - await this._readLine(false); // skip structure - } + // if (!this.readedSchemaRow) { + // const line = await this._readLine(true); // skip structure + // if (!line.__isStreamHeader) { + // // line contains data + // this.firstRowToBeReturned = line; + // } + // } while (this.readedDataRowCount < offset) { - await this._readLine(false); + const line = await this._readLine(false); + if (line == null) break; + // if (this.firstRowToBeReturned) { + // this.firstRowToBeReturned = null; + // } else { + // await this._readLine(false); + // } } } @@ -148,7 +170,6 @@ class JsonLinesDatastore { res.push(line); } }); - // console.log('RETURN', res.length); return res; } } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 29604143..7e58fad1 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -90,6 +90,7 @@ export abstract class GridDisplay { isLoadedCorrectly = true; supportsReload = false; isDynamicStructure = false; + filterTypeOverride = null; setColumnVisibility(uniquePath: string[], isVisible: boolean) { const uniqueName = uniquePath.join('.'); @@ -622,16 +623,16 @@ export abstract class GridDisplay { if (!filters) return null; const conditions = []; for (const name in filters) { - const column = this.columns.find(x => (x.columnName = name)); - if (!column) continue; - const filterType = getFilterType(column.dataType); + const column = this.isDynamicStructure ? null : this.columns.find(x => x.columnName == name); + if (!this.isDynamicStructure && !column) continue; + const filterType = this.isDynamicStructure ? this.filterTypeOverride ?? 'mongo' : getFilterType(column.dataType); try { const condition = parseFilter(filters[name], filterType); const replaced = _.cloneDeepWith(condition, (expr: Expression) => { if (expr.exprType == 'placeholder') return { exprType: 'column', - columnName: column.columnName, + columnName: this.isDynamicStructure ? name : column.columnName, }; }); conditions.push(replaced); diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index b371277f..4b3bbef1 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -12,11 +12,14 @@ export class JslGridDisplay extends GridDisplay { setConfig: ChangeConfigFunc, cache: GridCache, setCache: ChangeCacheFunc, - rows: any + rows: any, + isDynamicStructure: boolean ) { super(config, setConfig, cache, setCache, null); this.filterable = true; + this.isDynamicStructure = isDynamicStructure; + if (isDynamicStructure) this.filterTypeOverride = 'string'; if (structure?.columns) { this.columns = _.uniqBy( diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index a6134f74..73571011 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -16,7 +16,7 @@ const config = writable(createGridConfig()); const cache = writable(createGridCache()); - $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows); + $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows, $info?.__isDynamicStructure); {#key jslid} diff --git a/packages/web/src/tabs/JsonLinesEditorTab.svelte b/packages/web/src/tabs/JsonLinesEditorTab.svelte index 81512bd1..3bf4b3bf 100644 --- a/packages/web/src/tabs/JsonLinesEditorTab.svelte +++ b/packages/web/src/tabs/JsonLinesEditorTab.svelte @@ -30,6 +30,7 @@ category: 'JSON Lines editor', name: 'Preview', icon: 'icon preview', + keyText: 'F5', testEnabled: () => getCurrentEditor() != null, onClick: () => getCurrentEditor().preview(), }); @@ -99,6 +100,7 @@ title: 'Preview #', icon: 'img archive', tabComponent: 'ArchiveFileTab', + forceNewTab: true, props: { jslid, },