fixed jsonl problems, support jsonl without header

packages/api/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -111,7 +112,16 @@ module.exports = {
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
+    if (firstLine) {
+      const parsed = JSON.parse(firstLine);
+      if (parsed.__isStreamHeader) {
+        return parsed;
+      }
+      return {
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      };
+    }
     return null;
   },
 

packages/api/src/utility/JsonLinesDatastore.js

@@ -27,6 +27,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
   }
@@ -37,6 +38,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     reader.close(() => {});
   }
@@ -61,6 +63,11 @@ class JsonLinesDatastore {
   }
 
   async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
     for (;;) {
       const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
@@ -70,7 +77,11 @@ class JsonLinesDatastore {
 
       if (!this.readedSchemaRow) {
         this.readedSchemaRow = true;
-        return true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
       }
       if (this.currentFilter) {
         const parsedLine = JSON.parse(line);
@@ -79,6 +90,7 @@ class JsonLinesDatastore {
           return parse ? parsedLine : true;
         }
       } else {
+        console.log('NO !!!');
         this.readedDataRowCount += 1;
         return parse ? JSON.parse(line) : true;
       }
@@ -130,11 +142,21 @@ class JsonLinesDatastore {
       this.reader = reader;
       this.currentFilter = filter;
     }
-    if (!this.readedSchemaRow) {
-      await this._readLine(false); // skip structure
-    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
     while (this.readedDataRowCount < offset) {
-      await this._readLine(false);
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
     }
   }
 
@@ -148,7 +170,6 @@ class JsonLinesDatastore {
         res.push(line);
       }
     });
-    // console.log('RETURN', res.length);
     return res;
   }
 }

packages/datalib/src/GridDisplay.ts

@@ -90,6 +90,7 @@ export abstract class GridDisplay {
   isLoadedCorrectly = true;
   supportsReload = false;
   isDynamicStructure = false;
+  filterTypeOverride = null;
 
   setColumnVisibility(uniquePath: string[], isVisible: boolean) {
     const uniqueName = uniquePath.join('.');
@@ -622,16 +623,16 @@ export abstract class GridDisplay {
     if (!filters) return null;
     const conditions = [];
     for (const name in filters) {
-      const column = this.columns.find(x => (x.columnName = name));
-      if (!column) continue;
-      const filterType = getFilterType(column.dataType);
+      const column = this.isDynamicStructure ? null : this.columns.find(x => x.columnName == name);
+      if (!this.isDynamicStructure && !column) continue;
+      const filterType = this.isDynamicStructure ? this.filterTypeOverride ?? 'mongo' : getFilterType(column.dataType);
       try {
         const condition = parseFilter(filters[name], filterType);
         const replaced = _.cloneDeepWith(condition, (expr: Expression) => {
           if (expr.exprType == 'placeholder')
             return {
               exprType: 'column',
-              columnName: column.columnName,
+              columnName: this.isDynamicStructure ? name : column.columnName,
             };
         });
         conditions.push(replaced);

packages/datalib/src/JslGridDisplay.ts

@@ -12,11 +12,14 @@ export class JslGridDisplay extends GridDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
-    rows: any
+    rows: any,
+    isDynamicStructure: boolean
   ) {
     super(config, setConfig, cache, setCache, null);
 
     this.filterable = true;
+    this.isDynamicStructure = isDynamicStructure;
+    if (isDynamicStructure) this.filterTypeOverride = 'string';
 
     if (structure?.columns) {
       this.columns = _.uniqBy(

packages/web/src/datagrid/JslDataGrid.svelte

@@ -16,7 +16,7 @@
   const config = writable(createGridConfig());
   const cache = writable(createGridCache());
 
-  $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows);
+  $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows, $info?.__isDynamicStructure);
 </script>
 
 {#key jslid}

packages/web/src/tabs/JsonLinesEditorTab.svelte

@@ -30,6 +30,7 @@
     category: 'JSON Lines editor',
     name: 'Preview',
     icon: 'icon preview',
+    keyText: 'F5',
     testEnabled: () => getCurrentEditor() != null,
     onClick: () => getCurrentEditor().preview(),
   });
@@ -99,6 +100,7 @@
       title: 'Preview #',
       icon: 'img archive',
       tabComponent: 'ArchiveFileTab',
+      forceNewTab: true,
       props: {
         jslid,
       },