jsl data filter (archive, query result)

packages/api/src/controllers/jsldata.js

@@ -1,6 +1,6 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
-const { off } = require('process');
+const _ = require('lodash');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -112,10 +112,10 @@ module.exports = {
     return null;
   },
 
-  getRows_meta: 'get',
+  getRows_meta: 'post',
-  async getRows({ jslid, offset, limit }) {
+  async getRows({ jslid, offset, limit, filters }) {
     const datastore = await this.ensureDatastore(jslid);
-    return datastore.getRows(offset, limit);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
   getStats_meta: 'get',

packages/api/src/utility/JsonLinesDatastore.js

@@ -1,6 +1,25 @@
 const lineReader = require('line-reader');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
+const stableStringify = require('json-stable-stringify');
+const { evaluateCondition } = require('dbgate-sqltree');
+
+async function fetchNextLine(reader) {
+  return new Promise((resolve, reject) => {
+    if (!reader.hasNextLine()) {
+      resolve(null);
+      return;
+    }
+
+    reader.nextLine((err, line) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(line);
+      }
+    });
+  });
+}
 
 class JsonLinesDatastore {
   constructor(file) {
@@ -9,6 +28,7 @@ class JsonLinesDatastore {
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
     this.notifyChangedCallback = null;
+    this.currentFilter = null;
   }
 
   _closeReader() {
@@ -17,6 +37,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    this.currentFilter = null;
     reader.close(() => {});
   }
 
@@ -39,46 +60,92 @@ class JsonLinesDatastore {
     );
   }
 
-  _readLine() {
+  async _readLine(parse) {
-    return new Promise((resolve, reject) => {
+    for (;;) {
-      const reader = this.reader;
+      const line = await fetchNextLine(this.reader);
-      if (!reader.hasNextLine()) {
+      if (!line) {
-        resolve(null);
+        // EOF
-        return;
+        return null;
       }
-      reader.nextLine((err, line) => {
+
-        if (this.readedSchemaRow) this.readedDataRowCount += 1;
+      if (!this.readedSchemaRow) {
-        else this.readedSchemaRow = true;
+        this.readedSchemaRow = true;
-        if (err) reject(err);
+        return true;
-        resolve(line);
+      }
-      });
+      if (this.currentFilter) {
-    });
+        const parsedLine = JSON.parse(line);
+        if (evaluateCondition(this.currentFilter, parsedLine)) {
+          this.readedDataRowCount += 1;
+          return parse ? parsedLine : true;
+        }
+      } else {
+        this.readedDataRowCount += 1;
+        return parse ? JSON.parse(line) : true;
+      }
+    }
+
+    // return new Promise((resolve, reject) => {
+    //   const reader = this.reader;
+    //   if (!reader.hasNextLine()) {
+    //     resolve(null);
+    //     return;
+    //   }
+
+    //   reader.nextLine((err, line) => {
+    //     if (err) {
+    //       reject(err);
+    //       return;
+    //     }
+    //     if (!this.readedSchemaRow) {
+    //       this.readedSchemaRow = true;
+    //       resolve(true);
+    //       return;
+    //     }
+    //     if (this.currentFilter) {
+    //       const parsedLine = JSON.parse(line);
+    //       if (evaluateCondition(this.currentFilter, parsedLine)) {
+    //         console.log('TRUE');
+    //         resolve(parse ? parsedLine : true);
+    //         this.readedDataRowCount += 1;
+    //         return;
+    //       } else {
+    //         console.log('FALSE');
+    //         // skip row
+    //         return;
+    //       }
+    //     }
+
+    //     this.readedDataRowCount += 1;
+    //     resolve(parse ? JSON.parse(line) : true);
+    //   });
+    // });
   }
 
-  async _ensureReader(offset) {
+  async _ensureReader(offset, filter) {
-    if (this.readedDataRowCount > offset) {
+    if (this.readedDataRowCount > offset || stableStringify(filter) != stableStringify(this.currentFilter)) {
       this._closeReader();
     }
     if (!this.reader) {
       const reader = await this._openReader();
       this.reader = reader;
+      this.currentFilter = filter;
     }
     if (!this.readedSchemaRow) {
-      await this._readLine(); // skip structure
+      await this._readLine(false); // skip structure
     }
     while (this.readedDataRowCount < offset) {
-      await this._readLine();
+      await this._readLine(false);
     }
   }
 
-  async getRows(offset, limit) {
+  async getRows(offset, limit, filter) {
     const res = [];
     await lock.acquire('reader', async () => {
-      await this._ensureReader(offset);
+      await this._ensureReader(offset, filter);
       for (let i = 0; i < limit; i += 1) {
-        const line = await this._readLine();
+        const line = await this._readLine(true);
         if (line == null) break;
-        res.push(JSON.parse(line));
+        res.push(line);
       }
     });
     // console.log('RETURN', res.length);

packages/datalib/src/GridDisplay.ts

@@ -4,7 +4,7 @@ import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, D
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
 import { filterName } from './filterName';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { Expression, Select, treeToSql, dumpSqlSelect } from 'dbgate-sqltree';
+import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
 import { isTypeLogical } from 'dbgate-tools';
 
 export interface DisplayColumn {
@@ -487,4 +487,33 @@ export abstract class GridDisplay {
     const sql = treeToSql(this.driver, select, dumpSqlSelect);
     return sql;
   }
+
+  compileFilters(): Condition {
+    const filters = this.config && this.config.filters;
+    if (!filters) return null;
+    const conditions = [];
+    for (const name in filters) {
+      const column = this.columns.find((x) => (x.columnName = name));
+      if (!column) continue;
+      const filterType = getFilterType(column.dataType);
+      try {
+        const condition = parseFilter(filters[name], filterType);
+        const replaced = _.cloneDeepWith(condition, (expr: Expression) => {
+          if (expr.exprType == 'placeholder')
+            return {
+              exprType: 'column',
+              columnName: column.columnName,
+            };
+        });
+        conditions.push(replaced);
+      } catch (err) {
+        // filter parse error - ignore filter
+      }
+    }
+    if (conditions.length == 0) return null;
+    return {
+      conditionType: 'and',
+      conditions,
+    };
+  }
 }

packages/datalib/src/JslGridDisplay.ts

@@ -13,6 +13,8 @@ export class JslGridDisplay extends GridDisplay {
   ) {
     super(config, setConfig, cache, setCache, null);
 
+    this.filterable = true;
+
     this.columns = columns
       .map((col) => ({
         columnName: col.columnName,

packages/sqltree/src/evaluateCondition.ts

@@ -0,0 +1,62 @@
+import { SqlDumper } from 'dbgate-types';
+import _ from 'lodash';
+import { Condition, BinaryCondition } from './types';
+import { dumpSqlExpression } from './dumpSqlExpression';
+import { link } from 'fs';
+import { evaluateExpression } from './evaluateExpression';
+import { cond } from 'lodash';
+
+function isEmpty(value) {
+  if (value == null) return true;
+  return value.toString().trim() == '';
+}
+
+function isLike(value, test) {
+  if (!value) return false;
+  if (!test) return false;
+  const regex = new RegExp(`^${_.escapeRegExp(test).replace(/%/g, '.*')}$`, 'i');
+  const res = !!value.toString().match(regex);
+  return res;
+}
+
+export function evaluateCondition(condition: Condition, values) {
+  switch (condition.conditionType) {
+    case 'binary':
+      const left = evaluateExpression(condition.left, values);
+      const right = evaluateExpression(condition.right, values);
+      switch (condition.operator) {
+        case '=':
+          return left == right;
+        case '!=':
+          return left != right;
+        case '<=':
+          return left <= right;
+        case '>=':
+          return left >= right;
+        case '<':
+          return left < right;
+        case '>':
+          return left > right;
+      }
+      break;
+    case 'isNull':
+      return evaluateExpression(condition.expr, values) == null;
+    case 'isNotNull':
+      return evaluateExpression(condition.expr, values) != null;
+    case 'isEmpty':
+      return isEmpty(evaluateExpression(condition.expr, values));
+    case 'isNotEmpty':
+      return !isEmpty(evaluateExpression(condition.expr, values));
+    case 'and':
+      return condition.conditions.every((cond) => evaluateCondition(cond, values));
+    case 'or':
+      return condition.conditions.some((cond) => evaluateCondition(cond, values));
+    case 'like':
+      return isLike(evaluateExpression(condition.left, values), evaluateExpression(condition.right, values));
+      break;
+    case 'notLike':
+      return !isLike(evaluateExpression(condition.left, values), evaluateExpression(condition.right, values));
+    case 'not':
+      return !evaluateCondition(condition.condition, values);
+  }
+}

packages/sqltree/src/evaluateExpression.ts

@@ -0,0 +1,26 @@
+import _ from 'lodash';
+import { SqlDumper } from 'dbgate-types';
+import { Expression, ColumnRefExpression } from './types';
+import { dumpSqlSourceRef } from './dumpSqlSource';
+
+export function evaluateExpression(expr: Expression, values) {
+  switch (expr.exprType) {
+    case 'column':
+      return values[expr.columnName];
+
+    case 'placeholder':
+      return values.__placeholder;
+
+    case 'value':
+      return expr.value;
+
+    case 'raw':
+      return expr.sql;
+
+    case 'call':
+      return null;
+
+    case 'transform':
+      return null;
+  }
+}

packages/sqltree/src/index.ts

@@ -3,3 +3,5 @@ export * from './dumpSqlCommand';
 export * from './utility';
 export * from './dumpSqlSource';
 export * from './dumpSqlCondition';
+export * from './evaluateCondition';
+export * from './evaluateExpression';

packages/web/src/datagrid/JslDataGridCore.js

@@ -8,17 +8,15 @@ import LoadingDataGridCore from './LoadingDataGridCore';
 import RowsArrayGrider from './RowsArrayGrider';
 
 async function loadDataPage(props, offset, limit) {
-  const { jslid } = props;
+  const { jslid, display } = props;
 
-  const response = await axios.request({
+  const response = await axios.post('jsldata/get-rows', {
-    url: 'jsldata/get-rows',
+    jslid,
-    method: 'get',
+    offset,
-    params: {
+    limit,
-      jslid,
+    filters: display ? display.compileFilters() : null,
-      offset,
-      limit,
-    },
   });
+
   return response.data;
 }
 
@@ -63,10 +61,13 @@ export default function JslDataGridCore(props) {
     showModal((modalState) => <ImportExportModal modalState={modalState} initialValues={initialValues} />);
   }
 
-  const handleJslDataStats = React.useCallback((stats) => {
+  const handleJslDataStats = React.useCallback(
-    if (stats.changeIndex < changeIndex) return;
+    (stats) => {
-    setChangeIndex(stats.changeIndex);
+      if (stats.changeIndex < changeIndex) return;
-  }, [changeIndex]);
+      setChangeIndex(stats.changeIndex);
+    },
+    [changeIndex]
+  );
 
   React.useEffect(() => {
     if (jslid && socket) {