diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 2873ce9d..1bf19df0 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -1,6 +1,6 @@ const fs = require('fs'); const lineReader = require('line-reader'); -const { off } = require('process'); +const _ = require('lodash'); const DatastoreProxy = require('../utility/DatastoreProxy'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const getJslFileName = require('../utility/getJslFileName'); @@ -112,10 +112,10 @@ module.exports = { return null; }, - getRows_meta: 'get', - async getRows({ jslid, offset, limit }) { + getRows_meta: 'post', + async getRows({ jslid, offset, limit, filters }) { const datastore = await this.ensureDatastore(jslid); - return datastore.getRows(offset, limit); + return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters); }, getStats_meta: 'get', diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index f3fa048b..b9965ae8 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -1,6 +1,25 @@ const lineReader = require('line-reader'); const AsyncLock = require('async-lock'); const lock = new AsyncLock(); +const stableStringify = require('json-stable-stringify'); +const { evaluateCondition } = require('dbgate-sqltree'); + +async function fetchNextLine(reader) { + return new Promise((resolve, reject) => { + if (!reader.hasNextLine()) { + resolve(null); + return; + } + + reader.nextLine((err, line) => { + if (err) { + reject(err); + } else { + resolve(line); + } + }); + }); +} class JsonLinesDatastore { constructor(file) { @@ -9,6 +28,7 @@ class JsonLinesDatastore { this.readedDataRowCount = 0; this.readedSchemaRow = false; this.notifyChangedCallback = null; + this.currentFilter = null; } _closeReader() { @@ -17,6 +37,7 @@ class JsonLinesDatastore { this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; + this.currentFilter = null; reader.close(() => {}); } @@ -39,46 +60,92 @@ class JsonLinesDatastore { ); } - _readLine() { - return new Promise((resolve, reject) => { - const reader = this.reader; - if (!reader.hasNextLine()) { - resolve(null); - return; + async _readLine(parse) { + for (;;) { + const line = await fetchNextLine(this.reader); + if (!line) { + // EOF + return null; } - reader.nextLine((err, line) => { - if (this.readedSchemaRow) this.readedDataRowCount += 1; - else this.readedSchemaRow = true; - if (err) reject(err); - resolve(line); - }); - }); + + if (!this.readedSchemaRow) { + this.readedSchemaRow = true; + return true; + } + if (this.currentFilter) { + const parsedLine = JSON.parse(line); + if (evaluateCondition(this.currentFilter, parsedLine)) { + this.readedDataRowCount += 1; + return parse ? parsedLine : true; + } + } else { + this.readedDataRowCount += 1; + return parse ? JSON.parse(line) : true; + } + } + + // return new Promise((resolve, reject) => { + // const reader = this.reader; + // if (!reader.hasNextLine()) { + // resolve(null); + // return; + // } + + // reader.nextLine((err, line) => { + // if (err) { + // reject(err); + // return; + // } + // if (!this.readedSchemaRow) { + // this.readedSchemaRow = true; + // resolve(true); + // return; + // } + // if (this.currentFilter) { + // const parsedLine = JSON.parse(line); + // if (evaluateCondition(this.currentFilter, parsedLine)) { + // console.log('TRUE'); + // resolve(parse ? parsedLine : true); + // this.readedDataRowCount += 1; + // return; + // } else { + // console.log('FALSE'); + // // skip row + // return; + // } + // } + + // this.readedDataRowCount += 1; + // resolve(parse ? JSON.parse(line) : true); + // }); + // }); } - async _ensureReader(offset) { - if (this.readedDataRowCount > offset) { + async _ensureReader(offset, filter) { + if (this.readedDataRowCount > offset || stableStringify(filter) != stableStringify(this.currentFilter)) { this._closeReader(); } if (!this.reader) { const reader = await this._openReader(); this.reader = reader; + this.currentFilter = filter; } if (!this.readedSchemaRow) { - await this._readLine(); // skip structure + await this._readLine(false); // skip structure } while (this.readedDataRowCount < offset) { - await this._readLine(); + await this._readLine(false); } } - async getRows(offset, limit) { + async getRows(offset, limit, filter) { const res = []; await lock.acquire('reader', async () => { - await this._ensureReader(offset); + await this._ensureReader(offset, filter); for (let i = 0; i < limit; i += 1) { - const line = await this._readLine(); + const line = await this._readLine(true); if (line == null) break; - res.push(JSON.parse(line)); + res.push(line); } }); // console.log('RETURN', res.length); diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 6aada156..d3ffc940 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -4,7 +4,7 @@ import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, D import { parseFilter, getFilterType } from 'dbgate-filterparser'; import { filterName } from './filterName'; import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet'; -import { Expression, Select, treeToSql, dumpSqlSelect } from 'dbgate-sqltree'; +import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree'; import { isTypeLogical } from 'dbgate-tools'; export interface DisplayColumn { @@ -487,4 +487,33 @@ export abstract class GridDisplay { const sql = treeToSql(this.driver, select, dumpSqlSelect); return sql; } + + compileFilters(): Condition { + const filters = this.config && this.config.filters; + if (!filters) return null; + const conditions = []; + for (const name in filters) { + const column = this.columns.find((x) => (x.columnName = name)); + if (!column) continue; + const filterType = getFilterType(column.dataType); + try { + const condition = parseFilter(filters[name], filterType); + const replaced = _.cloneDeepWith(condition, (expr: Expression) => { + if (expr.exprType == 'placeholder') + return { + exprType: 'column', + columnName: column.columnName, + }; + }); + conditions.push(replaced); + } catch (err) { + // filter parse error - ignore filter + } + } + if (conditions.length == 0) return null; + return { + conditionType: 'and', + conditions, + }; + } } diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index c7cae160..1993cc4e 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -13,6 +13,8 @@ export class JslGridDisplay extends GridDisplay { ) { super(config, setConfig, cache, setCache, null); + this.filterable = true; + this.columns = columns .map((col) => ({ columnName: col.columnName, diff --git a/packages/sqltree/src/evaluateCondition.ts b/packages/sqltree/src/evaluateCondition.ts new file mode 100644 index 00000000..ee03fbeb --- /dev/null +++ b/packages/sqltree/src/evaluateCondition.ts @@ -0,0 +1,62 @@ +import { SqlDumper } from 'dbgate-types'; +import _ from 'lodash'; +import { Condition, BinaryCondition } from './types'; +import { dumpSqlExpression } from './dumpSqlExpression'; +import { link } from 'fs'; +import { evaluateExpression } from './evaluateExpression'; +import { cond } from 'lodash'; + +function isEmpty(value) { + if (value == null) return true; + return value.toString().trim() == ''; +} + +function isLike(value, test) { + if (!value) return false; + if (!test) return false; + const regex = new RegExp(`^${_.escapeRegExp(test).replace(/%/g, '.*')}$`, 'i'); + const res = !!value.toString().match(regex); + return res; +} + +export function evaluateCondition(condition: Condition, values) { + switch (condition.conditionType) { + case 'binary': + const left = evaluateExpression(condition.left, values); + const right = evaluateExpression(condition.right, values); + switch (condition.operator) { + case '=': + return left == right; + case '!=': + return left != right; + case '<=': + return left <= right; + case '>=': + return left >= right; + case '<': + return left < right; + case '>': + return left > right; + } + break; + case 'isNull': + return evaluateExpression(condition.expr, values) == null; + case 'isNotNull': + return evaluateExpression(condition.expr, values) != null; + case 'isEmpty': + return isEmpty(evaluateExpression(condition.expr, values)); + case 'isNotEmpty': + return !isEmpty(evaluateExpression(condition.expr, values)); + case 'and': + return condition.conditions.every((cond) => evaluateCondition(cond, values)); + case 'or': + return condition.conditions.some((cond) => evaluateCondition(cond, values)); + case 'like': + return isLike(evaluateExpression(condition.left, values), evaluateExpression(condition.right, values)); + break; + case 'notLike': + return !isLike(evaluateExpression(condition.left, values), evaluateExpression(condition.right, values)); + case 'not': + return !evaluateCondition(condition.condition, values); + } +} diff --git a/packages/sqltree/src/evaluateExpression.ts b/packages/sqltree/src/evaluateExpression.ts new file mode 100644 index 00000000..5be138ec --- /dev/null +++ b/packages/sqltree/src/evaluateExpression.ts @@ -0,0 +1,26 @@ +import _ from 'lodash'; +import { SqlDumper } from 'dbgate-types'; +import { Expression, ColumnRefExpression } from './types'; +import { dumpSqlSourceRef } from './dumpSqlSource'; + +export function evaluateExpression(expr: Expression, values) { + switch (expr.exprType) { + case 'column': + return values[expr.columnName]; + + case 'placeholder': + return values.__placeholder; + + case 'value': + return expr.value; + + case 'raw': + return expr.sql; + + case 'call': + return null; + + case 'transform': + return null; + } +} diff --git a/packages/sqltree/src/index.ts b/packages/sqltree/src/index.ts index d40c0596..0fd29aef 100644 --- a/packages/sqltree/src/index.ts +++ b/packages/sqltree/src/index.ts @@ -3,3 +3,5 @@ export * from './dumpSqlCommand'; export * from './utility'; export * from './dumpSqlSource'; export * from './dumpSqlCondition'; +export * from './evaluateCondition'; +export * from './evaluateExpression'; diff --git a/packages/web/src/datagrid/JslDataGridCore.js b/packages/web/src/datagrid/JslDataGridCore.js index 88351a87..4c09f3f8 100644 --- a/packages/web/src/datagrid/JslDataGridCore.js +++ b/packages/web/src/datagrid/JslDataGridCore.js @@ -8,17 +8,15 @@ import LoadingDataGridCore from './LoadingDataGridCore'; import RowsArrayGrider from './RowsArrayGrider'; async function loadDataPage(props, offset, limit) { - const { jslid } = props; + const { jslid, display } = props; - const response = await axios.request({ - url: 'jsldata/get-rows', - method: 'get', - params: { - jslid, - offset, - limit, - }, + const response = await axios.post('jsldata/get-rows', { + jslid, + offset, + limit, + filters: display ? display.compileFilters() : null, }); + return response.data; } @@ -63,10 +61,13 @@ export default function JslDataGridCore(props) { showModal((modalState) => ); } - const handleJslDataStats = React.useCallback((stats) => { - if (stats.changeIndex < changeIndex) return; - setChangeIndex(stats.changeIndex); - }, [changeIndex]); + const handleJslDataStats = React.useCallback( + (stats) => { + if (stats.changeIndex < changeIndex) return; + setChangeIndex(stats.changeIndex); + }, + [changeIndex] + ); React.useEffect(() => { if (jslid && socket) {