mongo conditions support

packages/datalib/src/CollectionGridDisplay.ts

@@ -94,6 +94,7 @@ export class CollectionGridDisplay extends GridDisplay {
       uniquePath,
       isStructured: true,
       parentHeaderText: createHeaderText(basePath),
+      filterType: 'mongo',
     };
   }
 }

packages/datalib/src/GridDisplay.ts

@@ -22,6 +22,7 @@ export interface DisplayColumn {
   isChecked?: boolean;
   hintColumnName?: string;
   dataType?: string;
+  filterType?: boolean;
   isStructured?: boolean;
 }
 

packages/filterparser/src/common.ts

@@ -0,0 +1,32 @@
+import P from 'parsimmon';
+
+export const whitespace = P.regexp(/\s*/m);
+
+export function token(parser) {
+  return parser.skip(whitespace);
+}
+
+export function word(str) {
+  return P.string(str).thru(token);
+}
+
+export function interpretEscapes(str) {
+  let escapes = {
+    b: '\b',
+    f: '\f',
+    n: '\n',
+    r: '\r',
+    t: '\t',
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+    let type = escape.charAt(0);
+    let hex = escape.slice(1);
+    if (type === 'u') {
+      return String.fromCharCode(parseInt(hex, 16));
+    }
+    if (escapes.hasOwnProperty(type)) {
+      return escapes[type];
+    }
+    return type;
+  });
+}

packages/filterparser/src/mongoParser.ts

@@ -0,0 +1,121 @@
+import P from 'parsimmon';
+import { interpretEscapes, token, word, whitespace } from './common';
+
+const operatorCondition = operator => value => ({
+  __placeholder__: {
+    [operator]: value,
+  },
+});
+
+const regexCondition = regexString => value => ({
+  __placeholder__: {
+    $regex: regexString.replace('#VALUE#', value),
+    $options: 'i',
+  },
+});
+
+const numberTestCondition = () => value => ({
+  $or: [
+    {
+      __placeholder__: {
+        $regex: `.*${value}.*`,
+        $options: 'i',
+      },
+    },
+    {
+      __placeholder__: value,
+    },
+  ],
+});
+
+const testCondition = (operator, value) => () => ({
+  __placeholder__: {
+    [operator]: value,
+  },
+});
+
+const compoudCondition = conditionType => conditions => {
+  if (conditions.length == 1) return conditions[0];
+  return {
+    [conditionType]: conditions,
+  };
+};
+
+const negateCondition = condition => ({
+  __placeholder__: {
+    $not: condition.__placeholder__,
+  },
+});
+
+const createParser = () => {
+  const langDef = {
+    string1: () =>
+      token(P.regexp(/"((?:\\.|.)*?)"/, 1))
+        .map(interpretEscapes)
+        .desc('string quoted'),
+
+    string2: () =>
+      token(P.regexp(/'((?:\\.|.)*?)'/, 1))
+        .map(interpretEscapes)
+        .desc('string quoted'),
+
+    number: () =>
+      token(P.regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/))
+        .map(Number)
+        .desc('number'),
+
+    noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),
+
+    value: r => P.alt(r.string1, r.string2, r.number, r.noQuotedString),
+    valueTestNum: r => r.number.map(numberTestCondition()),
+    valueTest: r => r.value.map(regexCondition('.*#VALUE#.*')),
+
+    comma: () => word(','),
+    not: () => word('NOT'),
+    notExists: r => r.not.then(r.exists).map(testCondition('$exists', false)),
+    exists: () => word('EXISTS').map(testCondition('$exists', true)),
+    true: () => word('TRUE').map(testCondition('$eq', true)),
+    false: () => word('FALSE').map(testCondition('$eq', false)),
+
+    eq: r => word('=').then(r.value).map(operatorCondition('$eq')),
+    ne: r => word('!=').then(r.value).map(operatorCondition('$ne')),
+    lt: r => word('<').then(r.value).map(operatorCondition('$lt')),
+    gt: r => word('>').then(r.value).map(operatorCondition('$gt')),
+    le: r => word('<=').then(r.value).map(operatorCondition('$lte')),
+    ge: r => word('>=').then(r.value).map(operatorCondition('$gte')),
+    startsWith: r => word('^').then(r.value).map(regexCondition('#VALUE#.*')),
+    endsWith: r => word('$').then(r.value).map(regexCondition('.*#VALUE#')),
+    contains: r => word('+').then(r.value).map(regexCondition('.*#VALUE#.*')),
+    startsWithNot: r => word('!^').then(r.value).map(regexCondition('#VALUE#.*')).map(negateCondition),
+    endsWithNot: r => word('!$').then(r.value).map(regexCondition('.*#VALUE#')).map(negateCondition),
+    containsNot: r => word('~').then(r.value).map(regexCondition('.*#VALUE#.*')).map(negateCondition),
+
+    element: r =>
+      P.alt(
+        r.exists,
+        r.notExists,
+        r.true,
+        r.false,
+        r.eq,
+        r.ne,
+        r.lt,
+        r.gt,
+        r.le,
+        r.ge,
+        r.startsWith,
+        r.endsWith,
+        r.contains,
+        r.startsWithNot,
+        r.endsWithNot,
+        r.containsNot,
+        r.valueTestNum,
+        r.valueTest
+      ).trim(whitespace),
+    factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')),
+    list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')),
+  };
+
+  return P.createLanguage(langDef);
+};
+
+export const mongoParser = createParser();

packages/filterparser/src/parseFilter.ts

@@ -3,37 +3,8 @@ import moment from 'moment';
 import { FilterType } from './types';
 import { Condition } from 'dbgate-sqltree';
 import { TransformType } from 'dbgate-types';
-
+import { interpretEscapes, token, word, whitespace } from './common';
-const whitespace = P.regexp(/\s*/m);
+import { mongoParser } from './mongoParser';
-
-function token(parser) {
-  return parser.skip(whitespace);
-}
-
-function word(str) {
-  return P.string(str).thru(token);
-}
-
-function interpretEscapes(str) {
-  let escapes = {
-    b: '\b',
-    f: '\f',
-    n: '\n',
-    r: '\r',
-    t: '\t',
-  };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
-    let type = escape.charAt(0);
-    let hex = escape.slice(1);
-    if (type === 'u') {
-      return String.fromCharCode(parseInt(hex, 16));
-    }
-    if (escapes.hasOwnProperty(type)) {
-      return escapes[type];
-    }
-    return type;
-  });
-}
 
 const binaryCondition = operator => value => ({
   conditionType: 'binary',
@@ -239,7 +210,8 @@ const createParser = (filterType: FilterType) => {
     yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()),
     yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()),
     yearMonthDayMinute: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteCondition()),
-    yearMonthDaySecond: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()),
+    yearMonthDaySecond: () =>
+      P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()),
 
     value: r => P.alt(...allowedValues.map(x => r[x])),
     valueTestEq: r => r.value.map(binaryCondition('=')),
@@ -348,9 +320,11 @@ const parsers = {
   string: createParser('string'),
   datetime: createParser('datetime'),
   logical: createParser('logical'),
+  mongo: mongoParser,
 };
 
 export function parseFilter(value: string, filterType: FilterType): Condition {
+  // console.log('PARSING', value, 'WITH', filterType);
   const ast = parsers[filterType].list.tryParse(value);
   // console.log('AST', ast);
   return ast;

packages/filterparser/src/types.ts

@@ -1,3 +1,3 @@
 // import types from 'dbgate-types';
 
-export type FilterType = 'number' | 'string' | 'datetime' | 'logical';
+export type FilterType = 'number' | 'string' | 'datetime' | 'logical' | 'mongo';

packages/web/src/datagrid/CollectionDataGridCore.svelte

@@ -1,6 +1,26 @@
 <script context="module" lang="ts">
   async function loadDataPage(props, offset, limit) {
-    const { conid, database } = props;
+    const { conid, database, display } = props;
+
+    const filters = display?.config?.filters;
+
+    const conditions = [];
+    for (const uniqueName in filters) {
+      if (!filters[uniqueName]) continue;
+      try {
+        const ast = parseFilter(filters[uniqueName], 'mongo');
+        const cond = _.cloneDeepWith(ast, expr => {
+          if (expr.__placeholder__) {
+            return {
+              [uniqueName]: expr.__placeholder__,
+            };
+          }
+        });
+        conditions.push(cond);
+      } catch (err) {
+        // error in filter
+      }
+    }
 
     const response = await axiosInstance.request({
       url: 'database-connections/collection-data',
@@ -14,6 +34,12 @@
           pureName: props.pureName,
           limit,
           skip: offset,
+          condition:
+            conditions.length > 0
+              ? {
+                  $and: conditions,
+                }
+              : undefined,
         },
       },
     });
@@ -53,7 +79,9 @@
 
 <script lang="ts">
   import { changeSetToSql, createChangeSet } from 'dbgate-datalib';
+  import { parseFilter } from 'dbgate-filterparser';
   import { scriptToSql } from 'dbgate-sqltree';
+  import _ from 'lodash';
   import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
   import ImportExportModal from '../modals/ImportExportModal.svelte';

packages/web/src/datagrid/DataFilterControl.svelte

@@ -124,6 +124,29 @@
           { onClick: () => openFilterWindow('$'), text: 'Ends With...' },
           { onClick: () => openFilterWindow('!$'), text: 'Does Not End With...' },
         ];
+      case 'mongo':
+        return [
+          { onClick: () => setFilter(''), text: 'Clear Filter' },
+          { onClick: () => filterMultipleValues(), text: 'Filter multiple values' },
+          { onClick: () => openFilterWindow('='), text: 'Equals...' },
+          { onClick: () => openFilterWindow('['), text: 'Does Not Equal...' },
+          { onClick: () => setFilter('EXISTS'), text: 'Field exists' },
+          { onClick: () => setFilter('NOT EXISTS'), text: 'Field does not exist' },
+          { onClick: () => openFilterWindow('>'), text: 'Greater Than...' },
+          { onClick: () => openFilterWindow('>='), text: 'Greater Than Or Equal To...' },
+          { onClick: () => openFilterWindow('<'), text: 'Less Than...' },
+          { onClick: () => openFilterWindow('<='), text: 'Less Than Or Equal To...' },
+          { divider: true },
+          { onClick: () => openFilterWindow('+'), text: 'Contains...' },
+          { onClick: () => openFilterWindow('~'), text: 'Does Not Contain...' },
+          { onClick: () => openFilterWindow('^'), text: 'Begins With...' },
+          { onClick: () => openFilterWindow('!^'), text: 'Does Not Begin With...' },
+          { onClick: () => openFilterWindow('$'), text: 'Ends With...' },
+          { onClick: () => openFilterWindow('!$'), text: 'Does Not End With...' },
+          { divider: true },
+          { onClick: () => setFilter('TRUE'), text: 'Is True' },
+          { onClick: () => setFilter('FALSE'), text: 'Is False' },
+        ];
     }
 
     // return [
@@ -178,6 +201,7 @@
         isOk = true;
       }
     } catch (err) {
+      // console.error(err);
       isError = true;
     }
   }

packages/web/src/datagrid/DataGridCell.svelte

@@ -38,6 +38,7 @@
   export let domCell = undefined;
   export let hideContent = false;
   export let onSetFormView;
+  export let isDynamicStructure = false;
 
   $: value = col.isStructured ? _.get(rowData || {}, col.uniquePath) : (rowData || {})[col.uniqueName];
 </script>
@@ -66,12 +67,26 @@
     {:else if _.isDate(value)}
       {moment(value).format('YYYY-MM-DD HH:mm:ss')}
     {:else if value === true}
-      1
+      {#if isDynamicStructure}
+        <span class="value">true</span>
+      {:else}
+        1
+      {/if}
     {:else if value === false}
-      0
+      {#if isDynamicStructure}
+        <span class="value">false</span>
+      {:else}
+        0
+      {/if}
     {:else if _.isNumber(value)}
       {#if value >= 10000 || value <= -10000}
-        {value.toLocaleString()}
+        {#if isDynamicStructure}
+          <span class="value">{value.toLocaleString()}</span>
+        {:else}
+          {value.toLocaleString()}
+        {/if}
+      {:else if isDynamicStructure}
+        <span class="value">{value.toString()}</span>
       {:else}
         {value.toString()}
       {/if}
@@ -157,4 +172,7 @@
     color: var(--theme-font-3);
     font-style: italic;
   }
+  .value {
+    color: var(--theme-icon-green);
+  }
 </style>

packages/web/src/datagrid/DataGridCore.svelte

@@ -245,7 +245,6 @@
   import openReferenceForm, { openPrimaryKeyForm } from '../formview/openReferenceForm';
   import openNewTab from '../utility/openNewTab';
   import ErrorInfo from '../elements/ErrorInfo.svelte';
-  import resizeObserver from '../utility/resizeObserver';
   import { dataGridRowHeight } from './DataGridRowHeightMeter.svelte';
 
   export let onLoadNextData = undefined;
@@ -1033,7 +1032,7 @@
 {#if !display || (!isDynamicStructure && (!columns || columns.length == 0))}
   <LoadingInfo wrapper message="Waiting for structure" />
 {:else if errorMessage}
-  <ErrorInfo message={errorMessage} />
+  <ErrorInfo message={errorMessage} alignTop />
 {:else if grider.errors && grider.errors.length > 0}
   <div>
     {#each grider.errors as err}
@@ -1121,7 +1120,7 @@
               >
                 <DataFilterControl
                   onGetReference={value => (domFilterControlsRef.get()[col.uniqueName] = value)}
-                  filterType={getFilterType(col.dataType)}
+                  filterType={col.filterType || getFilterType(col.dataType)}
                   filter={display.getFilter(col.uniqueName)}
                   setFilter={value => display.setFilter(col.uniqueName, value)}
                   showResizeSplitter
@@ -1146,6 +1145,7 @@
             {visibleRealColumns}
             {rowHeight}
             {autofillSelectedCells}
+            {isDynamicStructure}
             selectedCells={filterCellsForRow(selectedCells, rowIndex)}
             autofillMarkerCell={filterCellForRow(autofillMarkerCell, rowIndex)}
             focusedColumn={display.focusedColumn}

packages/web/src/datagrid/DataGridRow.svelte

@@ -19,6 +19,7 @@
   export let inplaceEditorState;
   export let dispatchInsplaceEditor;
   export let onSetFormView;
+  export let isDynamicStructure = false;
 
   $: rowData = grider.getRowData(rowIndex);
   $: rowStatus = grider.getRowStatus(rowIndex);
@@ -62,6 +63,7 @@
         isDeleted={rowStatus.status == 'deleted' ||
           (rowStatus.deletedFields && rowStatus.deletedFields.has(col.uniqueName))}
         {onSetFormView}
+        {isDynamicStructure}
       />
     {/if}
   {/each}