perspective - pattern for SQL sources

packages/api/src/proc/databaseConnectionProcess.js

@@ -177,7 +177,7 @@ async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message || 'Error executing SQL script' });
   }
 }
 

packages/datalib/src/PerspectiveDataLoader.ts

@@ -2,6 +2,26 @@ import { Condition, Expression, Select } from 'dbgate-sqltree';
 import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
 import debug from 'debug';
 import _zipObject from 'lodash/zipObject';
+import _mapValues from 'lodash/mapValues';
+import _isArray from 'lodash/isArray';
+import { safeJsonParse } from 'dbgate-tools';
+
+function normalizeLoadedRow(row) {
+  return _mapValues(row, v => safeJsonParse(v) || v);
+}
+
+function normalizeResult(result) {
+  if (_isArray(result)) {
+    return result.map(normalizeLoadedRow);
+  }
+  if (result.errorMessage) {
+    return result;
+  }
+  return {
+    ...result,
+    errorMessage: 'Unspecified error',
+  };
+}
 
 const dbg = debug('dbgate:PerspectiveDataLoader');
 
@@ -187,14 +207,17 @@ export class PerspectiveDataLoader {
         },
       })),
       selectAll: !dataColumns,
-      orderBy: orderBy?.map(({ columnName, order }) => ({
-        exprType: 'column',
-        columnName,
-        direction: order,
-        source: {
-          name: { schemaName, pureName },
-        },
-      })),
+      orderBy:
+        orderBy?.length > 0
+          ? orderBy?.map(({ columnName, order }) => ({
+              exprType: 'column',
+              columnName,
+              direction: order,
+              source: {
+                name: { schemaName, pureName },
+              },
+            }))
+          : null,
       range: props.range,
       where: this.buildSqlCondition(props),
     };
@@ -271,9 +294,9 @@ export class PerspectiveDataLoader {
     const { engineType } = props;
     switch (engineType) {
       case 'sqldb':
-        return this.loadDataSqlDb(props);
+        return normalizeResult(await this.loadDataSqlDb(props));
       case 'docdb':
-        return this.loadDataDocDb(props);
+        return normalizeResult(await this.loadDataDocDb(props));
     }
   }
 

packages/datalib/src/PerspectiveDataPattern.ts

@@ -5,6 +5,7 @@ import _isPlainObject from 'lodash/isPlainObject';
 import _isNumber from 'lodash/isNumber';
 import _isBoolean from 'lodash/isBoolean';
 import _isArray from 'lodash/isArray';
+import { safeJsonParse } from 'dbgate-tools';
 
 export type PerspectiveDataPatternColumnType = 'null' | 'string' | 'number' | 'boolean' | 'json';
 
@@ -57,6 +58,22 @@ function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
           addObjectToColumns(column.columns, item);
         }
       }
+      if (_isString(value)) {
+        const json = safeJsonParse(value);
+        if (json && (_isPlainObject(json) || _isArray(json))) {
+          if (!column.types.includes('json')) {
+            column.types.push('json');
+          }
+          if (_isPlainObject(json)) {
+            addObjectToColumns(column.columns, json);
+          }
+          if (_isArray(json)) {
+            for (const item of json) {
+              addObjectToColumns(column.columns, item);
+            }
+          }
+        }
+      }
     }
   }
 }
@@ -69,6 +86,7 @@ export function analyseDataPattern(
     ...patternBase,
     columns: [],
   };
+  // console.log('ROWS', rows);
   for (const row of rows) {
     addObjectToColumns(res.columns, row);
   }

packages/datalib/src/PerspectiveDataProvider.ts

@@ -1,7 +1,6 @@
 import debug from 'debug';
 import { Condition } from 'dbgate-sqltree';
 import { RangeDefinition } from 'dbgate-types';
-import { format } from 'path';
 import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache';
 import { PerspectiveDataLoader } from './PerspectiveDataLoader';
 import { PerspectiveDataPatternDict } from './PerspectiveDataPattern';
@@ -10,6 +9,7 @@ export const PERSPECTIVE_PAGE_SIZE = 100;
 
 const dbg = debug('dbgate:PerspectiveDataProvider');
 
+
 export interface PerspectiveDatabaseConfig {
   conid: string;
   database: string;

packages/datalib/src/PerspectiveTreeNode.ts

@@ -309,6 +309,10 @@ export abstract class PerspectiveTreeNode {
         ...this.childNodes.map(x => x.childDataColumn),
         ..._flatten(this.childNodes.filter(x => x.isExpandable && x.isChecked).map(x => x.getChildMatchColumns())),
         ...this.getParentMatchColumns(),
+        ...this.childNodes
+          .filter(x => x instanceof PerspectivePatternColumnNode)
+          .filter(x => this.nodeConfig?.checkedColumns?.find(y => y.startsWith(x.codeName + '::')))
+          .map(x => x.columnName),
       ])
     );
   }
@@ -1282,21 +1286,30 @@ export function getTableChildPerspectiveNodes(
 
   const columnNodes =
     tableOrView?.columns?.map(col =>
-      findDesignerIdForNode(
-        config,
-        parentNode,
-        designerId =>
-          new PerspectiveTableColumnNode(
-            col,
-            tableOrView,
-            dbs,
-            config,
-            setConfig,
-            dataProvider,
-            databaseConfig,
-            parentNode,
-            designerId
-          )
+      findDesignerIdForNode(config, parentNode, designerId =>
+        pattern?.columns?.find(x => x.name == col.columnName)?.types.includes('json')
+          ? new PerspectivePatternColumnNode(
+              table,
+              pattern?.columns?.find(x => x.name == col.columnName),
+              dbs,
+              config,
+              setConfig,
+              dataProvider,
+              databaseConfig,
+              parentNode,
+              designerId
+            )
+          : new PerspectiveTableColumnNode(
+              col,
+              tableOrView,
+              dbs,
+              config,
+              setConfig,
+              dataProvider,
+              databaseConfig,
+              parentNode,
+              designerId
+            )
       )
     ) ||
     pattern?.columns?.map(col =>

packages/web/src/utility/usePerspectiveDataPatterns.ts

@@ -29,15 +29,7 @@ export function getPerspectiveDataPatternsFromCache(
     );
     if (cached) {
       res[node.designerId] = cached;
-      continue;
     }
-
-    const db = dbInfos?.[conid]?.[database];
-
-    if (!db) continue;
-
-    const collection = db.collections?.find(x => x.pureName == pureName && x.schemaName == schemaName);
-    if (!collection) continue;
   }
 
   return res;
@@ -69,20 +61,38 @@ export async function getPerspectiveDataPatterns(
 
     if (!db) continue;
 
+    const table = db.tables?.find(x => x.pureName == pureName && x.schemaName == schemaName);
+    const view = db.views?.find(x => x.pureName == pureName && x.schemaName == schemaName);
     const collection = db.collections?.find(x => x.pureName == pureName && x.schemaName == schemaName);
-    if (!collection) continue;
+    if (!table && !view && !collection) continue;
+
+    // console.log('LOAD PATTERN FOR', pureName);
 
     const props: PerspectiveDataLoadProps = {
       databaseConfig: { conid, database },
-      engineType: 'docdb',
+      engineType: collection ? 'docdb' : 'sqldb',
+      schemaName,
       pureName,
-      orderBy: [],
+      orderBy: table?.primaryKey
+        ? table?.primaryKey.columns.map(x => ({ columnName: x.columnName, order: 'ASC' }))
+        : table || view
+        ? [{ columnName: (table || view).columns[0].columnName, order: 'ASC' }]
+        : null,
       range: {
         offset: 0,
         limit: 10,
       },
     };
+    // console.log('LOAD PROPS', props);
     const rows = await dataLoader.loadData(props);
+
+    if (rows.errorMessage) {
+      console.error('Error loading pattern for', pureName, ':', rows.errorMessage);
+      continue;
+    }
+
+    // console.log('PATTERN ROWS', rows);
+
     const pattern = analyseDataPattern(
       {
         conid,