diff --git a/packages/datalib/src/PerspectiveCache.ts b/packages/datalib/src/PerspectiveCache.ts index d26a6697..a2d8846f 100644 --- a/packages/datalib/src/PerspectiveCache.ts +++ b/packages/datalib/src/PerspectiveCache.ts @@ -5,6 +5,7 @@ import _zip from 'lodash/zip'; import _difference from 'lodash/difference'; import debug from 'debug'; import stableStringify from 'json-stable-stringify'; +import { PerspectiveDataPattern } from './PerspectiveDataPattern'; const dbg = debug('dbgate:PerspectiveCache'); @@ -86,6 +87,7 @@ export class PerspectiveCache { constructor() {} tables: { [tableKey: string]: PerspectiveCacheTable } = {}; + dataPatterns: PerspectiveDataPattern[] = []; getTableCache(props: PerspectiveDataLoadProps) { const tableKey = stableStringify( @@ -113,5 +115,6 @@ export class PerspectiveCache { clear() { this.tables = {}; + this.dataPatterns = []; } } diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 716b89bd..580549d5 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -93,8 +93,8 @@ export class PerspectiveDataLoader { })); } - async loadData(props: PerspectiveDataLoadProps) { - const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props; + async loadDataSqlDb(props: PerspectiveDataLoadProps) { + const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition, engineType } = props; if (dataColumns?.length == 0) { return []; @@ -143,7 +143,53 @@ export class PerspectiveDataLoader { return response.rows; } - async loadRowCount(props: PerspectiveDataLoadProps) { + getDocDbLoadOptions(props: PerspectiveDataLoadProps) { + const { pureName } = props; + return { + pureName, + skip: props.range?.offset, + limit: props.range?.limit, + }; + } + + async loadDataDocDb(props: PerspectiveDataLoadProps) { + const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition, engineType } = props; + + if (dataColumns?.length == 0) { + return []; + } + + if (dbg?.enabled) { + dbg( + `LOAD DATA, collection=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${ + props.range?.offset + },${props.range?.limit}` + ); + } + + const options = this.getDocDbLoadOptions(props); + + const response = await this.apiCall('database-connections/collection-data', { + conid: props.databaseConfig.conid, + database: props.databaseConfig.database, + options, + }); + + if (response.errorMessage) return response; + return response.rows; + } + + async loadData(props: PerspectiveDataLoadProps) { + const { engineType } = props; + switch (engineType) { + case 'sqldb': + return this.loadDataSqlDb(props); + case 'docdb': + return this.loadDataDocDb(props); + } + } + + async loadRowCountSqlDb(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props; const select: Select = { @@ -170,4 +216,31 @@ export class PerspectiveDataLoader { if (response.errorMessage) return response; return response.rows[0]; } + + async loadRowCountDocDb(props: PerspectiveDataLoadProps) { + const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props; + + const options = { + ...this.getDocDbLoadOptions(props), + countDocuments: true, + }; + + const response = await this.apiCall('database-connections/collection-data', { + conid: props.databaseConfig.conid, + database: props.databaseConfig.database, + options, + }); + + return response; + } + + async loadRowCount(props: PerspectiveDataLoadProps) { + const { engineType } = props; + switch (engineType) { + case 'sqldb': + return this.loadRowCountSqlDb(props); + case 'docdb': + return this.loadRowCountDocDb(props); + } + } } diff --git a/packages/datalib/src/PerspectiveDataPattern.ts b/packages/datalib/src/PerspectiveDataPattern.ts new file mode 100644 index 00000000..9540307f --- /dev/null +++ b/packages/datalib/src/PerspectiveDataPattern.ts @@ -0,0 +1,65 @@ +import { PerspectiveDataLoader } from './PerspectiveDataLoader'; +import { PerspectiveDataLoadProps } from './PerspectiveDataProvider'; +import _isString from 'lodash/isString'; +import _isPlainObject from 'lodash/isPlainObject'; +import _isNumber from 'lodash/isNumber'; +import _isBoolean from 'lodash/isBoolean'; + +export type PerspectiveDataPatternColumnType = 'null' | 'string' | 'number' | 'boolean' | 'object'; + +export interface PerspectiveDataPatternColumn { + name: string; + types: PerspectiveDataPatternColumnType[]; + columns: PerspectiveDataPatternColumn[]; +} + +export interface PerspectiveDataPattern { + conid: string; + database: string; + schemaName: string; + pureName: string; + columns: PerspectiveDataPatternColumn[]; +} + +export type PerspectiveDataPatternDict = { [designerId: string]: PerspectiveDataPattern }; + +function detectValueType(value): PerspectiveDataPatternColumnType { + if (_isString(value)) return 'string'; + if (_isNumber(value)) return 'number'; + if (_isBoolean(value)) return 'boolean'; + if (value == null) return 'null'; +} + +function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) { + if (_isPlainObject(row)) { + for (const key of Object.keys(row)) { + let column: PerspectiveDataPatternColumn = columns.find(x => x.name == key); + if (!column) { + column = { + name: key, + types: [], + columns: [], + }; + columns.push(column); + } + const type = detectValueType(row[key]); + if (!column.types.includes(type)) { + column.types.push(type); + } + } + } +} + +export function analyseDataPattern( + patternBase: Omit, + rows: any[] +): PerspectiveDataPattern { + const res: PerspectiveDataPattern = { + ...patternBase, + columns: [], + }; + for (const row of rows) { + addObjectToColumns(res.columns, row); + } + return res; +} diff --git a/packages/datalib/src/PerspectiveDataProvider.ts b/packages/datalib/src/PerspectiveDataProvider.ts index 17e3bcac..e06b223e 100644 --- a/packages/datalib/src/PerspectiveDataProvider.ts +++ b/packages/datalib/src/PerspectiveDataProvider.ts @@ -4,6 +4,7 @@ import { RangeDefinition } from 'dbgate-types'; import { format } from 'path'; import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache'; import { PerspectiveDataLoader } from './PerspectiveDataLoader'; +import { PerspectiveDataPatternDict } from './PerspectiveDataPattern'; export const PERSPECTIVE_PAGE_SIZE = 100; @@ -28,10 +29,15 @@ export interface PerspectiveDataLoadProps { range?: RangeDefinition; topCount?: number; condition?: Condition; + engineType: 'sqldb' | 'docdb'; } export class PerspectiveDataProvider { - constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {} + constructor( + public cache: PerspectiveCache, + public loader: PerspectiveDataLoader, + public dataPatterns: PerspectiveDataPatternDict + ) {} async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> { dbg('load data', props); // console.log('LOAD DATA', props); diff --git a/packages/datalib/src/PerspectiveTreeNode.ts b/packages/datalib/src/PerspectiveTreeNode.ts index 0e69f32b..d571b8a0 100644 --- a/packages/datalib/src/PerspectiveTreeNode.ts +++ b/packages/datalib/src/PerspectiveTreeNode.ts @@ -1,4 +1,5 @@ import { + CollectionInfo, ColumnInfo, DatabaseInfo, ForeignKeyInfo, @@ -313,7 +314,7 @@ export abstract class PerspectiveTreeNode { }; } - getOrderBy(table: TableInfo | ViewInfo): PerspectiveDataLoadProps['orderBy'] { + getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] { const res = _compact( this.childNodes.map(node => { const sort = this.nodeConfig?.sort?.find(x => x.columnName == node.columnName); @@ -325,11 +326,15 @@ export abstract class PerspectiveTreeNode { } }) ); - return res.length > 0 - ? res - : (table as TableInfo)?.primaryKey?.columns.map(x => ({ columnName: x.columnName, order: 'ASC' })) || [ - { columnName: table?.columns[0].columnName, order: 'ASC' }, - ]; + if (res.length > 0) return res; + const pkColumns = (table as TableInfo)?.primaryKey?.columns.map(x => ({ + columnName: x.columnName, + order: 'ASC' as 'ASC', + })); + if (pkColumns) return pkColumns; + const columns = (table as TableInfo | ViewInfo)?.columns; + if (columns) return [{ columnName: columns[0].columnName, order: 'ASC' }]; + return [{ columnName: '_id', order: 'ASC' }]; } getBaseTables() { @@ -553,6 +558,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode { databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.refTable), condition: this.getChildrenCondition(), + engineType: 'sqldb', }; } @@ -715,6 +721,7 @@ export class PerspectiveTableNode extends PerspectiveTreeNode { databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), condition: this.getChildrenCondition(), + engineType: 'sqldb', }; } @@ -771,6 +778,88 @@ export class PerspectiveTableNode extends PerspectiveTreeNode { } } +export class PerspectiveCollectionNode extends PerspectiveTreeNode { + constructor( + public collection: CollectionInfo, + dbs: MultipleDatabaseInfo, + config: PerspectiveConfig, + setConfig: ChangePerspectiveConfigFunc, + public dataProvider: PerspectiveDataProvider, + databaseConfig: PerspectiveDatabaseConfig, + parentNode: PerspectiveTreeNode, + designerId: string + ) { + super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId); + } + + getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps { + return { + schemaName: this.collection.schemaName, + pureName: this.collection.pureName, + dataColumns: this.getDataLoadColumns(), + databaseConfig: this.databaseConfig, + orderBy: this.getOrderBy(this.collection), + condition: this.getChildrenCondition(), + engineType: 'docdb', + }; + } + + get codeName() { + return this.collection.schemaName + ? `${this.collection.schemaName}:${this.collection.pureName}` + : this.collection.pureName; + } + + get title() { + return this.nodeConfig?.alias || this.collection.pureName; + } + + get isExpandable() { + return true; + } + + generateChildNodes(): PerspectiveTreeNode[] { + return []; + // return getTableChildPerspectiveNodes( + // this.table, + // this.dbs, + // this.config, + // this.setConfig, + // this.dataProvider, + // this.databaseConfig, + // this + // ); + } + + get icon() { + return 'img collection'; + } + + getBaseTableFromThis() { + return this.collection; + } + + get headerTableAttributes() { + return { + schemaName: this.collection.schemaName, + pureName: this.collection.pureName, + conid: this.databaseConfig.conid, + database: this.databaseConfig.database, + }; + } + + get tableCode() { + return `${this.collection.schemaName}|${this.collection.pureName}`; + } + + get namedObject(): NamedObjectInfo { + return { + schemaName: this.collection.schemaName, + pureName: this.collection.pureName, + }; + } +} + // export class PerspectiveViewNode extends PerspectiveTreeNode { // constructor( // public view: ViewInfo, @@ -873,6 +962,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode { databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), condition: this.getChildrenCondition(), + engineType: 'sqldb', }; } @@ -978,6 +1068,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode { databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), condition: this.getChildrenCondition(), + engineType: 'sqldb', }; } diff --git a/packages/datalib/src/index.ts b/packages/datalib/src/index.ts index dace2858..d4626bb5 100644 --- a/packages/datalib/src/index.ts +++ b/packages/datalib/src/index.ts @@ -19,3 +19,4 @@ export * from './PerspectiveDataProvider'; export * from './PerspectiveCache'; export * from './PerspectiveConfig'; export * from './processPerspectiveDefaultColunns'; +export * from './PerspectiveDataPattern'; diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index a6cceab6..47dbbad1 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -345,6 +345,12 @@ }, }, }, + { + label: 'Open perspective', + tab: 'PerspectiveTab', + forceNewTab: true, + icon: 'img perspective', + }, { label: 'Export', isExport: true, diff --git a/packages/web/src/designer/DesignerTable.svelte b/packages/web/src/designer/DesignerTable.svelte index a7bab85f..ad8fa924 100644 --- a/packages/web/src/designer/DesignerTable.svelte +++ b/packages/web/src/designer/DesignerTable.svelte @@ -238,6 +238,7 @@ class:isGrayed class:isTable={objectTypeField == 'tables'} class:isView={objectTypeField == 'views'} + class:isCollection={objectTypeField == 'collections'} use:moveDrag={settings?.canSelectColumns ? [handleMoveStart, handleMove, handleMoveEnd] : null} use:contextMenu={settings?.canSelectColumns ? createMenu : '__no_menu'} style={getTableColorStyle($currentThemeDefinition, table)} @@ -358,6 +359,10 @@ .header.isView { background: var(--theme-bg-magenta); } + .header.isCollection { + background: var(--theme-bg-red); + } + .header.isGrayed { background: var(--theme-bg-2); } diff --git a/packages/web/src/perspectives/PerspectiveDesigner.svelte b/packages/web/src/perspectives/PerspectiveDesigner.svelte index 7a8b8b1c..6d7baaa0 100644 --- a/packages/web/src/perspectives/PerspectiveDesigner.svelte +++ b/packages/web/src/perspectives/PerspectiveDesigner.svelte @@ -3,10 +3,12 @@ createPerspectiveNodeConfig, MultipleDatabaseInfo, PerspectiveConfig, + PerspectiveDataPatternDict, perspectiveNodesHaveStructure, PerspectiveTreeNode, switchPerspectiveReferenceDirection, } from 'dbgate-datalib'; + import { CollectionInfo } from 'dbgate-types'; import _ from 'lodash'; import { tick } from 'svelte'; import runCommand from '../commands/runCommand'; @@ -18,6 +20,7 @@ export let config: PerspectiveConfig; export let dbInfos: MultipleDatabaseInfo; + export let dataPatterns: PerspectiveDataPatternDict; export let root: PerspectiveTreeNode; export let conid; @@ -27,7 +30,11 @@ export let onClickTableHeader = null; - function createDesignerModel(config: PerspectiveConfig, dbInfos: MultipleDatabaseInfo) { + function createDesignerModel( + config: PerspectiveConfig, + dbInfos: MultipleDatabaseInfo, + dataPatterns: PerspectiveDataPatternDict + ) { return { ...config, tables: _.compact( @@ -38,11 +45,26 @@ const view = dbInfos?.[node.conid || conid]?.[node.database || database]?.views?.find( x => x.pureName == node.pureName && x.schemaName == node.schemaName ); - if (!table && !view) return null; + let collection: CollectionInfo & { columns?: any[] } = dbInfos?.[node.conid || conid]?.[ + node.database || database + ]?.collections?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName); + + if (collection) { + const pattern = dataPatterns?.[node.designerId]; + if (!pattern) return null; + collection = { + ...collection, + columns: pattern.columns.map(x => ({ + columnName: x.name, + })), + }; + } + + if (!table && !view && !collection) return null; const { designerId } = node; return { - ...(table || view), + ...(table || view || collection), left: node?.position?.x || 0, top: node?.position?.y || 0, alias: node.alias, @@ -55,7 +77,7 @@ function handleChange(value, skipUndoChain, settings) { setConfig(oldValue => { - const newValue = _.isFunction(value) ? value(createDesignerModel(oldValue, dbInfos)) : value; + const newValue = _.isFunction(value) ? value(createDesignerModel(oldValue, dbInfos, dataPatterns)) : value; let isArranged = oldValue.isArranged; if (settings?.isCalledFromArrange) { isArranged = true; @@ -277,6 +299,6 @@ onClickTableHeader, }} referenceComponent={QueryDesignerReference} - value={createDesignerModel(config, dbInfos)} + value={createDesignerModel(config, dbInfos, dataPatterns)} onChange={handleChange} /> diff --git a/packages/web/src/perspectives/PerspectiveView.svelte b/packages/web/src/perspectives/PerspectiveView.svelte index e5d64440..877c08ac 100644 --- a/packages/web/src/perspectives/PerspectiveView.svelte +++ b/packages/web/src/perspectives/PerspectiveView.svelte @@ -28,6 +28,7 @@ import { ChangePerspectiveConfigFunc, extractPerspectiveDatabases, + PerspectiveCollectionNode, PerspectiveConfig, PerspectiveDataProvider, PerspectiveTableNode, @@ -65,6 +66,7 @@ import { sleep } from '../utility/common'; import FontIcon from '../icons/FontIcon.svelte'; import InlineButton from '../buttons/InlineButton.svelte'; + import { usePerspectiveDataPatterns } from '../utility/usePerspectiveDataPatterns'; const dbg = debug('dbgate:PerspectiveView'); @@ -128,13 +130,17 @@ } $: dbInfos = useMultipleDatabaseInfo(perspectiveDatabases); + $: loader = new PerspectiveDataLoader(apiCall); + $: dataPatterns = usePerspectiveDataPatterns({ conid, database }, config, cache, $dbInfos, loader); $: rootObject = config?.nodes?.find(x => x.designerId == config?.rootDesignerId); $: rootDb = rootObject ? $dbInfos?.[rootObject.conid || conid]?.[rootObject.database || database] : null; $: tableInfo = rootDb?.tables.find(x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName); $: viewInfo = rootDb?.views.find(x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName); + $: collectionInfo = rootDb?.collections.find( + x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName + ); - $: loader = new PerspectiveDataLoader(apiCall); - $: dataProvider = new PerspectiveDataProvider(cache, loader); + $: dataProvider = new PerspectiveDataProvider(cache, loader, $dataPatterns); $: root = tableInfo || viewInfo ? new PerspectiveTableNode( @@ -147,6 +153,17 @@ null, config.rootDesignerId ) + : collectionInfo + ? new PerspectiveCollectionNode( + collectionInfo, + $dbInfos, + config, + setConfig, + dataProvider, + { conid, database }, + null, + config.rootDesignerId + ) : null; $: tempRoot = root?.findNodeByDesignerId(tempRootDesignerId); @@ -158,6 +175,7 @@ // $: console.log('PERSPECTIVE', config); // $: console.log('VIEW ROOT', root); + // $: console.log('dataPatterns', $dataPatterns); @@ -205,6 +223,7 @@ {database} {setConfig} dbInfos={$dbInfos} + dataPatterns={$dataPatterns} {root} onClickTableHeader={designerId => { sleep(100).then(() => { diff --git a/packages/web/src/utility/usePerspectiveDataPatterns.ts b/packages/web/src/utility/usePerspectiveDataPatterns.ts new file mode 100644 index 00000000..fdabd902 --- /dev/null +++ b/packages/web/src/utility/usePerspectiveDataPatterns.ts @@ -0,0 +1,77 @@ +import { + analyseDataPattern, + MultipleDatabaseInfo, + PerspectiveCache, + PerspectiveConfig, + PerspectiveDatabaseConfig, + PerspectiveDataLoadProps, + PerspectiveDataPattern, + PerspectiveDataPatternDict, +} from 'dbgate-datalib'; +import { PerspectiveDataLoader } from 'dbgate-datalib/lib/PerspectiveDataLoader'; +import { writable, Readable } from 'svelte/store'; + +export async function getPerspectiveDataPatterns( + databaseConfig: PerspectiveDatabaseConfig, + config: PerspectiveConfig, + cache: PerspectiveCache, + dbInfos: MultipleDatabaseInfo, + dataLoader: PerspectiveDataLoader +): Promise { + const res = {}; + + for (const node of config.nodes) { + const conid = node.conid || databaseConfig.conid; + const database = node.database || databaseConfig.database; + const { schemaName, pureName } = node; + + const cached = cache.dataPatterns.find( + x => x.conid == conid && x.database == database && x.schemaName == schemaName && x.pureName == pureName + ); + if (cached) { + res[node.designerId] = cached; + continue; + } + + const db = dbInfos?.[conid]?.[database]; + + if (!db) continue; + + const collection = db.collections?.find(x => x.pureName == pureName && x.schemaName == schemaName); + if (!collection) continue; + + const props: PerspectiveDataLoadProps = { + databaseConfig: { conid, database }, + engineType: 'docdb', + pureName, + orderBy: [], + }; + const rows = await dataLoader.loadData(props); + const pattern = analyseDataPattern( + { + conid, + database, + pureName, + schemaName, + }, + rows + ); + + cache.dataPatterns.push(pattern); + res[node.designerId] = pattern; + } + + return res; +} + +export function usePerspectiveDataPatterns( + databaseConfig: PerspectiveDatabaseConfig, + config: PerspectiveConfig, + cache: PerspectiveCache, + dbInfos: MultipleDatabaseInfo, + dataLoader: PerspectiveDataLoader +): Readable { + const res = writable({}); + getPerspectiveDataPatterns(databaseConfig, config, cache, dbInfos, dataLoader).then(value => res.set(value)); + return res; +}