diff --git a/packages/core/database/src/__tests__/eager-loading/eager-loading-tree.test.ts b/packages/core/database/src/__tests__/eager-loading/eager-loading-tree.test.ts new file mode 100644 index 0000000000..dcda07ccfe --- /dev/null +++ b/packages/core/database/src/__tests__/eager-loading/eager-loading-tree.test.ts @@ -0,0 +1,393 @@ +import Database, { mockDatabase } from '@nocobase/database'; +import { EagerLoadingTree } from '../../eager-loading/eager-loading-tree'; + +describe('Eager loading tree', () => { + let db: Database; + beforeEach(async () => { + db = mockDatabase({ + tablePrefix: '', + }); + + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should handle fields filter', async () => { + const User = db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'hasOne', name: 'profile' }, + ], + }); + + const Profile = db.collection({ + name: 'profiles', + fields: [ + { type: 'integer', name: 'age' }, + { type: 'string', name: 'address' }, + ], + }); + + await db.sync(); + + const users = await User.repository.create({ + values: [ + { + name: 'u1', + profile: { age: 1, address: 'u1 address' }, + }, + { + name: 'u2', + profile: { age: 2, address: 'u2 address' }, + }, + ], + }); + + const findOptions = User.repository.buildQueryOptions({ + fields: ['profile', 'profile.age', 'name'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: User.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + + await eagerLoadingTree.load(users.map((item) => item.id)); + const root = eagerLoadingTree.root; + + const u1 = root.instances.find((item) => item.get('name') === 'u1'); + const data = u1.toJSON(); + expect(data['id']).not.toBeDefined(); + expect(data['name']).toBeDefined(); + expect(data['profile']).toBeDefined(); + expect(data['profile']['age']).toBeDefined(); + }); + + it('should load has many', async () => { + const User = db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'hasMany', name: 'posts' }, + ], + }); + + const Post = db.collection({ + name: 'posts', + fields: [{ type: 'string', name: 'title' }], + }); + + await db.sync(); + + await User.repository.create({ + values: [ + { + name: 'u1', + posts: [{ title: 'u1p1' }, { title: 'u1p2' }], + }, + { + name: 'u2', + posts: [{ title: 'u2p1' }, { title: 'u2p2' }], + }, + ], + }); + + const findOptions = User.repository.buildQueryOptions({ + appends: ['posts'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: User.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + + await eagerLoadingTree.load([1, 2]); + + const root = eagerLoadingTree.root; + const u1 = root.instances.find((item) => item.get('name') === 'u1'); + const u1Posts = u1.get('posts') as any; + expect(u1Posts.length).toBe(2); + + const u1JSON = u1.toJSON(); + expect(u1JSON['posts'].length).toBe(2); + }); + + it('should load has one', async () => { + const User = db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'hasOne', name: 'profile' }, + ], + }); + + const Profile = db.collection({ + name: 'profiles', + fields: [{ type: 'integer', name: 'age' }], + }); + + await db.sync(); + + const users = await User.repository.create({ + values: [ + { + name: 'u1', + profile: { age: 1 }, + }, + { + name: 'u2', + profile: { age: 2 }, + }, + ], + }); + + const findOptions = User.repository.buildQueryOptions({ + appends: ['profile'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: User.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + + await eagerLoadingTree.load(users.map((item) => item.id)); + + const root = eagerLoadingTree.root; + const u1 = root.instances.find((item) => item.get('name') === 'u1'); + const u1Profile = u1.get('profile') as any; + expect(u1Profile).toBeDefined(); + expect(u1Profile.get('age')).toBe(1); + }); + + it('should load belongs to', async () => { + const Post = db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { + type: 'belongsTo', + name: 'user', + }, + ], + }); + + const User = db.collection({ + name: 'users', + fields: [{ type: 'string', name: 'name' }], + }); + + await db.sync(); + + await Post.repository.create({ + values: [ + { + title: 'p1', + user: { + name: 'u1', + }, + }, + { + title: 'p2', + user: { + name: 'u2', + }, + }, + ], + }); + + const findOptions = Post.repository.buildQueryOptions({ + appends: ['user'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: Post.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + + await eagerLoadingTree.load([1, 2]); + + const root = eagerLoadingTree.root; + const p1 = root.instances.find((item) => item.get('title') === 'p1'); + const p1User = p1.get('user') as any; + expect(p1User).toBeDefined(); + expect(p1User.get('name')).toBe('u1'); + }); + + it('should load belongs to many', async () => { + const Post = db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { type: 'belongsToMany', name: 'tags' }, + ], + }); + + const Tag = db.collection({ + name: 'tags', + fields: [{ type: 'string', name: 'name' }], + }); + + await db.sync(); + + const tags = await Tag.repository.create({ + values: [ + { + name: 't1', + }, + { + name: 't2', + }, + { + name: 't3', + }, + ], + }); + + await Post.repository.create({ + values: [ + { + title: 'p1', + tags: [{ id: tags[0].id }, { id: tags[1].id }], + }, + { + title: 'p2', + tags: [{ id: tags[1].id }, { id: tags[2].id }], + }, + ], + }); + + const findOptions = Post.repository.buildQueryOptions({ + appends: ['tags'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: Post.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + await eagerLoadingTree.load([1, 2]); + const root = eagerLoadingTree.root; + + const p1 = root.instances.find((item) => item.get('title') === 'p1'); + const p1Tags = p1.get('tags') as any; + expect(p1Tags).toBeDefined(); + expect(p1Tags.length).toBe(2); + expect(p1Tags.map((t) => t.get('name'))).toEqual(['t1', 't2']); + + const p2 = root.instances.find((item) => item.get('title') === 'p2'); + const p2Tags = p2.get('tags') as any; + expect(p2Tags).toBeDefined(); + expect(p2Tags.length).toBe(2); + expect(p2Tags.map((t) => t.get('name'))).toEqual(['t2', 't3']); + }); + + it('should build eager loading tree', async () => { + const User = db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'hasMany', + name: 'posts', + }, + ], + }); + + const Post = db.collection({ + name: 'posts', + fields: [ + { + type: 'array', + name: 'tags', + }, + { + type: 'string', + name: 'title', + }, + { + type: 'belongsToMany', + name: 'tags', + }, + ], + }); + + const Tag = db.collection({ + name: 'tags', + fields: [ + { type: 'string', name: 'name' }, + { type: 'belongsTo', name: 'tagCategory' }, + ], + }); + + const TagCategory = db.collection({ + name: 'tagCategories', + fields: [{ type: 'string', name: 'name' }], + }); + + await db.sync(); + + await User.repository.create({ + values: [ + { + name: 'u1', + posts: [ + { + title: 'u1p1', + tags: [ + { name: 't1', tagCategory: { name: 'c1' } }, + { name: 't2', tagCategory: { name: 'c2' } }, + ], + }, + ], + }, + { + name: 'u2', + posts: [ + { + title: 'u2p1', + tags: [ + { name: 't3', tagCategory: { name: 'c3' } }, + { name: 't4', tagCategory: { name: 'c4' } }, + ], + }, + ], + }, + ], + }); + + const findOptions = User.repository.buildQueryOptions({ + appends: ['posts.tags.tagCategory'], + }); + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: User.model, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + }); + + expect(eagerLoadingTree.root.children).toHaveLength(1); + expect(eagerLoadingTree.root.children[0].model).toBe(Post.model); + expect(eagerLoadingTree.root.children[0].children[0].model).toBe(Tag.model); + expect(eagerLoadingTree.root.children[0].children[0].children[0].model).toBe(TagCategory.model); + + await eagerLoadingTree.load((await User.model.findAll()).map((item) => item[User.model.primaryKeyAttribute])); + + expect(eagerLoadingTree.root.instances).toHaveLength(2); + const u1 = eagerLoadingTree.root.instances.find((item) => item.get('name') === 'u1'); + expect(u1.get('posts')).toHaveLength(1); + expect(u1.get('posts')[0].get('tags')).toHaveLength(2); + expect(u1.get('posts')[0].get('tags')[0].get('tagCategory')).toBeDefined(); + expect(u1.get('posts')[0].get('tags')[0].get('tagCategory').get('name')).toBe('c1'); + }); +}); diff --git a/packages/core/database/src/__tests__/relation-repository/hasone-repository.test.ts b/packages/core/database/src/__tests__/relation-repository/hasone-repository.test.ts index e56ea2a5ea..4300a6f890 100644 --- a/packages/core/database/src/__tests__/relation-repository/hasone-repository.test.ts +++ b/packages/core/database/src/__tests__/relation-repository/hasone-repository.test.ts @@ -93,6 +93,7 @@ describe('has one repository', () => { }); const data = profile.toJSON(); + expect(data['a1']).toBeDefined(); expect(data['a2']).toBeDefined(); }); diff --git a/packages/core/database/src/__tests__/repository/count.test.ts b/packages/core/database/src/__tests__/repository/count.test.ts index 70f9faabfc..87716daeb3 100644 --- a/packages/core/database/src/__tests__/repository/count.test.ts +++ b/packages/core/database/src/__tests__/repository/count.test.ts @@ -117,7 +117,7 @@ describe('count', () => { appends: ['tags'], }); - expect(posts[0][0]['tags']).toBeDefined(); + expect(posts[0][0].get('tags')).toBeDefined(); }); test('without filter params', async () => { diff --git a/packages/core/database/src/__tests__/repository/find.test.ts b/packages/core/database/src/__tests__/repository/find.test.ts index 142cc5b21c..310b88376a 100644 --- a/packages/core/database/src/__tests__/repository/find.test.ts +++ b/packages/core/database/src/__tests__/repository/find.test.ts @@ -176,7 +176,7 @@ describe('find with associations', () => { }, }); - expect(filterResult[0].user.department).toBeDefined(); + expect(filterResult[0].get('user').get('department')).toBeDefined(); }); it('should filter by association field', async () => { diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 6d2158c04e..71697e7fa0 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -188,6 +188,7 @@ export class Database extends EventEmitter implements AsyncEmitter { logger: Logger; collectionGroupManager = new CollectionGroupManager(this); + declare emitAsync: (event: string | symbol, ...args: any[]) => Promise; constructor(options: DatabaseOptions) { super(); @@ -387,36 +388,6 @@ export class Database extends EventEmitter implements AsyncEmitter { } }); - this.on('afterRepositoryFind', ({ findOptions, dataCollection, data }) => { - if (dataCollection.isParent()) { - for (const row of data) { - const rowCollection = this.tableNameCollectionMap.get( - findOptions.raw - ? `${row['__schemaName']}.${row['__tableName']}` - : `${row.get('__schemaName')}.${row.get('__tableName')}`, - ); - - if (!rowCollection) { - this.logger.warn( - `Can not find collection by table name ${JSON.stringify(row)}, current collections: ${Array.from( - this.tableNameCollectionMap.keys(), - ).join(', ')}`, - ); - - return; - } - - const rowCollectionName = rowCollection.name; - - findOptions.raw - ? (row['__collection'] = rowCollectionName) - : row.set('__collection', rowCollectionName, { - raw: true, - }); - } - } - }); - registerBuiltInListeners(this); } @@ -565,7 +536,9 @@ export class Database extends EventEmitter implements AsyncEmitter { } getRepository(name: string): R; + getRepository(name: string, relationId: string | number): R; + getRepository(name: string, relationId: string | number): R; getRepository(name: string, relationId?: string | number): Repository | R { @@ -779,21 +752,34 @@ export class Database extends EventEmitter implements AsyncEmitter { } on(event: EventType, listener: any): this; + on(event: ModelValidateEventTypes, listener: SyncListener): this; + on(event: ModelValidateEventTypes, listener: ValidateListener): this; + on(event: ModelCreateEventTypes, listener: CreateListener): this; + on(event: ModelUpdateEventTypes, listener: UpdateListener): this; + on(event: ModelSaveEventTypes, listener: SaveListener): this; + on(event: ModelDestroyEventTypes, listener: DestroyListener): this; + on(event: ModelCreateWithAssociationsEventTypes, listener: CreateWithAssociationsListener): this; + on(event: ModelUpdateWithAssociationsEventTypes, listener: UpdateWithAssociationsListener): this; + on(event: ModelSaveWithAssociationsEventTypes, listener: SaveWithAssociationsListener): this; + on(event: DatabaseBeforeDefineCollectionEventType, listener: BeforeDefineCollectionListener): this; + on(event: DatabaseAfterDefineCollectionEventType, listener: AfterDefineCollectionListener): this; + on( event: DatabaseBeforeRemoveCollectionEventType | DatabaseAfterRemoveCollectionEventType, listener: RemoveCollectionListener, ): this; + on(event: EventType, listener: any): this { // NOTE: to match if event is a sequelize or model type const type = this.modelHook.match(event); @@ -844,8 +830,6 @@ export class Database extends EventEmitter implements AsyncEmitter { return result; } - - declare emitAsync: (event: string | symbol, ...args: any[]) => Promise; } export function extendCollection(collectionOptions: CollectionOptions, mergeOptions?: MergeOptions) { diff --git a/packages/core/database/src/eager-loading/eager-loading-tree.ts b/packages/core/database/src/eager-loading/eager-loading-tree.ts new file mode 100644 index 0000000000..58f360bbff --- /dev/null +++ b/packages/core/database/src/eager-loading/eager-loading-tree.ts @@ -0,0 +1,296 @@ +import { Association, Includeable, Model, ModelStatic, Transaction } from 'sequelize'; +import lodash from 'lodash'; + +interface EagerLoadingNode { + model: ModelStatic; + association: Association; + attributes: Array; + rawAttributes: Array; + children: Array; + parent?: EagerLoadingNode; + instances?: Array; + order?: any; +} + +export class EagerLoadingTree { + public root: EagerLoadingNode; + + constructor(root: EagerLoadingNode) { + this.root = root; + } + + static buildFromSequelizeOptions(options: { + model: ModelStatic; + rootAttributes: Array; + rootOrder?: any; + includeOption: Includeable | Includeable[]; + }): EagerLoadingTree { + const { model, rootAttributes, includeOption } = options; + + const root = { + model, + association: null, + rawAttributes: lodash.cloneDeep(rootAttributes), + attributes: lodash.cloneDeep(rootAttributes), + order: options.rootOrder, + children: [], + }; + + const pushAttribute = (node, attribute) => { + if (lodash.isArray(node.attributes) && !node.attributes.includes(attribute)) { + node.attributes.push(attribute); + } + }; + + const traverseIncludeOption = (includeOption, eagerLoadingTreeParent) => { + const includeOptions = lodash.castArray(includeOption); + + if (includeOption.length > 0) { + const modelPrimaryKey = eagerLoadingTreeParent.model.primaryKeyAttribute; + pushAttribute(eagerLoadingTreeParent, modelPrimaryKey); + } + + for (const include of includeOptions) { + // skip fromFilter include option + if (include.fromFilter) { + continue; + } + + const association = eagerLoadingTreeParent.model.associations[include.association]; + const associationType = association.associationType; + + const child = { + model: association.target, + association, + rawAttributes: lodash.cloneDeep(include.attributes), + attributes: lodash.cloneDeep(include.attributes), + parent: eagerLoadingTreeParent, + children: [], + }; + + if (associationType == 'HasOne' || associationType == 'HasMany') { + const { sourceKey, foreignKey } = association; + + pushAttribute(eagerLoadingTreeParent, sourceKey); + pushAttribute(child, foreignKey); + } + + if (associationType == 'BelongsTo') { + const { sourceKey, foreignKey } = association; + + pushAttribute(eagerLoadingTreeParent, foreignKey); + pushAttribute(child, sourceKey); + } + + eagerLoadingTreeParent.children.push(child); + + if (include.include) { + traverseIncludeOption(include.include, child); + } + } + }; + + traverseIncludeOption(includeOption, root); + + return new EagerLoadingTree(root); + } + + async load(pks: Array, transaction?: Transaction) { + const result = {}; + + const loadRecursive = async (node, ids) => { + const modelPrimaryKey = node.model.primaryKeyAttribute; + + let instances = []; + + // load instances from database + if (!node.parent) { + const findOptions = { + where: { [modelPrimaryKey]: ids }, + attributes: node.attributes, + }; + + if (node.order) { + findOptions['order'] = node.order; + } + + instances = await node.model.findAll({ + ...findOptions, + transaction, + }); + } else if (ids.length > 0) { + const association = node.association; + const associationType = association.associationType; + + if (associationType == 'HasOne' || associationType == 'HasMany') { + const foreignKey = association.foreignKey; + const foreignKeyValues = node.parent.instances.map((instance) => instance.get(association.sourceKey)); + + const findOptions = { + where: { [foreignKey]: foreignKeyValues }, + attributes: node.attributes, + transaction, + }; + + instances = await node.model.findAll(findOptions); + } + + if (associationType == 'BelongsTo') { + const foreignKey = association.foreignKey; + + const parentInstancesForeignKeyValues = node.parent.instances.map((instance) => instance.get(foreignKey)); + + instances = await node.model.findAll({ + transaction, + where: { + [association.targetKey]: parentInstancesForeignKeyValues, + }, + attributes: node.attributes, + }); + } + + if (associationType == 'BelongsToMany') { + instances = await node.model.findAll({ + transaction, + include: [ + { + association: association.oneFromTarget, + where: { + [association.foreignKey]: ids, + }, + }, + ], + }); + } + } + + node.instances = instances; + + for (const child of node.children) { + const nodeIds = instances.map((instance) => instance.get(modelPrimaryKey)); + + await loadRecursive(child, nodeIds); + } + + // merge instances to parent + if (!node.parent) { + return; + } else { + const association = node.association; + const associationType = association.associationType; + + const setParentAccessor = (parentInstance) => { + const key = association.as; + + const children = parentInstance.getDataValue(association.as); + + if (association.isSingleAssociation) { + const isEmpty = !children; + parentInstance[key] = parentInstance.dataValues[key] = isEmpty ? null : children; + } else { + const isEmpty = !children || children.length == 0; + parentInstance[key] = parentInstance.dataValues[key] = isEmpty ? [] : children; + } + }; + + if (associationType == 'HasMany' || associationType == 'HasOne') { + const foreignKey = association.foreignKey; + const sourceKey = association.sourceKey; + + for (const instance of node.instances) { + const parentInstance = node.parent.instances.find( + (parentInstance) => parentInstance.get(sourceKey) == instance.get(foreignKey), + ); + + if (parentInstance) { + if (associationType == 'HasMany') { + const children = parentInstance.getDataValue(association.as); + if (!children) { + parentInstance.setDataValue(association.as, [instance]); + } else { + children.push(instance); + } + } + + if (associationType == 'HasOne') { + parentInstance.setDataValue(association.as, instance); + } + } + } + } + + if (associationType == 'BelongsTo') { + const foreignKey = association.foreignKey; + const targetKey = association.targetKey; + + for (const instance of node.instances) { + const parentInstance = node.parent.instances.find( + (parentInstance) => parentInstance.get(foreignKey) == instance.get(targetKey), + ); + + if (parentInstance) { + parentInstance.setDataValue(association.as, instance); + } + } + } + + if (associationType == 'BelongsToMany') { + const sourceKey = association.sourceKey; + const foreignKey = association.foreignKey; + + const oneFromTarget = association.oneFromTarget; + + for (const instance of node.instances) { + const parentInstance = node.parent.instances.find( + (parentInstance) => parentInstance.get(sourceKey) == instance.get(oneFromTarget.as).get(foreignKey), + ); + + if (parentInstance) { + const children = parentInstance.getDataValue(association.as); + + if (!children) { + parentInstance.setDataValue(association.as, [instance]); + } else { + children.push(instance); + } + } + } + } + + for (const parent of node.parent.instances) { + setParentAccessor(parent); + } + } + }; + + await loadRecursive(this.root, pks); + + const setInstanceAttributes = (node) => { + const nodeRawAttributes = node.rawAttributes; + + if (!lodash.isArray(nodeRawAttributes)) { + return; + } + + const nodeChildrenAs = node.children.map((child) => child.association.as); + const includeAttributes = [...nodeRawAttributes, ...nodeChildrenAs]; + + for (const instance of node.instances) { + const attributes = lodash.pick(instance.dataValues, includeAttributes); + instance.dataValues = attributes; + } + }; + + // traverse tree and set instance attributes + const traverse = (node) => { + setInstanceAttributes(node); + + for (const child of node.children) { + traverse(child); + } + }; + + traverse(this.root); + return result; + } +} diff --git a/packages/core/database/src/filter-parser.ts b/packages/core/database/src/filter-parser.ts index 90e305d867..c14048ede1 100644 --- a/packages/core/database/src/filter-parser.ts +++ b/packages/core/database/src/filter-parser.ts @@ -56,7 +56,7 @@ export default class FilterParser { return filter; } - toSequelizeParams() { + toSequelizeParams(): any { debug('filter %o', this.filter); if (!this.filter) { @@ -230,7 +230,21 @@ export default class FilterParser { }); }; debug('where %o, include %o', where, include); - return { where, include: toInclude(include) }; + const results = { where, include: toInclude(include) }; + + //traverse filter include, set fromFiler to true + const traverseInclude = (include) => { + for (const item of include) { + if (item.include) { + traverseInclude(item.include); + } + item.fromFilter = true; + } + }; + + traverseInclude(results.include); + + return results; } private getFieldNameFromQueryPath(queryPath: string) { diff --git a/packages/core/database/src/listeners/adjacency-list.ts b/packages/core/database/src/listeners/adjacency-list.ts index d0f829de43..e683734f3e 100644 --- a/packages/core/database/src/listeners/adjacency-list.ts +++ b/packages/core/database/src/listeners/adjacency-list.ts @@ -1,6 +1,4 @@ -import lodash from 'lodash'; -import { Collection, CollectionOptions } from '../collection'; -import { Model } from '../model'; +import { CollectionOptions } from '../collection'; export const beforeDefineAdjacencyListCollection = (options: CollectionOptions) => { if (!options.tree) { diff --git a/packages/core/database/src/listeners/append-child-collection-name-after-repository-find.ts b/packages/core/database/src/listeners/append-child-collection-name-after-repository-find.ts new file mode 100644 index 0000000000..bbcae3e95c --- /dev/null +++ b/packages/core/database/src/listeners/append-child-collection-name-after-repository-find.ts @@ -0,0 +1,31 @@ +export const appendChildCollectionNameAfterRepositoryFind = (db) => { + return ({ findOptions, dataCollection, data }) => { + if (dataCollection.isParent()) { + for (const row of data) { + const rowCollection = db.tableNameCollectionMap.get( + findOptions.raw + ? `${row['__schemaName']}.${row['__tableName']}` + : `${row.get('__schemaName')}.${row.get('__tableName')}`, + ); + + if (!rowCollection) { + db.logger.warn( + `Can not find collection by table name ${JSON.stringify(row)}, current collections: ${Array.from( + db.tableNameCollectionMap.keys(), + ).join(', ')}`, + ); + + return; + } + + const rowCollectionName = rowCollection.name; + + findOptions.raw + ? (row['__collection'] = rowCollectionName) + : row.set('__collection', rowCollectionName, { + raw: true, + }); + } + } + }; +}; diff --git a/packages/core/database/src/listeners/index.ts b/packages/core/database/src/listeners/index.ts index 71a9b814cd..446eff2ac0 100644 --- a/packages/core/database/src/listeners/index.ts +++ b/packages/core/database/src/listeners/index.ts @@ -1,6 +1,8 @@ import { Database } from '../database'; import { beforeDefineAdjacencyListCollection } from './adjacency-list'; +import { appendChildCollectionNameAfterRepositoryFind } from './append-child-collection-name-after-repository-find'; export const registerBuiltInListeners = (db: Database) => { db.on('beforeDefineCollection', beforeDefineAdjacencyListCollection); + db.on('afterRepositoryFind', appendChildCollectionNameAfterRepositoryFind(db)); }; diff --git a/packages/core/database/src/options-parser.ts b/packages/core/database/src/options-parser.ts index 01d188d55c..d67d74a878 100644 --- a/packages/core/database/src/options-parser.ts +++ b/packages/core/database/src/options-parser.ts @@ -269,6 +269,11 @@ export class OptionsParser { (include) => include['association'] == appendAssociation, ); + // if include from filter, remove fromFilter attribute + if (existIncludeIndex != -1) { + delete queryParams['include'][existIncludeIndex]['fromFilter']; + } + // if association not exist, create it if (existIncludeIndex == -1) { // association not exists diff --git a/packages/core/database/src/relation-repository/multiple-relation-repository.ts b/packages/core/database/src/relation-repository/multiple-relation-repository.ts index f368e93f5c..a99813fb44 100644 --- a/packages/core/database/src/relation-repository/multiple-relation-repository.ts +++ b/packages/core/database/src/relation-repository/multiple-relation-repository.ts @@ -1,5 +1,4 @@ -import { omit } from 'lodash'; -import { MultiAssociationAccessors, Op, Sequelize, Transaction, Transactionable } from 'sequelize'; +import { MultiAssociationAccessors, Sequelize, Transaction, Transactionable } from 'sequelize'; import { CommonFindOptions, CountOptions, @@ -14,7 +13,7 @@ import { import { updateModelByValues } from '../update-associations'; import { UpdateGuard } from '../update-guard'; import { RelationRepository, transaction } from './relation-repository'; -import { handleAppendsQuery } from '../utils'; +import { EagerLoadingTree } from '../eager-loading/eager-loading-tree'; export type FindAndCountOptions = CommonFindOptions; @@ -61,23 +60,19 @@ export abstract class MultipleRelationRepository extends RelationRepository { return []; } - return await handleAppendsQuery({ - templateModel: ids[0].row, - queryPromises: findOptions.include.map((include) => { - return sourceModel[getAccessor]({ - ...omit(findOptions, ['limit', 'offset']), - include: [include], - where: { - [this.targetKey()]: { - [Op.in]: ids.map((id) => id.pk), - }, - }, - transaction, - }).then((rows) => { - return { rows, include }; - }); - }), + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: this.targetModel, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, + rootOrder: findOptions.order, }); + + await eagerLoadingTree.load( + ids.map((i) => i.pk), + transaction, + ); + + return eagerLoadingTree.root.instances; } const data = await sourceModel[getAccessor]({ diff --git a/packages/core/database/src/relation-repository/single-relation-repository.ts b/packages/core/database/src/relation-repository/single-relation-repository.ts index 00c211cd07..8f24519d46 100644 --- a/packages/core/database/src/relation-repository/single-relation-repository.ts +++ b/packages/core/database/src/relation-repository/single-relation-repository.ts @@ -3,8 +3,8 @@ import { SingleAssociationAccessors, Transactionable } from 'sequelize'; import { Model } from '../model'; import { Appends, Except, Fields, Filter, TargetKey, UpdateOptions } from '../repository'; import { updateModelByValues } from '../update-associations'; -import { handleAppendsQuery } from '../utils'; import { RelationRepository, transaction } from './relation-repository'; +import { EagerLoadingTree } from '../eager-loading/eager-loading-tree'; export interface SingleRelationFindOption extends Transactionable { fields?: Fields; @@ -44,7 +44,7 @@ export abstract class SingleRelationRepository extends RelationRepository { }); } - async find(options?: SingleRelationFindOption): Promise | null> { + async find(options?: SingleRelationFindOption): Promise { const transaction = await this.getTransaction(options); const findOptions = this.buildQueryOptions({ @@ -61,23 +61,21 @@ export abstract class SingleRelationRepository extends RelationRepository { ...findOptions, includeIgnoreAttributes: false, transaction, - attributes: [this.targetKey()], - group: `${this.targetModel.name}.${this.targetKey()}`, + attributes: [this.targetModel.primaryKeyAttribute], + group: `${this.targetModel.name}.${this.targetModel.primaryKeyAttribute}`, }); - const results = await handleAppendsQuery({ - templateModel, - queryPromises: findOptions.include.map((include) => { - return sourceModel[getAccessor]({ - ...findOptions, - include: [include], - }).then((row) => { - return { rows: [row], include }; - }); - }), + if (!templateModel) return null; + + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model: this.targetModel, + rootAttributes: findOptions.attributes, + includeOption: findOptions.include, }); - return results[0]; + await eagerLoadingTree.load([templateModel.get(this.targetModel.primaryKeyAttribute)], transaction); + + return eagerLoadingTree.root.instances[0]; } return await sourceModel[getAccessor]({ diff --git a/packages/core/database/src/repository.ts b/packages/core/database/src/repository.ts index 585866a62a..826a960078 100644 --- a/packages/core/database/src/repository.ts +++ b/packages/core/database/src/repository.ts @@ -1,4 +1,4 @@ -import lodash, { omit } from 'lodash'; +import lodash from 'lodash'; import { Association, BulkCreateOptions, @@ -31,7 +31,7 @@ import { HasOneRepository } from './relation-repository/hasone-repository'; import { RelationRepository } from './relation-repository/relation-repository'; import { updateAssociations, updateModelByValues } from './update-associations'; import { UpdateGuard } from './update-guard'; -import { handleAppendsQuery } from './utils'; +import { EagerLoadingTree } from './eager-loading/eager-loading-tree'; const debug = require('debug')('noco-database'); @@ -190,10 +190,6 @@ class RelationRepositoryBuilder { } } - protected builder() { - return this.builderMap; - } - of(id: string | number): R { if (!this.association) { return; @@ -201,6 +197,10 @@ class RelationRepositoryBuilder { const klass = this.builder()[this.association.associationType]; return new klass(this.collection, this.associationName, id); } + + protected builder() { + return this.builderMap; + } } export interface AggregateOptions { @@ -276,7 +276,6 @@ export class Repository { if (queryOptions.include && queryOptions.include.length > 0) { const filterInclude = queryOptions.include.filter((include) => { @@ -320,6 +319,7 @@ export class Repository id['pk']), - }, - }; - - rows = await handleAppendsQuery({ - queryPromises: opts.include.map((include) => { - const options = { - ...omit(opts, ['limit', 'offset', 'filter']), - include: include, - where, - transaction, - }; - - return model.findAll(options).then((rows) => { - return { rows, include }; - }); - }), - templateModel: templateModel, + // find all rows + const eagerLoadingTree = EagerLoadingTree.buildFromSequelizeOptions({ + model, + rootAttributes: opts.attributes, + includeOption: opts.include, + rootOrder: opts.order, }); + + await eagerLoadingTree.load( + ids.map((i) => i.pk), + transaction, + ); + + rows = eagerLoadingTree.root.instances; } else { rows = await model.findAll({ ...opts, diff --git a/packages/core/database/src/utils.ts b/packages/core/database/src/utils.ts index ad7a32144d..bf6bf428ac 100644 --- a/packages/core/database/src/utils.ts +++ b/packages/core/database/src/utils.ts @@ -1,69 +1,8 @@ import crypto from 'crypto'; import Database from './database'; import { IdentifierError } from './errors/identifier-error'; -import { Model } from './model'; import lodash from 'lodash'; -type HandleAppendsQueryOptions = { - templateModel: any; - queryPromises: Array; -}; - -export async function handleAppendsQuery(options: HandleAppendsQueryOptions) { - const { templateModel, queryPromises } = options; - - if (!templateModel) { - return []; - } - - const primaryKey = templateModel.constructor.primaryKeyAttribute; - - const results = await Promise.all(queryPromises); - - let rows: Array; - - for (const appendedResult of results) { - if (!rows) { - rows = appendedResult.rows; - - if (rows.length == 0) { - return []; - } - - const modelOptions = templateModel['_options']; - for (const row of rows) { - row['_options'] = { - ...row['_options'], - include: modelOptions['include'], - includeNames: modelOptions['includeNames'], - includeMap: modelOptions['includeMap'], - }; - } - continue; - } - - for (let i = 0; i < appendedResult.rows.length; i++) { - const appendingRow = appendedResult.rows[i]; - const key = appendedResult.include.association; - const val = appendingRow.get(key); - - const rowKey = appendingRow.get(primaryKey); - - const targetIndex = rows.findIndex((row) => row.get(primaryKey) === rowKey); - - if (targetIndex === -1) { - throw new Error('target row not found'); - } - - rows[targetIndex].set(key, val, { - raw: true, - }); - } - } - - return rows; -} - export function md5(value: string) { return crypto.createHash('md5').update(value).digest('hex'); } diff --git a/packages/plugins/acl/src/__tests__/role-check.test.ts b/packages/plugins/acl/src/__tests__/role-check.test.ts index 17dbbda61c..bacdbc8aff 100644 --- a/packages/plugins/acl/src/__tests__/role-check.test.ts +++ b/packages/plugins/acl/src/__tests__/role-check.test.ts @@ -23,11 +23,13 @@ describe('role check action', () => { name: 'test', }, }); + const user = await db.getRepository('users').create({ values: { roles: ['test'], }, }); + const userPlugin = app.getPlugin('users') as UsersPlugin; const agent = app.agent().auth( userPlugin.jwtService.sign({ diff --git a/packages/plugins/acl/src/server.ts b/packages/plugins/acl/src/server.ts index 15a694be4e..0070a9621d 100644 --- a/packages/plugins/acl/src/server.ts +++ b/packages/plugins/acl/src/server.ts @@ -129,6 +129,7 @@ export class PluginACL extends Plugin { for (const role of roles) { role.writeToAcl({ acl: this.acl }); + for (const resource of role.get('resources') as RoleResourceModel[]) { await this.writeResourceToACL(resource, null); } diff --git a/packages/plugins/collection-manager/src/__tests__/http-api/collections.test.ts b/packages/plugins/collection-manager/src/__tests__/http-api/collections.test.ts index 3a50c91bd4..d211541e0d 100644 --- a/packages/plugins/collection-manager/src/__tests__/http-api/collections.test.ts +++ b/packages/plugins/collection-manager/src/__tests__/http-api/collections.test.ts @@ -292,6 +292,8 @@ describe('collections repository', () => { sort: ['-createdAt', '-id'], }); + console.log(JSON.stringify(response1.body.data)); + expect(response1.body.data[0]['id']).toEqual(3); }); diff --git a/packages/plugins/users/src/__tests__/fields.test.ts b/packages/plugins/users/src/__tests__/fields.test.ts index 7c864f0cd1..0efdc57624 100644 --- a/packages/plugins/users/src/__tests__/fields.test.ts +++ b/packages/plugins/users/src/__tests__/fields.test.ts @@ -48,6 +48,7 @@ describe('createdBy/updatedBy', () => { const p2 = await Post.repository.findOne({ appends: ['createdBy', 'updatedBy'], }); + const data = p2.toJSON(); expect(data.createdBy.id).toBe(currentUser.get('id')); expect(data.updatedBy.id).toBe(currentUser.get('id')); diff --git a/packages/plugins/users/src/middlewares/parseToken.ts b/packages/plugins/users/src/middlewares/parseToken.ts index 64d801552d..e0fbd609ef 100644 --- a/packages/plugins/users/src/middlewares/parseToken.ts +++ b/packages/plugins/users/src/middlewares/parseToken.ts @@ -23,12 +23,14 @@ async function findUserByToken(ctx: Context) { ctx.state.currentUserAppends.push(field.name); } } + const user = await ctx.db.getRepository('users').findOne({ appends: ctx.state.currentUserAppends, filter: { id: userId, }, }); + ctx.logger.info(`Current user id: ${userId}`); return user; } catch (error) {