From e728501ddb2753eb9113c68cf98306baabdd11cb Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Sun, 1 Oct 2023 12:46:51 +0100 Subject: [PATCH] add create by --- .../BaseModel.ts} | 62 ++++++ Common/Types/AnalyticsDatabase/TableColumn.ts | 33 +++ .../Services/AnalyticsDatabaseService.ts | 207 ++++++++++++------ CommonServer/Services/DatabaseService.ts | 21 +- .../Types/AnalyticsDatabase/CreateBy.ts | 2 +- .../Types/AnalyticsDatabase/DeleteBy.ts | 7 + .../Types/AnalyticsDatabase/FindBy.ts | 7 + CommonServer/Types/AnalyticsDatabase/Hooks.ts | 27 +++ .../Types/AnalyticsDatabase/UpdateBy.ts | 7 + CommonServer/Types/Database/Hooks.ts | 27 +++ Llama/Dockerfile.tpl | 23 ++ Model/Models/Log.ts | 2 +- README.md | 4 +- 13 files changed, 333 insertions(+), 96 deletions(-) rename Common/{Models/AnalyticsBaseModel.ts => AnalyticsModels/BaseModel.ts} (68%) create mode 100644 CommonServer/Types/AnalyticsDatabase/DeleteBy.ts create mode 100644 CommonServer/Types/AnalyticsDatabase/FindBy.ts create mode 100644 CommonServer/Types/AnalyticsDatabase/Hooks.ts create mode 100644 CommonServer/Types/AnalyticsDatabase/UpdateBy.ts create mode 100644 CommonServer/Types/Database/Hooks.ts create mode 100644 Llama/Dockerfile.tpl diff --git a/Common/Models/AnalyticsBaseModel.ts b/Common/AnalyticsModels/BaseModel.ts similarity index 68% rename from Common/Models/AnalyticsBaseModel.ts rename to Common/AnalyticsModels/BaseModel.ts index 51baf31015..aa5a8cbea4 100644 --- a/Common/Models/AnalyticsBaseModel.ts +++ b/Common/AnalyticsModels/BaseModel.ts @@ -2,6 +2,7 @@ import TableColumnType from '../Types/BaseDatabase/TableColumnType'; import AnalyticsTableColumn from '../Types/AnalyticsDatabase/TableColumn'; import BadDataException from '../Types/Exception/BadDataException'; import AnalyticsTableEngine from '../Types/AnalyticsDatabase/AnalyticsTableEngine'; +import { JSONValue } from '../Types/JSON'; export default class AnalyticsDataModel { private _tableColumns: Array = []; @@ -101,4 +102,65 @@ export default class AnalyticsDataModel { this.primaryKeys = data.primaryKeys; this.tableColumns = columns; } + + public setColumnValue( + columnName: string, + value: JSONValue + ): void { + if (this.getTableColumn(columnName)) { + return ((this as any)[columnName] = value as any); + } + } + + public getTableColumn(name: string): AnalyticsTableColumn | null { + const column: AnalyticsTableColumn | undefined = this.tableColumns.find( + (column: AnalyticsTableColumn) => { + return column.key === name; + } + ); + + if (!column) { + return null; + } + + return column; + } + + public getTableColumns(): Array { + return this.tableColumns; + } + + public getTenantColumn(): AnalyticsTableColumn | null { + const column: AnalyticsTableColumn | undefined = this.tableColumns.find( + (column: AnalyticsTableColumn) => { + return column.isTenantId; + } + ); + + if (!column) { + return null; + } + + return column; + } + + public getRequiredColumns(): Array { + return this.tableColumns.filter((column: AnalyticsTableColumn) => { + return column.required; + }); + } + + public isDefaultValueColumn(columnName: string): boolean { + const column: AnalyticsTableColumn | null = this.getTableColumn( + columnName + ); + + if (!column) { + return false; + } + + return column.isDefaultValueColumn; + } + + } diff --git a/Common/Types/AnalyticsDatabase/TableColumn.ts b/Common/Types/AnalyticsDatabase/TableColumn.ts index 36b8e5c205..6d687871bc 100644 --- a/Common/Types/AnalyticsDatabase/TableColumn.ts +++ b/Common/Types/AnalyticsDatabase/TableColumn.ts @@ -33,6 +33,16 @@ export default class AnalyticsTableColumn { this._required = v; } + + private _isTenantId : boolean = false; + public get isTenantId() : boolean { + return this._isTenantId; + } + public set isTenantId(v : boolean) { + this._isTenantId = v; + } + + private _type: TableColumnType = TableColumnType.ShortText; public get type(): TableColumnType { return this._type; @@ -40,6 +50,23 @@ export default class AnalyticsTableColumn { public set type(v: TableColumnType) { this._type = v; } + + private _forceGetDefaultValueOnCreate?: (() => Date | string | number | boolean) | undefined; + public get forceGetDefaultValueOnCreate(): (() => Date | string | number | boolean) | undefined { + return this._forceGetDefaultValueOnCreate; + } + public set forceGetDefaultValueOnCreate(v: (() => Date | string | number | boolean) | undefined) { + this._forceGetDefaultValueOnCreate = v; + } + + private _isDefaultValueColumn : boolean = false; + public get isDefaultValueColumn() : boolean { + return this._isDefaultValueColumn; + } + public set isDefaultValueColumn(v : boolean) { + this._isDefaultValueColumn = v; + } + public constructor(data: { key: string; @@ -47,11 +74,17 @@ export default class AnalyticsTableColumn { description: string; required: boolean; type: TableColumnType; + isDefaultValueColumn? : boolean | undefined; + isTenantId?: boolean | undefined; + forceGetDefaultValueOnCreate?: (() => Date | string | number | boolean) | undefined; }) { this.key = data.key; this.title = data.title; this.description = data.description; this.required = data.required; this.type = data.type; + this.isTenantId = data.isTenantId || false; + this.forceGetDefaultValueOnCreate = data.forceGetDefaultValueOnCreate; + this.isDefaultValueColumn = data.isDefaultValueColumn || false; } } diff --git a/CommonServer/Services/AnalyticsDatabaseService.ts b/CommonServer/Services/AnalyticsDatabaseService.ts index dffbd8cdb7..378c43658e 100644 --- a/CommonServer/Services/AnalyticsDatabaseService.ts +++ b/CommonServer/Services/AnalyticsDatabaseService.ts @@ -4,11 +4,14 @@ import ClickhouseDatabase, { ClickhouseClient, } from '../Infrastructure/ClickhouseDatabase'; import BaseService from './BaseService'; -import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import AnalyticsBaseModel from 'Common/AnalyticsModels/BaseModel'; import BadDataException from 'Common/Types/Exception/BadDataException'; import logger from '../Utils/Logger'; import AnalyticsTableColumn from 'Common/Types/AnalyticsDatabase/TableColumn'; -// import CreateBy from "../Types/AnalyticsDatabase/CreateBy"; +import CreateBy from '../Types/AnalyticsDatabase/CreateBy'; +import { OnCreate } from '../Types/AnalyticsDatabase/Hooks'; +import Typeof from 'Common/Types/Typeof'; +import ModelPermission from '../Utils/ModelPermission'; export default class AnalyticsDatabaseService< TBaseModel extends AnalyticsBaseModel @@ -59,6 +62,19 @@ export default class AnalyticsDatabaseService< return statement; } + protected generateDefaultValues(data: TBaseModel): TBaseModel { + const tableColumns: Array = data.getTableColumns(); + + for (const column of tableColumns) { + + if (column.forceGetDefaultValueOnCreate) { + data.setColumnValue(column.key, column.forceGetDefaultValueOnCreate()); + } + } + + return data; + } + public useDefaultDatabase(): void { this.database = ClickhouseAppInstance; this.databaseClient = this.database.getDataSource() as ClickhouseClient; @@ -74,99 +90,120 @@ export default class AnalyticsDatabaseService< }); } - // public async create(createBy: CreateBy): Promise { + protected async onBeforeCreate( + createBy: CreateBy + ): Promise> { + // A place holder method used for overriding. + return Promise.resolve({ + createBy: createBy as CreateBy, + carryForward: undefined, + }); + } - // const onCreate: OnCreate = createBy.props.ignoreHooks - // ? { createBy, carryForward: [] } - // : await this._onBeforeCreate(createBy); + private async _onBeforeCreate( + createBy: CreateBy + ): Promise> { + // Private method that runs before create. + const projectIdColumn: string | null = this.model.getTenantColumn()?.key || null; - // let _createdBy: CreateBy = onCreate.createBy; + if (projectIdColumn && createBy.props.tenantId) { + (createBy.data as any)[projectIdColumn] = createBy.props.tenantId; + } - // const carryForward: any = onCreate.carryForward; + return await this.onBeforeCreate(createBy); + } - // _createdBy = this.generateSlug(_createdBy); + public async create(createBy: CreateBy): Promise { - // let data: TBaseModel = _createdBy.data; + const onCreate: OnCreate = createBy.props.ignoreHooks + ? { createBy, carryForward: [] } + : await this._onBeforeCreate(createBy); - // // add tenantId if present. - // const tenantColumnName: string | null = data.getTenantColumn(); + let _createdBy: CreateBy = onCreate.createBy; - // if (tenantColumnName && _createdBy.props.tenantId) { - // data.setColumnValue(tenantColumnName, _createdBy.props.tenantId); - // } + const carryForward: any = onCreate.carryForward; - // data = this.generateDefaultValues(data); - // data = this.checkRequiredFields(data); + let data: TBaseModel = _createdBy.data; - // if (!this.isValid(data)) { - // throw new BadDataException('Data is not valid'); - // } + // add tenantId if present. + const tenantColumnName: string | null = data.getTenantColumn()?.key || null; - // // check total items by. + if (tenantColumnName && _createdBy.props.tenantId) { + data.setColumnValue(tenantColumnName, _createdBy.props.tenantId); + } - // await this.checkTotalItemsBy(_createdBy); + data = this.generateDefaultValues(data); + data = this.checkRequiredFields(data); - // // Encrypt data - // data = this.encrypt(data); + if (!this.isValid(data)) { + throw new BadDataException('Data is not valid'); + } - // // hash data - // data = await this.hash(data); + // check total items by - // ModelPermission.checkCreatePermissions( - // this.entityType, - // data, - // _createdBy.props - // ); + ModelPermission.checkCreatePermissions( + this.entityType, + data, + _createdBy.props + ); - // createBy.data = data; + createBy.data = data; - // // check uniqueColumns by: - // createBy = await this.checkUniqueColumnBy(createBy); + // check uniqueColumns by: + createBy = await this.checkUniqueColumnBy(createBy); - // // serialize. - // createBy.data = (await this.sanitizeCreateOrUpdate( - // createBy.data, - // createBy.props - // )) as TBaseModel; + // serialize. + createBy.data = (await this.sanitizeCreateOrUpdate( + createBy.data, + createBy.props + )) as TBaseModel; - // try { - // createBy.data = await this.getRepository().save(createBy.data); + try { + createBy.data = await this.getRepository().save(createBy.data); - // if (!createBy.props.ignoreHooks) { - // createBy.data = await this.onCreateSuccess( - // { - // createBy, - // carryForward, - // }, - // createBy.data - // ); - // } + if (!createBy.props.ignoreHooks) { + createBy.data = await this.onCreateSuccess( + { + createBy, + carryForward, + }, + createBy.data + ); + } - // // hit workflow.; - // if (this.getModel().enableWorkflowOn?.create) { - // let tenantId: ObjectID | undefined = createBy.props.tenantId; + // hit workflow.; + if (this.getModel().enableWorkflowOn?.create) { + let tenantId: ObjectID | undefined = createBy.props.tenantId; - // if (!tenantId && this.getModel().getTenantColumn()) { - // tenantId = createBy.data.getValue( - // this.getModel().getTenantColumn()! - // ); - // } + if (!tenantId && this.getModel().getTenantColumn()) { + tenantId = createBy.data.getValue( + this.getModel().getTenantColumn()! + ); + } - // if (tenantId) { - // await this.onTrigger( - // createBy.data.id!, - // tenantId, - // 'on-create' - // ); - // } - // } + if (tenantId) { + await this.onTrigger( + createBy.data.id!, + tenantId, + 'on-create' + ); + } + } - // return createBy.data; - // } catch (error) { - // await this.onCreateError(error as Exception); - // throw this.getException(error as Exception); - // } - // } + return createBy.data; + } catch (error) { + await this.onCreateError(error as Exception); + throw this.getException(error as Exception); + } + } + + protected isValid(data: TBaseModel): boolean { + if (!data) { + throw new BadDataException('Data cannot be null'); + } + + return true; + } public toColumnsCreateStatement(): string { let columns: string = ''; @@ -180,6 +217,32 @@ export default class AnalyticsDatabaseService< return columns; } + protected checkRequiredFields(data: TBaseModel): TBaseModel { + // Check required fields. + + for (const columns of data.getRequiredColumns()) { + const requiredField: string = columns.key; + if (typeof (data as any)[requiredField] === Typeof.Boolean) { + if ( + !(data as any)[requiredField] && + (data as any)[requiredField] !== false && + !data.isDefaultValueColumn(requiredField) + ) { + throw new BadDataException(`${requiredField} is required`); + } + } else if ( + !(data as any)[requiredField] && + !data.isDefaultValueColumn(requiredField) + ) { + throw new BadDataException(`${requiredField} is required`); + } + } + + return data; + } + + + public toColumnType(type: TableColumnType): string { if (type === TableColumnType.ShortText) { return 'String'; diff --git a/CommonServer/Services/DatabaseService.ts b/CommonServer/Services/DatabaseService.ts index 4a151d904d..f39183252b 100644 --- a/CommonServer/Services/DatabaseService.ts +++ b/CommonServer/Services/DatabaseService.ts @@ -53,28 +53,9 @@ import Text from 'Common/Types/Text'; import logger from '../Utils/Logger'; import BaseService from './BaseService'; import { getMaxLengthFromTableColumnType } from 'Common/Types/Database/ColumnLength'; +import { OnCreate, OnDelete, OnFind, OnUpdate } from '../Types/Database/Hooks'; -export type DatabaseTriggerType = 'on-create' | 'on-update' | 'on-delete'; -export interface OnCreate { - createBy: CreateBy; - carryForward: any; -} - -export interface OnFind { - findBy: FindBy; - carryForward: any; -} - -export interface OnDelete { - deleteBy: DeleteBy; - carryForward: any; -} - -export interface OnUpdate { - updateBy: UpdateBy; - carryForward: any; -} class DatabaseService extends BaseService { private postgresDatabase!: PostgresDatabase; diff --git a/CommonServer/Types/AnalyticsDatabase/CreateBy.ts b/CommonServer/Types/AnalyticsDatabase/CreateBy.ts index 9b0800f484..4e6bfbb5c5 100644 --- a/CommonServer/Types/AnalyticsDatabase/CreateBy.ts +++ b/CommonServer/Types/AnalyticsDatabase/CreateBy.ts @@ -1,4 +1,4 @@ -import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import AnalyticsBaseModel from 'Common/AnalyticsModels/BaseModel'; import DatabaseCommonInteractionProps from 'Common/Types/Database/DatabaseCommonInteractionProps'; export default interface CreateBy { diff --git a/CommonServer/Types/AnalyticsDatabase/DeleteBy.ts b/CommonServer/Types/AnalyticsDatabase/DeleteBy.ts new file mode 100644 index 0000000000..98c24985b8 --- /dev/null +++ b/CommonServer/Types/AnalyticsDatabase/DeleteBy.ts @@ -0,0 +1,7 @@ +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import DatabaseCommonInteractionProps from 'Common/Types/Database/DatabaseCommonInteractionProps'; + +export default interface DeleteBy { + data: TBaseModel; + props: DatabaseCommonInteractionProps; +} diff --git a/CommonServer/Types/AnalyticsDatabase/FindBy.ts b/CommonServer/Types/AnalyticsDatabase/FindBy.ts new file mode 100644 index 0000000000..dd24a63ddd --- /dev/null +++ b/CommonServer/Types/AnalyticsDatabase/FindBy.ts @@ -0,0 +1,7 @@ +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import DatabaseCommonInteractionProps from 'Common/Types/Database/DatabaseCommonInteractionProps'; + +export default interface FindBy { + data: TBaseModel; + props: DatabaseCommonInteractionProps; +} diff --git a/CommonServer/Types/AnalyticsDatabase/Hooks.ts b/CommonServer/Types/AnalyticsDatabase/Hooks.ts new file mode 100644 index 0000000000..cf2ffc00f1 --- /dev/null +++ b/CommonServer/Types/AnalyticsDatabase/Hooks.ts @@ -0,0 +1,27 @@ +import BaseModel from "Common/AnalyticsModels/BaseModel"; +import CreateBy from "./CreateBy"; +import DeleteBy from "./DeleteBy"; +import FindBy from "./FindBy"; +import UpdateBy from "./UpdateBy"; + +export type DatabaseTriggerType = 'on-create' | 'on-update' | 'on-delete'; + +export interface OnCreate { + createBy: CreateBy; + carryForward: any; +} + +export interface OnFind { + findBy: FindBy; + carryForward: any; +} + +export interface OnDelete { + deleteBy: DeleteBy; + carryForward: any; +} + +export interface OnUpdate { + updateBy: UpdateBy; + carryForward: any; +} \ No newline at end of file diff --git a/CommonServer/Types/AnalyticsDatabase/UpdateBy.ts b/CommonServer/Types/AnalyticsDatabase/UpdateBy.ts new file mode 100644 index 0000000000..fcd26fd84b --- /dev/null +++ b/CommonServer/Types/AnalyticsDatabase/UpdateBy.ts @@ -0,0 +1,7 @@ +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import DatabaseCommonInteractionProps from 'Common/Types/Database/DatabaseCommonInteractionProps'; + +export default interface UpdateBy { + data: TBaseModel; + props: DatabaseCommonInteractionProps; +} diff --git a/CommonServer/Types/Database/Hooks.ts b/CommonServer/Types/Database/Hooks.ts new file mode 100644 index 0000000000..8cf22f5949 --- /dev/null +++ b/CommonServer/Types/Database/Hooks.ts @@ -0,0 +1,27 @@ +import BaseModel from "Common/Models/BaseModel"; +import CreateBy from "./CreateBy"; +import DeleteBy from "./DeleteBy"; +import FindBy from "./FindBy"; +import UpdateBy from "./UpdateBy"; + +export type DatabaseTriggerType = 'on-create' | 'on-update' | 'on-delete'; + +export interface OnCreate { + createBy: CreateBy; + carryForward: any; +} + +export interface OnFind { + findBy: FindBy; + carryForward: any; +} + +export interface OnDelete { + deleteBy: DeleteBy; + carryForward: any; +} + +export interface OnUpdate { + updateBy: UpdateBy; + carryForward: any; +} \ No newline at end of file diff --git a/Llama/Dockerfile.tpl b/Llama/Dockerfile.tpl new file mode 100644 index 0000000000..d0429d0243 --- /dev/null +++ b/Llama/Dockerfile.tpl @@ -0,0 +1,23 @@ +ARG TAG=latest +FROM continuumio/miniconda3:$TAG + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + git \ + locales \ + sudo \ + build-essential \ + dpkg-dev \ + wget \ + openssh-server \ + nano \ + && rm -rf /var/lib/apt/lists/* + +# Setting up locales + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 + +# Updating conda to the latest version +RUN conda update conda -y + diff --git a/Model/Models/Log.ts b/Model/Models/Log.ts index 188885948b..b38bc00fd0 100644 --- a/Model/Models/Log.ts +++ b/Model/Models/Log.ts @@ -1,4 +1,4 @@ -import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import AnalyticsBaseModel from 'Common/AnalyticsModels/BaseModel'; import AnalyticsTableColumn from 'Common/Types/AnalyticsDatabase/TableColumn'; import TableColumnType from 'Common/Types/BaseDatabase/TableColumnType'; import AnalyticsTableEngine from 'Common/Types/AnalyticsDatabase/AnalyticsTableEngine'; diff --git a/README.md b/README.md index 15019c658f..beb0a6b990 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ If you need advanced features, such as API Access, Advances Workflows, or Advanc ## Installation - [Install on Kubernetes with Helm](https://artifacthub.io/packages/helm/oneuptime/oneuptime) (recommended for production) -- [Install with Docker Compose](https://github.com/OneUptime/oneuptime/blob/master/Docs/Installation/DockerCompose.md) (hobby install, not recommended for production use) -- [Install for local development](https://github.com/OneUptime/oneuptime/blob/master/Docs/Installation/Development.md) +- [Install with Docker Compose](https://github.com/OneUptime/oneuptime/blob/master/Docs/Installation/DockerCompose.md) (hobby install, not recommended for production) +- [Install for Local Development](https://github.com/OneUptime/oneuptime/blob/master/Docs/Installation/Development.md) ## Philosophy