From 10ed38197e6451b56a7cc411ac0e4ec526755ed9 Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Mon, 29 Jul 2024 16:45:14 -0600 Subject: [PATCH] feat: Update TwoFactorAuth utility class to use otpauth library --- App/FeatureSet/BaseAPI/Index.ts | 13 ++ Common/Types/ArrayUtil.ts | 14 ++ Common/Types/Telemetry/TelemetryType.ts | 7 + CommonServer/API/TelemetryAPI.ts | 42 ++--- CommonServer/Infrastructure/GlobalCache.ts | 29 ++++ CommonServer/Services/Index.ts | 9 +- .../Services/TelemetryAttributeService.ts | 83 +++++++++ CommonServer/Utils/TwoFactorAuth.ts | 4 +- .../Global/UserProfile/TwoFactorAuth.tsx | 1 - Ingestor/Service/OTelIngest.ts | 60 +++++++ Model/AnalyticsModels/Index.ts | 2 + Model/AnalyticsModels/TelemetryAttribute.ts | 162 ++++++++++++++++++ 12 files changed, 399 insertions(+), 27 deletions(-) create mode 100644 Common/Types/Telemetry/TelemetryType.ts create mode 100644 CommonServer/Services/TelemetryAttributeService.ts create mode 100644 Model/AnalyticsModels/TelemetryAttribute.ts diff --git a/App/FeatureSet/BaseAPI/Index.ts b/App/FeatureSet/BaseAPI/Index.ts index c40a594b9e..843bbca260 100644 --- a/App/FeatureSet/BaseAPI/Index.ts +++ b/App/FeatureSet/BaseAPI/Index.ts @@ -88,6 +88,10 @@ import LabelService, { import LogService, { LogService as LogServiceType, } from "CommonServer/Services/LogService"; +import TelemetryAttributeService, { + TelemetryAttributeService as TelemetryAttributeServiceType, +} from "CommonServer/Services/TelemetryAttributeService"; + import MetricService, { MetricService as MetricServiceType, } from "CommonServer/Services/MetricService"; @@ -405,6 +409,7 @@ import WorkflowVariable from "Model/Models/WorkflowVariable"; import ProbeOwnerTeam from "Model/Models/ProbeOwnerTeam"; import ProbeOwnerUser from "Model/Models/ProbeOwnerUser"; import ServiceCatalogDependency from "Model/Models/ServiceCatalogDependency"; +import TelemetryAttribute from "Model/AnalyticsModels/TelemetryAttribute"; const BaseAPIFeatureSet: FeatureSet = { init: async (): Promise => { @@ -412,6 +417,14 @@ const BaseAPIFeatureSet: FeatureSet = { const APP_NAME: string = "api"; + app.use( + `/${APP_NAME.toLocaleLowerCase()}`, + new BaseAnalyticsAPI( + TelemetryAttribute, + TelemetryAttributeService, + ).getRouter(), + ); + app.use( `/${APP_NAME.toLocaleLowerCase()}`, new BaseAnalyticsAPI(Log, LogService).getRouter(), diff --git a/Common/Types/ArrayUtil.ts b/Common/Types/ArrayUtil.ts index c56a6b1a70..8deff11e56 100644 --- a/Common/Types/ArrayUtil.ts +++ b/Common/Types/ArrayUtil.ts @@ -1,6 +1,20 @@ import ObjectID from "./ObjectID"; export default class ArrayUtil { + public static mergeStringArrays( + array1: Array, + array2: Array, + ): Array { + return ArrayUtil.removeDuplicates([...array1, ...array2]); + } + + public static isStringArrayEqual( + array1: Array, + array2: Array, + ): boolean { + return ArrayUtil.isEqual(array1, array2); + } + public static removeDuplicates(array: Array): Array { return array.filter((value: any, index: number, self: Array) => { return self.indexOf(value) === index; diff --git a/Common/Types/Telemetry/TelemetryType.ts b/Common/Types/Telemetry/TelemetryType.ts new file mode 100644 index 0000000000..831ca61cbc --- /dev/null +++ b/Common/Types/Telemetry/TelemetryType.ts @@ -0,0 +1,7 @@ +enum TelemetryType { + Metric = "Metric", + Trace = "Trace", + Log = "Log", +} + +export default TelemetryType; diff --git a/CommonServer/API/TelemetryAPI.ts b/CommonServer/API/TelemetryAPI.ts index f05e6f910c..ab9999411a 100644 --- a/CommonServer/API/TelemetryAPI.ts +++ b/CommonServer/API/TelemetryAPI.ts @@ -7,11 +7,10 @@ import Express, { } from "../Utils/Express"; import Response from "../Utils/Response"; import BadDataException from "Common/Types/Exception/BadDataException"; -import JSONFunctions from "Common/Types/JSONFunctions"; import CommonAPI from "./CommonAPI"; import DatabaseCommonInteractionProps from "Common/Types/BaseDatabase/DatabaseCommonInteractionProps"; -import MetricService from "../Services/MetricService"; -import { JSONArray } from "Common/Types/JSON"; +import TelemetryType from "Common/Types/Telemetry/TelemetryType"; +import TelemetryAttributeService from "../Services/TelemetryAttributeService"; const router: ExpressRouter = Express.getRouter(); @@ -19,7 +18,7 @@ router.post( "/telemetry/metrics/get-attributes", UserMiddleware.getUserMiddleware, async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => { - return getAttributes(req, res, next, TelemetryTableName.Metric); + return getAttributes(req, res, next, TelemetryType.Metric); }, ); @@ -27,7 +26,7 @@ router.post( "/telemetry/logs/get-attributes", UserMiddleware.getUserMiddleware, async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => { - return getAttributes(req, res, next, TelemetryTableName.Log); + return getAttributes(req, res, next, TelemetryType.Log); }, ); @@ -35,28 +34,22 @@ router.post( "/telemetry/traces/get-attributes", UserMiddleware.getUserMiddleware, async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => { - return getAttributes(req, res, next, TelemetryTableName.Span); + return getAttributes(req, res, next, TelemetryType.Trace); }, ); -enum TelemetryTableName { - Metric = "Metric", - Span = "Span", - Log = "Log", -} - type GetAttributesFunction = ( req: ExpressRequest, res: ExpressResponse, next: NextFunction, - telemetryTableName: TelemetryTableName, + telemetryType: TelemetryType, ) => Promise; const getAttributes: GetAttributesFunction = async ( req: ExpressRequest, res: ExpressResponse, next: NextFunction, - telemetryTableName: TelemetryTableName, + telemetryType: TelemetryType, ) => { try { const databaseProps: DatabaseCommonInteractionProps = @@ -70,19 +63,22 @@ const getAttributes: GetAttributesFunction = async ( ); } - // Metric Query - - const arrayOfAttributeKeysAsString: string = - await MetricService.executeQuery( - `SELECT groupUniqArrayArray(JSONExtractKeys(attributes)) AS keys FROM ${telemetryTableName} WHERE projectId = '${databaseProps.tenantId?.toString()}'`, + if (!databaseProps.tenantId) { + return Response.sendErrorResponse( + req, + res, + new BadDataException("Invalid Project ID"), ); + } - const arrayOfAttributeKeys: JSONArray = JSONFunctions.parseJSONArray( - arrayOfAttributeKeysAsString, - ); + const attributes: string[] = + await TelemetryAttributeService.fetchAttributes({ + projectId: databaseProps.tenantId, + telemetryType, + }); return Response.sendJsonObjectResponse(req, res, { - attributes: arrayOfAttributeKeys.sort(), + attributes: attributes, }); } catch (err: any) { next(err); diff --git a/CommonServer/Infrastructure/GlobalCache.ts b/CommonServer/Infrastructure/GlobalCache.ts index bbcd40c559..fefe03f73a 100644 --- a/CommonServer/Infrastructure/GlobalCache.ts +++ b/CommonServer/Infrastructure/GlobalCache.ts @@ -27,6 +27,35 @@ export default abstract class GlobalCache { return json; } + public static async getStringArray( + namespace: string, + key: string, + ): Promise { + const value: string | null = await this.getString(namespace, key); + + if (!value) { + return null; + } + + const stringArr: string[] = JSON.parse(value) as string[]; + + if (!Array.isArray(stringArr)) { + throw new BadDataException( + "Expected String Array, but got something else", + ); + } + + return stringArr; + } + + public static async setStringArray( + namespace: string, + key: string, + value: string[], + ): Promise { + await this.setString(namespace, key, JSON.stringify(value)); + } + public static async getJSONArray( namespace: string, key: string, diff --git a/CommonServer/Services/Index.ts b/CommonServer/Services/Index.ts index 3f31d2e7f6..de25d9d79f 100644 --- a/CommonServer/Services/Index.ts +++ b/CommonServer/Services/Index.ts @@ -129,6 +129,7 @@ import WorkflowVariablesService from "./WorkflowVariableService"; import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel"; import CopilotPullRequestService from "./CopilotPullRequestService"; import ServiceCatalogDependencyService from "./ServiceCatalogDependencyService"; +import TelemetryAttributeService from "./TelemetryAttributeService"; const services: Array = [ AcmeCertificateService, @@ -274,6 +275,12 @@ const services: Array = [ export const AnalyticsServices: Array< AnalyticsDatabaseService -> = [LogService, SpanService, MetricService, MonitorMetricsByMinuteService]; +> = [ + LogService, + SpanService, + MetricService, + MonitorMetricsByMinuteService, + TelemetryAttributeService, +]; export default services; diff --git a/CommonServer/Services/TelemetryAttributeService.ts b/CommonServer/Services/TelemetryAttributeService.ts new file mode 100644 index 0000000000..f86c9457cb --- /dev/null +++ b/CommonServer/Services/TelemetryAttributeService.ts @@ -0,0 +1,83 @@ +import TelemetryType from "Common/Types/Telemetry/TelemetryType"; +import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase"; +import AnalyticsDatabaseService from "./AnalyticsDatabaseService"; +import TelemetryAttribute from "Model/AnalyticsModels/TelemetryAttribute"; +import { LIMIT_PER_PROJECT } from "Common/Types/Database/LimitMax"; +import ObjectID from "Common/Types/ObjectID"; + +export class TelemetryAttributeService extends AnalyticsDatabaseService { + public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) { + super({ modelType: TelemetryAttribute, database: clickhouseDatabase }); + } + + public async fetchAttributes(data: { + projectId: ObjectID; + telemetryType: TelemetryType; + }): Promise { + const attributes: TelemetryAttribute[] = await this.findBy({ + query: { + projectId: data.projectId, + telemetryType: data.telemetryType, + }, + select: { + attribute: true, + }, + limit: LIMIT_PER_PROJECT, + skip: 0, + props: { + isRoot: true, + }, + }); + + const dbAttributes: string[] = attributes + .map((attribute: TelemetryAttribute) => { + return attribute.attribute; + }) + .filter((attribute: string | undefined) => { + return Boolean(attribute); + }) as string[]; + + return dbAttributes.sort(); + } + + public async refreshAttributes(data: { + projectId: ObjectID; + telemetryType: TelemetryType; + attributes: string[]; + }): Promise { + const { projectId, telemetryType, attributes } = data; + + // delete existing attributes + await this.deleteBy({ + query: { + projectId, + telemetryType, + }, + props: { + isRoot: true, + }, + }); + + const telemetryAttributes: TelemetryAttribute[] = []; + + // insert new attributes + for (const attribute of attributes) { + const telemetryAttribute: TelemetryAttribute = new TelemetryAttribute(); + + telemetryAttribute.projectId = projectId; + telemetryAttribute.telemetryType = telemetryType; + telemetryAttribute.attribute = attribute; + + telemetryAttributes.push(telemetryAttribute); + } + + await this.createMany({ + items: telemetryAttributes, + props: { + isRoot: true, + }, + }); + } +} + +export default new TelemetryAttributeService(); diff --git a/CommonServer/Utils/TwoFactorAuth.ts b/CommonServer/Utils/TwoFactorAuth.ts index b81a115337..32cdcc58d6 100644 --- a/CommonServer/Utils/TwoFactorAuth.ts +++ b/CommonServer/Utils/TwoFactorAuth.ts @@ -14,7 +14,7 @@ export default class TwoFactorAuth { } public static getLabel(data: { email: Email }): string { - return "OneUptime:" + data.email.toString(); + return data.email.toString(); } public static getTotp(data: { secret: string; email: Email }): OTPAuth.TOTP { @@ -52,7 +52,7 @@ export default class TwoFactorAuth { const totp: OTPAuth.TOTP = this.getTotp({ secret, email }); - const delta: number | null = totp.validate({ token, window: 1 }); + const delta: number | null = totp.validate({ token, window: 3 }); return delta !== null; } diff --git a/Dashboard/src/Pages/Global/UserProfile/TwoFactorAuth.tsx b/Dashboard/src/Pages/Global/UserProfile/TwoFactorAuth.tsx index 17b9ff29d6..39234b3101 100644 --- a/Dashboard/src/Pages/Global/UserProfile/TwoFactorAuth.tsx +++ b/Dashboard/src/Pages/Global/UserProfile/TwoFactorAuth.tsx @@ -117,7 +117,6 @@ const Home: FunctionComponent = (): ReactElement => { selectMoreFields={{ twoFactorOtpUrl: true, }} - deleteButtonText="Reject" columns={[ { field: { diff --git a/Ingestor/Service/OTelIngest.ts b/Ingestor/Service/OTelIngest.ts index 6be520401a..d2cbddddf0 100644 --- a/Ingestor/Service/OTelIngest.ts +++ b/Ingestor/Service/OTelIngest.ts @@ -1,8 +1,12 @@ +import ArrayUtil from "Common/Types/ArrayUtil"; import OneUptimeDate from "Common/Types/Date"; import { JSONArray, JSONObject, JSONValue } from "Common/Types/JSON"; import JSONFunctions from "Common/Types/JSONFunctions"; import ObjectID from "Common/Types/ObjectID"; +import GlobalCache from "CommonServer/Infrastructure/GlobalCache"; import Metric, { AggregationTemporality } from "Model/AnalyticsModels/Metric"; +import TelemetryType from "Common/Types/Telemetry/TelemetryType"; +import TelemetryAttributeService from "CommonServer/Services/TelemetryAttributeService"; export enum OtelAggregationTemporality { Cumulative = "AGGREGATION_TEMPORALITY_CUMULATIVE", @@ -10,6 +14,62 @@ export enum OtelAggregationTemporality { } export default class OTelIngestService { + public static async indexAttributes(data: { + attributes: JSONObject; + projectId: ObjectID; + telemetryType: TelemetryType; + }): Promise { + // index attributes + + const attributes: JSONObject = data.attributes; + const keys: Array = Object.keys(attributes); + + const cacheKey: string = + data.projectId.toString() + "_" + data.telemetryType; + + // get keys from cache + const cacheKeys: string[] = + (await GlobalCache.getStringArray("telemetryAttributesKeys", cacheKey)) || + []; + + let isKeysMissingInCache: boolean = false; + + // check if keys are missing in cache + + for (const key of keys) { + if (!cacheKeys.includes(key)) { + isKeysMissingInCache = true; + break; + } + } + + // merge keys and remove duplicates + if (isKeysMissingInCache) { + const dbKeys: string[] = await TelemetryAttributeService.fetchAttributes({ + projectId: data.projectId, + telemetryType: data.telemetryType, + }); + + const mergedKeys: Array = ArrayUtil.removeDuplicates([ + ...dbKeys, + ...keys, + ...cacheKey, + ]); + + await GlobalCache.setStringArray( + "telemetryAttributesKeys", + cacheKey, + mergedKeys, + ); + + await TelemetryAttributeService.refreshAttributes({ + projectId: data.projectId, + telemetryType: data.telemetryType, + attributes: mergedKeys, + }); + } + } + public static getAttributes(data: { items: JSONArray; telemetryServiceId?: ObjectID; diff --git a/Model/AnalyticsModels/Index.ts b/Model/AnalyticsModels/Index.ts index fe1a7a3acc..d240941b3e 100644 --- a/Model/AnalyticsModels/Index.ts +++ b/Model/AnalyticsModels/Index.ts @@ -3,12 +3,14 @@ import Metric from "./Metric"; import MonitorMetricsByMinute from "./MonitorMetricsByMinute"; import Span from "./Span"; import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel"; +import TelemetryAttribute from "./TelemetryAttribute"; const AnalyticsModels: Array = [ Log, Span, Metric, MonitorMetricsByMinute, + TelemetryAttribute, ]; export default AnalyticsModels; diff --git a/Model/AnalyticsModels/TelemetryAttribute.ts b/Model/AnalyticsModels/TelemetryAttribute.ts new file mode 100644 index 0000000000..2f310f2874 --- /dev/null +++ b/Model/AnalyticsModels/TelemetryAttribute.ts @@ -0,0 +1,162 @@ +import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel"; +import Route from "Common/Types/API/Route"; +import AnalyticsTableEngine from "Common/Types/AnalyticsDatabase/AnalyticsTableEngine"; +import AnalyticsTableColumn from "Common/Types/AnalyticsDatabase/TableColumn"; +import TableColumnType from "Common/Types/AnalyticsDatabase/TableColumnType"; +import TelemetryType from "Common/Types/Telemetry/TelemetryType"; +import ObjectID from "Common/Types/ObjectID"; +import Permission from "Common/Types/Permission"; + +export default class TelemetryAttribute extends AnalyticsBaseModel { + public constructor() { + super({ + tableName: "TelemetryAttribute", + tableEngine: AnalyticsTableEngine.MergeTree, + singularName: "Telemetry Attribute", + pluralName: "Telemetry Attributes", + crudApiPath: new Route("/telemetry-attributes"), + accessControl: { + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadTelemetryServiceTraces, + Permission.ReadTelemetryServiceLog, + Permission.ReadTelemetryServiceMetrics, + ], + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.CreateTelemetryServiceTraces, + Permission.CreateTelemetryServiceLog, + Permission.CreateTelemetryServiceMetrics, + ], + update: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.EditTelemetryServiceTraces, + Permission.EditTelemetryServiceLog, + Permission.EditTelemetryServiceMetrics, + ], + delete: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.DeleteTelemetryServiceTraces, + Permission.DeleteTelemetryServiceLog, + Permission.DeleteTelemetryServiceMetrics, + ], + }, + tableColumns: [ + new AnalyticsTableColumn({ + key: "projectId", + title: "Project ID", + description: "ID of project", + required: true, + type: TableColumnType.ObjectID, + isTenantId: true, + accessControl: { + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadTelemetryServiceTraces, + Permission.ReadTelemetryServiceLog, + Permission.ReadTelemetryServiceMetrics, + ], + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.EditTelemetryServiceTraces, + Permission.EditTelemetryServiceLog, + Permission.EditTelemetryServiceMetrics, + ], + update: [], + }, + }), + + new AnalyticsTableColumn({ + key: "telemetryType", + title: "Telemetry Type", + description: "Type of telemetry", + required: true, + type: TableColumnType.Text, + accessControl: { + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadTelemetryServiceTraces, + Permission.ReadTelemetryServiceLog, + Permission.ReadTelemetryServiceMetrics, + ], + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.EditTelemetryServiceTraces, + Permission.EditTelemetryServiceLog, + Permission.EditTelemetryServiceMetrics, + ], + update: [], + }, + }), + + new AnalyticsTableColumn({ + key: "attribute", + title: "Attribute", + description: "Attribute", + required: true, + type: TableColumnType.Text, + accessControl: { + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadTelemetryServiceTraces, + Permission.ReadTelemetryServiceLog, + Permission.ReadTelemetryServiceMetrics, + ], + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.EditTelemetryServiceTraces, + Permission.EditTelemetryServiceLog, + Permission.EditTelemetryServiceMetrics, + ], + update: [], + }, + }), + ], + primaryKeys: ["projectId", "telemetryType"], + }); + } + + public get projectId(): ObjectID | undefined { + return this.getColumnValue("projectId") as ObjectID | undefined; + } + + public set projectId(v: ObjectID | undefined) { + this.setColumnValue("projectId", v); + } + + public get telemetryType(): TelemetryType | undefined { + return this.getColumnValue("telemetryType") as TelemetryType | undefined; + } + + public set telemetryType(v: TelemetryType | undefined) { + this.setColumnValue("telemetryType", v); + } + + public get attribute(): string | undefined { + return this.getColumnValue("attribute") as string | undefined; + } + + public set attribute(v: string | undefined) { + this.setColumnValue("attribute", v); + } +}