feat: Update TwoFactorAuth utility class to use otpauth library

App/FeatureSet/BaseAPI/Index.ts

@@ -88,6 +88,10 @@ import LabelService, {
 import LogService, {
   LogService as LogServiceType,
 } from "CommonServer/Services/LogService";
+import TelemetryAttributeService, {
+  TelemetryAttributeService as TelemetryAttributeServiceType,
+} from "CommonServer/Services/TelemetryAttributeService";
+
 import MetricService, {
   MetricService as MetricServiceType,
 } from "CommonServer/Services/MetricService";
@@ -405,6 +409,7 @@ import WorkflowVariable from "Model/Models/WorkflowVariable";
 import ProbeOwnerTeam from "Model/Models/ProbeOwnerTeam";
 import ProbeOwnerUser from "Model/Models/ProbeOwnerUser";
 import ServiceCatalogDependency from "Model/Models/ServiceCatalogDependency";
+import TelemetryAttribute from "Model/AnalyticsModels/TelemetryAttribute";
 
 const BaseAPIFeatureSet: FeatureSet = {
   init: async (): Promise<void> => {
@@ -412,6 +417,14 @@ const BaseAPIFeatureSet: FeatureSet = {
 
     const APP_NAME: string = "api";
 
+    app.use(
+      `/${APP_NAME.toLocaleLowerCase()}`,
+      new BaseAnalyticsAPI<TelemetryAttribute, TelemetryAttributeServiceType>(
+        TelemetryAttribute,
+        TelemetryAttributeService,
+      ).getRouter(),
+    );
+
     app.use(
       `/${APP_NAME.toLocaleLowerCase()}`,
       new BaseAnalyticsAPI<Log, LogServiceType>(Log, LogService).getRouter(),

Common/Types/ArrayUtil.ts

@@ -1,6 +1,20 @@
 import ObjectID from "./ObjectID";
 
 export default class ArrayUtil {
+  public static mergeStringArrays(
+    array1: Array<string>,
+    array2: Array<string>,
+  ): Array<string> {
+    return ArrayUtil.removeDuplicates([...array1, ...array2]);
+  }
+
+  public static isStringArrayEqual(
+    array1: Array<string>,
+    array2: Array<string>,
+  ): boolean {
+    return ArrayUtil.isEqual(array1, array2);
+  }
+
   public static removeDuplicates(array: Array<any>): Array<any> {
     return array.filter((value: any, index: number, self: Array<any>) => {
       return self.indexOf(value) === index;

Common/Types/Telemetry/TelemetryType.ts

@@ -0,0 +1,7 @@
+enum TelemetryType {
+  Metric = "Metric",
+  Trace = "Trace",
+  Log = "Log",
+}
+
+export default TelemetryType;

CommonServer/API/TelemetryAPI.ts

@@ -7,11 +7,10 @@ import Express, {
 } from "../Utils/Express";
 import Response from "../Utils/Response";
 import BadDataException from "Common/Types/Exception/BadDataException";
-import JSONFunctions from "Common/Types/JSONFunctions";
 import CommonAPI from "./CommonAPI";
 import DatabaseCommonInteractionProps from "Common/Types/BaseDatabase/DatabaseCommonInteractionProps";
-import MetricService from "../Services/MetricService";
-import { JSONArray } from "Common/Types/JSON";
+import TelemetryType from "Common/Types/Telemetry/TelemetryType";
+import TelemetryAttributeService from "../Services/TelemetryAttributeService";
 
 const router: ExpressRouter = Express.getRouter();
 
@@ -19,7 +18,7 @@ router.post(
   "/telemetry/metrics/get-attributes",
   UserMiddleware.getUserMiddleware,
   async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
-    return getAttributes(req, res, next, TelemetryTableName.Metric);
+    return getAttributes(req, res, next, TelemetryType.Metric);
   },
 );
 
@@ -27,7 +26,7 @@ router.post(
   "/telemetry/logs/get-attributes",
   UserMiddleware.getUserMiddleware,
   async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
-    return getAttributes(req, res, next, TelemetryTableName.Log);
+    return getAttributes(req, res, next, TelemetryType.Log);
   },
 );
 
@@ -35,28 +34,22 @@ router.post(
   "/telemetry/traces/get-attributes",
   UserMiddleware.getUserMiddleware,
   async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
-    return getAttributes(req, res, next, TelemetryTableName.Span);
+    return getAttributes(req, res, next, TelemetryType.Trace);
   },
 );
 
-enum TelemetryTableName {
-  Metric = "Metric",
-  Span = "Span",
-  Log = "Log",
-}
-
 type GetAttributesFunction = (
   req: ExpressRequest,
   res: ExpressResponse,
   next: NextFunction,
-  telemetryTableName: TelemetryTableName,
+  telemetryType: TelemetryType,
 ) => Promise<void>;
 
 const getAttributes: GetAttributesFunction = async (
   req: ExpressRequest,
   res: ExpressResponse,
   next: NextFunction,
-  telemetryTableName: TelemetryTableName,
+  telemetryType: TelemetryType,
 ) => {
   try {
     const databaseProps: DatabaseCommonInteractionProps =
@@ -70,19 +63,22 @@ const getAttributes: GetAttributesFunction = async (
       );
     }
 
-    // Metric Query
-
-    const arrayOfAttributeKeysAsString: string =
-      await MetricService.executeQuery(
-        `SELECT groupUniqArrayArray(JSONExtractKeys(attributes)) AS keys FROM ${telemetryTableName} WHERE projectId = '${databaseProps.tenantId?.toString()}'`,
+    if (!databaseProps.tenantId) {
+      return Response.sendErrorResponse(
+        req,
+        res,
+        new BadDataException("Invalid Project ID"),
       );
+    }
 
-    const arrayOfAttributeKeys: JSONArray = JSONFunctions.parseJSONArray(
-      arrayOfAttributeKeysAsString,
-    );
+    const attributes: string[] =
+      await TelemetryAttributeService.fetchAttributes({
+        projectId: databaseProps.tenantId,
+        telemetryType,
+      });
 
     return Response.sendJsonObjectResponse(req, res, {
-      attributes: arrayOfAttributeKeys.sort(),
+      attributes: attributes,
     });
   } catch (err: any) {
     next(err);

CommonServer/Infrastructure/GlobalCache.ts

@@ -27,6 +27,35 @@ export default abstract class GlobalCache {
     return json;
   }
 
+  public static async getStringArray(
+    namespace: string,
+    key: string,
+  ): Promise<string[] | null> {
+    const value: string | null = await this.getString(namespace, key);
+
+    if (!value) {
+      return null;
+    }
+
+    const stringArr: string[] = JSON.parse(value) as string[];
+
+    if (!Array.isArray(stringArr)) {
+      throw new BadDataException(
+        "Expected String Array, but got something else",
+      );
+    }
+
+    return stringArr;
+  }
+
+  public static async setStringArray(
+    namespace: string,
+    key: string,
+    value: string[],
+  ): Promise<void> {
+    await this.setString(namespace, key, JSON.stringify(value));
+  }
+
   public static async getJSONArray(
     namespace: string,
     key: string,

CommonServer/Services/Index.ts

@@ -129,6 +129,7 @@ import WorkflowVariablesService from "./WorkflowVariableService";
 import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel";
 import CopilotPullRequestService from "./CopilotPullRequestService";
 import ServiceCatalogDependencyService from "./ServiceCatalogDependencyService";
+import TelemetryAttributeService from "./TelemetryAttributeService";
 
 const services: Array<BaseService> = [
   AcmeCertificateService,
@@ -274,6 +275,12 @@ const services: Array<BaseService> = [
 
 export const AnalyticsServices: Array<
   AnalyticsDatabaseService<AnalyticsBaseModel>
-> = [LogService, SpanService, MetricService, MonitorMetricsByMinuteService];
+> = [
+  LogService,
+  SpanService,
+  MetricService,
+  MonitorMetricsByMinuteService,
+  TelemetryAttributeService,
+];
 
 export default services;

CommonServer/Services/TelemetryAttributeService.ts

@@ -0,0 +1,83 @@
+import TelemetryType from "Common/Types/Telemetry/TelemetryType";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import TelemetryAttribute from "Model/AnalyticsModels/TelemetryAttribute";
+import { LIMIT_PER_PROJECT } from "Common/Types/Database/LimitMax";
+import ObjectID from "Common/Types/ObjectID";
+
+export class TelemetryAttributeService extends AnalyticsDatabaseService<TelemetryAttribute> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: TelemetryAttribute, database: clickhouseDatabase });
+  }
+
+  public async fetchAttributes(data: {
+    projectId: ObjectID;
+    telemetryType: TelemetryType;
+  }): Promise<string[]> {
+    const attributes: TelemetryAttribute[] = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        telemetryType: data.telemetryType,
+      },
+      select: {
+        attribute: true,
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const dbAttributes: string[] = attributes
+      .map((attribute: TelemetryAttribute) => {
+        return attribute.attribute;
+      })
+      .filter((attribute: string | undefined) => {
+        return Boolean(attribute);
+      }) as string[];
+
+    return dbAttributes.sort();
+  }
+
+  public async refreshAttributes(data: {
+    projectId: ObjectID;
+    telemetryType: TelemetryType;
+    attributes: string[];
+  }): Promise<void> {
+    const { projectId, telemetryType, attributes } = data;
+
+    // delete existing attributes
+    await this.deleteBy({
+      query: {
+        projectId,
+        telemetryType,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const telemetryAttributes: TelemetryAttribute[] = [];
+
+    // insert new attributes
+    for (const attribute of attributes) {
+      const telemetryAttribute: TelemetryAttribute = new TelemetryAttribute();
+
+      telemetryAttribute.projectId = projectId;
+      telemetryAttribute.telemetryType = telemetryType;
+      telemetryAttribute.attribute = attribute;
+
+      telemetryAttributes.push(telemetryAttribute);
+    }
+
+    await this.createMany({
+      items: telemetryAttributes,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+}
+
+export default new TelemetryAttributeService();

CommonServer/Utils/TwoFactorAuth.ts

@@ -14,7 +14,7 @@ export default class TwoFactorAuth {
   }
 
   public static getLabel(data: { email: Email }): string {
-    return "OneUptime:" + data.email.toString();
+    return data.email.toString();
   }
 
   public static getTotp(data: { secret: string; email: Email }): OTPAuth.TOTP {
@@ -52,7 +52,7 @@ export default class TwoFactorAuth {
 
     const totp: OTPAuth.TOTP = this.getTotp({ secret, email });
 
-    const delta: number | null = totp.validate({ token, window: 1 });
+    const delta: number | null = totp.validate({ token, window: 3 });
 
     return delta !== null;
   }

Dashboard/src/Pages/Global/UserProfile/TwoFactorAuth.tsx

@@ -117,7 +117,6 @@ const Home: FunctionComponent<PageComponentProps> = (): ReactElement => {
           selectMoreFields={{
             twoFactorOtpUrl: true,
           }}
-          deleteButtonText="Reject"
           columns={[
             {
               field: {

Ingestor/Service/OTelIngest.ts

@@ -1,8 +1,12 @@
+import ArrayUtil from "Common/Types/ArrayUtil";
 import OneUptimeDate from "Common/Types/Date";
 import { JSONArray, JSONObject, JSONValue } from "Common/Types/JSON";
 import JSONFunctions from "Common/Types/JSONFunctions";
 import ObjectID from "Common/Types/ObjectID";
+import GlobalCache from "CommonServer/Infrastructure/GlobalCache";
 import Metric, { AggregationTemporality } from "Model/AnalyticsModels/Metric";
+import TelemetryType from "Common/Types/Telemetry/TelemetryType";
+import TelemetryAttributeService from "CommonServer/Services/TelemetryAttributeService";
 
 export enum OtelAggregationTemporality {
   Cumulative = "AGGREGATION_TEMPORALITY_CUMULATIVE",
@@ -10,6 +14,62 @@ export enum OtelAggregationTemporality {
 }
 
 export default class OTelIngestService {
+  public static async indexAttributes(data: {
+    attributes: JSONObject;
+    projectId: ObjectID;
+    telemetryType: TelemetryType;
+  }): Promise<void> {
+    // index attributes
+
+    const attributes: JSONObject = data.attributes;
+    const keys: Array<string> = Object.keys(attributes);
+
+    const cacheKey: string =
+      data.projectId.toString() + "_" + data.telemetryType;
+
+    // get keys from cache
+    const cacheKeys: string[] =
+      (await GlobalCache.getStringArray("telemetryAttributesKeys", cacheKey)) ||
+      [];
+
+    let isKeysMissingInCache: boolean = false;
+
+    // check if keys are missing in cache
+
+    for (const key of keys) {
+      if (!cacheKeys.includes(key)) {
+        isKeysMissingInCache = true;
+        break;
+      }
+    }
+
+    // merge keys and remove duplicates
+    if (isKeysMissingInCache) {
+      const dbKeys: string[] = await TelemetryAttributeService.fetchAttributes({
+        projectId: data.projectId,
+        telemetryType: data.telemetryType,
+      });
+
+      const mergedKeys: Array<string> = ArrayUtil.removeDuplicates([
+        ...dbKeys,
+        ...keys,
+        ...cacheKey,
+      ]);
+
+      await GlobalCache.setStringArray(
+        "telemetryAttributesKeys",
+        cacheKey,
+        mergedKeys,
+      );
+
+      await TelemetryAttributeService.refreshAttributes({
+        projectId: data.projectId,
+        telemetryType: data.telemetryType,
+        attributes: mergedKeys,
+      });
+    }
+  }
+
   public static getAttributes(data: {
     items: JSONArray;
     telemetryServiceId?: ObjectID;

Model/AnalyticsModels/Index.ts

@@ -3,12 +3,14 @@ import Metric from "./Metric";
 import MonitorMetricsByMinute from "./MonitorMetricsByMinute";
 import Span from "./Span";
 import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel";
+import TelemetryAttribute from "./TelemetryAttribute";
 
 const AnalyticsModels: Array<typeof AnalyticsBaseModel> = [
   Log,
   Span,
   Metric,
   MonitorMetricsByMinute,
+  TelemetryAttribute,
 ];
 
 export default AnalyticsModels;

Model/AnalyticsModels/TelemetryAttribute.ts

@@ -0,0 +1,162 @@
+import AnalyticsBaseModel from "Common/AnalyticsModels/BaseModel";
+import Route from "Common/Types/API/Route";
+import AnalyticsTableEngine from "Common/Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableColumn from "Common/Types/AnalyticsDatabase/TableColumn";
+import TableColumnType from "Common/Types/AnalyticsDatabase/TableColumnType";
+import TelemetryType from "Common/Types/Telemetry/TelemetryType";
+import ObjectID from "Common/Types/ObjectID";
+import Permission from "Common/Types/Permission";
+
+export default class TelemetryAttribute extends AnalyticsBaseModel {
+  public constructor() {
+    super({
+      tableName: "TelemetryAttribute",
+      tableEngine: AnalyticsTableEngine.MergeTree,
+      singularName: "Telemetry Attribute",
+      pluralName: "Telemetry Attributes",
+      crudApiPath: new Route("/telemetry-attributes"),
+      accessControl: {
+        read: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.ReadTelemetryServiceTraces,
+          Permission.ReadTelemetryServiceLog,
+          Permission.ReadTelemetryServiceMetrics,
+        ],
+        create: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.CreateTelemetryServiceTraces,
+          Permission.CreateTelemetryServiceLog,
+          Permission.CreateTelemetryServiceMetrics,
+        ],
+        update: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.EditTelemetryServiceTraces,
+          Permission.EditTelemetryServiceLog,
+          Permission.EditTelemetryServiceMetrics,
+        ],
+        delete: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.DeleteTelemetryServiceTraces,
+          Permission.DeleteTelemetryServiceLog,
+          Permission.DeleteTelemetryServiceMetrics,
+        ],
+      },
+      tableColumns: [
+        new AnalyticsTableColumn({
+          key: "projectId",
+          title: "Project ID",
+          description: "ID of project",
+          required: true,
+          type: TableColumnType.ObjectID,
+          isTenantId: true,
+          accessControl: {
+            read: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.ReadTelemetryServiceTraces,
+              Permission.ReadTelemetryServiceLog,
+              Permission.ReadTelemetryServiceMetrics,
+            ],
+            create: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.EditTelemetryServiceTraces,
+              Permission.EditTelemetryServiceLog,
+              Permission.EditTelemetryServiceMetrics,
+            ],
+            update: [],
+          },
+        }),
+
+        new AnalyticsTableColumn({
+          key: "telemetryType",
+          title: "Telemetry Type",
+          description: "Type of telemetry",
+          required: true,
+          type: TableColumnType.Text,
+          accessControl: {
+            read: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.ReadTelemetryServiceTraces,
+              Permission.ReadTelemetryServiceLog,
+              Permission.ReadTelemetryServiceMetrics,
+            ],
+            create: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.EditTelemetryServiceTraces,
+              Permission.EditTelemetryServiceLog,
+              Permission.EditTelemetryServiceMetrics,
+            ],
+            update: [],
+          },
+        }),
+
+        new AnalyticsTableColumn({
+          key: "attribute",
+          title: "Attribute",
+          description: "Attribute",
+          required: true,
+          type: TableColumnType.Text,
+          accessControl: {
+            read: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.ReadTelemetryServiceTraces,
+              Permission.ReadTelemetryServiceLog,
+              Permission.ReadTelemetryServiceMetrics,
+            ],
+            create: [
+              Permission.ProjectOwner,
+              Permission.ProjectAdmin,
+              Permission.ProjectMember,
+              Permission.EditTelemetryServiceTraces,
+              Permission.EditTelemetryServiceLog,
+              Permission.EditTelemetryServiceMetrics,
+            ],
+            update: [],
+          },
+        }),
+      ],
+      primaryKeys: ["projectId", "telemetryType"],
+    });
+  }
+
+  public get projectId(): ObjectID | undefined {
+    return this.getColumnValue("projectId") as ObjectID | undefined;
+  }
+
+  public set projectId(v: ObjectID | undefined) {
+    this.setColumnValue("projectId", v);
+  }
+
+  public get telemetryType(): TelemetryType | undefined {
+    return this.getColumnValue("telemetryType") as TelemetryType | undefined;
+  }
+
+  public set telemetryType(v: TelemetryType | undefined) {
+    this.setColumnValue("telemetryType", v);
+  }
+
+  public get attribute(): string | undefined {
+    return this.getColumnValue("attribute") as string | undefined;
+  }
+
+  public set attribute(v: string | undefined) {
+    this.setColumnValue("attribute", v);
+  }
+}