From 0626669b02482c8945d7a251643f12ac72cc5e65 Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Mon, 23 Oct 2023 20:19:20 +0100 Subject: [PATCH] add metrics histogram and sum --- Ingestor/API/OTelIngest.ts | 116 ++++++++++++++++++++++- Model/AnalyticsModels/MetricHistogram.ts | 34 +++++++ Model/AnalyticsModels/MetricSum.ts | 38 ++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) diff --git a/Ingestor/API/OTelIngest.ts b/Ingestor/API/OTelIngest.ts index a6c82beddc..5009106dc1 100644 --- a/Ingestor/API/OTelIngest.ts +++ b/Ingestor/API/OTelIngest.ts @@ -12,6 +12,10 @@ import Span from 'Model/AnalyticsModels/Span'; import Log from 'Model/AnalyticsModels/Log'; import OneUptimeDate from 'Common/Types/Date'; import SpanService from 'CommonServer/Services/SpanService'; +import MetricSumService from 'CommonServer/Services/MetricSumService'; +import MetricHistogramService from 'CommonServer/Services/MetricHistogramService'; +import MetricSum from 'Model/AnalyticsModels/MetricSum'; +import MetricHistogram from 'Model/AnalyticsModels/MetricHistogram'; import LogService from 'CommonServer/Services/LogService'; import ObjectID from 'Common/Types/ObjectID'; import { JSONArray, JSONObject } from 'Common/Types/JSON'; @@ -140,9 +144,117 @@ router.post( next: NextFunction ): Promise => { try { - logger.info('OTel Ingestor API called'); + + req.body = req.body.toJSON(); + + const resourceMetrics: JSONArray = req.body[ + 'resourceMetrics' + ] as JSONArray; + + const dbMetricsSum: Array = []; + const dbMetricsHistogram: Array = []; + + + for(const resourceMetric of resourceMetrics) { + const scopeMetrics: JSONArray = resourceMetric[ + 'scopeMetrics' + ] as JSONArray; + + for(const scopeMetric of scopeMetrics) { + const metrics: JSONArray = scopeMetric[ + 'metrics' + ] as JSONArray; + + for(const metric of metrics) { + const metricName: string = metric['name'] as string; + const metricDescription: string = metric['description'] as string; + + if(metric['sum'] && (metric['sum'] as JSONObject)['dataPoints'] && ((metric['sum'] as JSONObject)['dataPoints'] as JSONArray).length > 0) { + for(const datapoint of ((metric['sum'] as JSONObject)['dataPoints'] as JSONArray)) { + const dbMetricSum: MetricSum = new MetricSum(); + + dbMetricSum.projectId = ObjectID.getZeroObjectID(); + dbMetricSum.serviceId = ObjectID.getZeroObjectID(); + + dbMetricSum.name = metricName; + dbMetricSum.description = metricDescription; + + dbMetricSum.startTimeUnixNano = datapoint['startTimeUnixNano'] as number; + dbMetricSum.startTime = OneUptimeDate.fromUnixNano( + datapoint['startTimeUnixNano'] as number + ); + + dbMetricSum.timeUnixNano = datapoint['timeUnixNano'] as number; + dbMetricSum.time = OneUptimeDate.fromUnixNano( + datapoint['timeUnixNano'] as number + ); + + dbMetricSum.value = datapoint['asInt'] as number; + + dbMetricSum.attributes = OTelIngestService.getKeyValues( + metric['attributes'] as JSONArray + ); + + dbMetricsSum.push(dbMetricSum); + } + } + + if(metric['histogram'] && (metric['histogram'] as JSONObject)['dataPoints'] && ((metric['histogram'] as JSONObject)['dataPoints'] as JSONArray).length > 0) { + for(const datapoint of ((metric['histogram'] as JSONObject)['dataPoints'] as JSONArray)) { + const dbMetricHistogram: MetricHistogram = new MetricHistogram(); + + dbMetricHistogram.projectId = ObjectID.getZeroObjectID(); + dbMetricHistogram.serviceId = ObjectID.getZeroObjectID(); + + dbMetricHistogram.name = metricName; + dbMetricHistogram.description = metricDescription; + + dbMetricHistogram.startTimeUnixNano = datapoint['startTimeUnixNano'] as number; + dbMetricHistogram.startTime = OneUptimeDate.fromUnixNano( + datapoint['startTimeUnixNano'] as number + ); + + dbMetricHistogram.timeUnixNano = datapoint['timeUnixNano'] as number; + dbMetricHistogram.time = OneUptimeDate.fromUnixNano( + datapoint['timeUnixNano'] as number + ); + + dbMetricHistogram.count = datapoint['count'] as number; + dbMetricHistogram.sum = datapoint['sum'] as number; + + dbMetricHistogram.min = datapoint['min'] as number; + dbMetricHistogram.max = datapoint['max'] as number; + + dbMetricHistogram.bucketCounts = datapoint['bucketCounts'] as Array; + dbMetricHistogram.explicitBounds = datapoint['explicitBounds'] as Array; + + + dbMetricHistogram.attributes = OTelIngestService.getKeyValues( + metric['attributes'] as JSONArray + ); + + dbMetricsHistogram.push(dbMetricHistogram); + } + } + } + } + } + + await MetricSumService.createMany({ + items: dbMetricsSum, + props: { + isRoot: true, + }, + }); + + + await MetricHistogramService.createMany({ + items: dbMetricsHistogram, + props: { + isRoot: true, + }, + }); - logger.info(req.body); return Response.sendEmptyResponse(req, res); } catch (err) { diff --git a/Model/AnalyticsModels/MetricHistogram.ts b/Model/AnalyticsModels/MetricHistogram.ts index 5747a7182c..1ebb252b03 100644 --- a/Model/AnalyticsModels/MetricHistogram.ts +++ b/Model/AnalyticsModels/MetricHistogram.ts @@ -31,6 +31,24 @@ export default class Metric extends AnalyticsBaseModel { type: TableColumnType.ObjectID, }), + // add name and description + + new AnalyticsTableColumn({ + key: 'name', + title: 'Name', + description: 'Name of the Metric', + required: true, + type: TableColumnType.Text, + }), + + new AnalyticsTableColumn({ + key: 'description', + title: 'Description', + description: 'Description of the Metric', + required: true, + type: TableColumnType.Text, + }), + new AnalyticsTableColumn({ key: 'time', title: 'Time', @@ -137,6 +155,22 @@ export default class Metric extends AnalyticsBaseModel { return this.getColumnValue('serviceId') as ObjectID | undefined; } + public get name(): string | undefined { + return this.getColumnValue('name') as string | undefined; + } + + public set name(v: string | undefined) { + this.setColumnValue('name', v); + } + + public get description(): string | undefined { + return this.getColumnValue('description') as string | undefined; + } + + public set description(v: string | undefined) { + this.setColumnValue('description', v); + } + public set serviceId(v: ObjectID | undefined) { this.setColumnValue('serviceId', v); } diff --git a/Model/AnalyticsModels/MetricSum.ts b/Model/AnalyticsModels/MetricSum.ts index d49f4a9ab6..558c364ead 100644 --- a/Model/AnalyticsModels/MetricSum.ts +++ b/Model/AnalyticsModels/MetricSum.ts @@ -31,6 +31,27 @@ export default class Metric extends AnalyticsBaseModel { type: TableColumnType.ObjectID, }), + // add name and description + + new AnalyticsTableColumn({ + key: 'name', + title: 'Name', + description: 'Name of the Metric', + required: true, + type: TableColumnType.Text, + }), + + new AnalyticsTableColumn({ + key: 'description', + title: 'Description', + description: 'Description of the Metric', + required: true, + type: TableColumnType.Text, + }), + + + + new AnalyticsTableColumn({ key: 'time', title: 'Time', @@ -109,6 +130,23 @@ export default class Metric extends AnalyticsBaseModel { this.setColumnValue('time', v); } + public get name(): string | undefined { + return this.getColumnValue('name') as string | undefined; + } + + public set name(v: string | undefined) { + this.setColumnValue('name', v); + } + + public get description(): string | undefined { + return this.getColumnValue('description') as string | undefined; + } + + public set description(v: string | undefined) { + this.setColumnValue('description', v); + } + + public get attributes(): Array | undefined { return this.getColumnValue('attributes') as | Array