From 26c900d8e29205879ed0a9c13995ef2cd05d5d20 Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Fri, 7 Jun 2024 16:57:24 +0100 Subject: [PATCH] refactor: Update CommonServer/package.json and Ingestor/Service/OTelIngest.ts This code change updates the CommonServer/package.json file to remove the "@opentelemetry/metrics" dependency and adds the "@opentelemetry/sdk-metrics" dependency with version "^1.21.0". It also updates the Ingestor/Service/OTelIngest.ts file to import the "JSONFunctions" module from "Common/Types/JSONFunctions" and use the "flattenObject" function from that module to flatten the final object before returning it. --- CommonServer/Utils/Telemetry.ts | 16 +- CommonServer/package-lock.json | 69 ------- CommonServer/package.json | 1 - .../src/Components/Metrics/MetricVIew.tsx | 4 +- Ingestor/API/OTelIngest.ts | 181 +++++++----------- Ingestor/Service/OTelIngest.ts | 3 +- Model/AnalyticsModels/Metric.ts | 31 +++ 7 files changed, 115 insertions(+), 190 deletions(-) diff --git a/CommonServer/Utils/Telemetry.ts b/CommonServer/Utils/Telemetry.ts index c877bc4c5f..8012ba5f0c 100644 --- a/CommonServer/Utils/Telemetry.ts +++ b/CommonServer/Utils/Telemetry.ts @@ -1,3 +1,4 @@ +import OpenTelemetryAPI, { Meter } from '@opentelemetry/api'; import { Logger, logs } from '@opentelemetry/api-logs'; import { Counter } from '@opentelemetry/api/build/src/metrics/Metric'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; @@ -11,15 +12,15 @@ import { BatchLogRecordProcessor, LoggerProvider, } from '@opentelemetry/sdk-logs'; -import { PeriodicExportingMetricReader, MeterProvider } from '@opentelemetry/sdk-metrics'; +import { + MeterProvider, + PeriodicExportingMetricReader, +} from '@opentelemetry/sdk-metrics'; import * as opentelemetry from '@opentelemetry/sdk-node'; import { SpanExporter } from '@opentelemetry/sdk-trace-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import URL from 'Common/Types/API/URL'; import Dictionary from 'Common/Types/Dictionary'; -import OpenTelemetryAPI, { Meter } from '@opentelemetry/api'; - - // Enable this line to see debug logs // diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); @@ -214,9 +215,10 @@ export default class Telemetry { }): Counter { const { name, description } = data; - const counter = this.getMeter().createCounter(name, { - description: description, - }); + const counter: Counter = + this.getMeter().createCounter(name, { + description: description, + }); return counter; } } diff --git a/CommonServer/package-lock.json b/CommonServer/package-lock.json index 4af9346988..1ee9ed06b1 100644 --- a/CommonServer/package-lock.json +++ b/CommonServer/package-lock.json @@ -18,7 +18,6 @@ "@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1", "@opentelemetry/id-generator-aws-xray": "^1.2.1", - "@opentelemetry/metrics": "^0.24.0", "@opentelemetry/sdk-logs": "^0.49.1", "@opentelemetry/sdk-metrics": "^1.21.0", "@opentelemetry/sdk-node": "^0.48.0", @@ -10078,18 +10077,6 @@ "node": ">=14" } }, - "node_modules/@opentelemetry/api-metrics": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-metrics/-/api-metrics-0.24.0.tgz", - "integrity": "sha512-hdpkMeVlRGTuMshD2ZFaDjA/U0cZTkxUkJFvS/4yOiWfw+kEASmGE+U0/i9lbdQKuCR7X1rXSjbcYumlHcMG+A==", - "deprecated": "Please use @opentelemetry/api >= 1.3.0", - "engines": { - "node": ">=8.0.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.1" - } - }, "node_modules/@opentelemetry/auto-instrumentations-node": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.43.0.tgz", @@ -12267,62 +12254,6 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/metrics": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/metrics/-/metrics-0.24.0.tgz", - "integrity": "sha512-QqmQCzrSuJE+sCOJ2xXNhctWPp/Am9ILs0Y01MDS08PRJoK20akKHM7eC4oU8ZdXphMg8rYgW2w7tY8rqvYnJg==", - "deprecated": "Package renamed to @opentelemetry/sdk-metrics-base", - "dependencies": { - "@opentelemetry/api-metrics": "0.24.0", - "@opentelemetry/core": "0.24.0", - "@opentelemetry/resources": "0.24.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=8.0.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.1" - } - }, - "node_modules/@opentelemetry/metrics/node_modules/@opentelemetry/core": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-0.24.0.tgz", - "integrity": "sha512-KpsfxBbFTZT9zaB4Es/fFLbvSzVl9Io/8UUu/TYl4/HgqkmyVInNlWTgRiKyz9nsHzFpGP1kdZJj+YIut0IFsw==", - "dependencies": { - "@opentelemetry/semantic-conventions": "0.24.0", - "semver": "^7.1.3" - }, - "engines": { - "node": ">=8.5.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.1" - } - }, - "node_modules/@opentelemetry/metrics/node_modules/@opentelemetry/resources": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-0.24.0.tgz", - "integrity": "sha512-uEr2m13IRkjQAjX6fsYqJ21aONCspRvuQunaCl8LbH1NS1Gj82TuRUHF6TM82ulBPK8pU+nrrqXKuky2cMcIzw==", - "dependencies": { - "@opentelemetry/core": "0.24.0", - "@opentelemetry/semantic-conventions": "0.24.0" - }, - "engines": { - "node": ">=8.0.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.1" - } - }, - "node_modules/@opentelemetry/metrics/node_modules/@opentelemetry/semantic-conventions": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-0.24.0.tgz", - "integrity": "sha512-a/szuMQV0Quy0/M7kKdglcbRSoorleyyOwbTNNJ32O+RBN766wbQlMTvdimImTmwYWGr+NJOni1EcC242WlRcA==", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/@opentelemetry/otlp-exporter-base": { "version": "0.48.0", "license": "Apache-2.0", diff --git a/CommonServer/package.json b/CommonServer/package.json index d7a356a912..8ef56c4011 100644 --- a/CommonServer/package.json +++ b/CommonServer/package.json @@ -23,7 +23,6 @@ "@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1", "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1", "@opentelemetry/id-generator-aws-xray": "^1.2.1", - "@opentelemetry/metrics": "^0.24.0", "@opentelemetry/sdk-logs": "^0.49.1", "@opentelemetry/sdk-metrics": "^1.21.0", "@opentelemetry/sdk-node": "^0.48.0", diff --git a/Dashboard/src/Components/Metrics/MetricVIew.tsx b/Dashboard/src/Components/Metrics/MetricVIew.tsx index 5d9dc81868..12ccc31157 100644 --- a/Dashboard/src/Components/Metrics/MetricVIew.tsx +++ b/Dashboard/src/Components/Metrics/MetricVIew.tsx @@ -29,9 +29,9 @@ const MetricView: FunctionComponent = ( serviceId: props.serviceId, }); - const [isLoading, setIsLoading] = React.useState(true); + // const [isLoading, setIsLoading] = React.useState(true); - const [values, setValues] = React.useState([]); + // const [values, setValues] = React.useState([]); useEffect(() => {}, []); diff --git a/Ingestor/API/OTelIngest.ts b/Ingestor/API/OTelIngest.ts index 7222d26953..9fa542ecaa 100644 --- a/Ingestor/API/OTelIngest.ts +++ b/Ingestor/API/OTelIngest.ts @@ -153,7 +153,7 @@ router.post( span['status'] && (span['status'] as JSONObject)?.['code'] && typeof (span['status'] as JSONObject)?.['code'] === - 'number' + 'number' ) { spanStatusCode = (span['status'] as JSONObject)?.[ 'code' @@ -164,7 +164,7 @@ router.post( span['status'] && (span['status'] as JSONObject)?.['code'] && typeof (span['status'] as JSONObject)?.['code'] === - 'string' + 'string' ) { if ( (span['status'] as JSONObject)?.['code'] === @@ -317,191 +317,152 @@ router.post( 'description' ] as string; + const metricUnit: string = metric['unit'] as string; + + const dbMetric: Metric = new Metric(); + + dbMetric.projectId = ( + req as TelemetryRequest + ).projectId; + dbMetric.serviceId = ( + req as TelemetryRequest + ).serviceId; + + dbMetric.name = metricName; + dbMetric.description = metricDescription; + dbMetric.unit = metricUnit; + + dbMetric.attributes = { + ...OTelIngestService.getAttributes( + metric['attributes'] as JSONArray + ), + resource: OTelIngestService.getAttributes( + resourceMetric['attributes'] as JSONArray + ), + }; + if ( metric['sum'] && (metric['sum'] as JSONObject)['dataPoints'] && ( (metric['sum'] as JSONObject)[ - 'dataPoints' + 'dataPoints' ] as JSONArray ).length > 0 ) { for (const datapoint of ( metric['sum'] as JSONObject )['dataPoints'] as JSONArray) { - const dbMetricSum: Metric = new Metric(); - - dbMetricSum.projectId = ( - req as TelemetryRequest - ).projectId; - dbMetricSum.serviceId = ( - req as TelemetryRequest - ).serviceId; - - dbMetricSum.name = metricName; - dbMetricSum.description = metricDescription; - - dbMetricSum.startTimeUnixNano = datapoint[ + dbMetric.startTimeUnixNano = datapoint[ 'startTimeUnixNano' ] as number; - dbMetricSum.startTime = - OneUptimeDate.fromUnixNano( - datapoint['startTimeUnixNano'] as number - ); + dbMetric.startTime = OneUptimeDate.fromUnixNano( + datapoint['startTimeUnixNano'] as number + ); - dbMetricSum.timeUnixNano = datapoint[ + dbMetric.timeUnixNano = datapoint[ 'timeUnixNano' ] as number; - dbMetricSum.time = OneUptimeDate.fromUnixNano( + dbMetric.time = OneUptimeDate.fromUnixNano( datapoint['timeUnixNano'] as number ); - if (Object.keys(datapoint).includes('asInt')) { - - dbMetricSum.value = datapoint[ + dbMetric.value = datapoint[ 'asInt' ] as number; - } else if (Object.keys(datapoint).includes('asDouble')) { - dbMetricSum.value = datapoint[ + } else if ( + Object.keys(datapoint).includes('asDouble') + ) { + dbMetric.value = datapoint[ 'asDouble' ] as number; } - - - - dbMetricSum.attributes = - OTelIngestService.getAttributes( - metric['attributes'] as JSONArray - ); - - dbMetrics.push(dbMetricSum); + dbMetrics.push(dbMetric); } } else if ( metric['gauge'] && (metric['gauge'] as JSONObject)['dataPoints'] && ( (metric['gauge'] as JSONObject)[ - 'dataPoints' + 'dataPoints' ] as JSONArray ).length > 0 ) { for (const datapoint of ( metric['gauge'] as JSONObject )['dataPoints'] as JSONArray) { - const dbMetricGauge: Metric = new Metric(); - - dbMetricGauge.projectId = ( - req as TelemetryRequest - ).projectId; - dbMetricGauge.serviceId = ( - req as TelemetryRequest - ).serviceId; - - dbMetricGauge.name = metricName; - dbMetricGauge.description = metricDescription; - - dbMetricGauge.startTimeUnixNano = datapoint[ + dbMetric.startTimeUnixNano = datapoint[ 'startTimeUnixNano' ] as number; - dbMetricGauge.startTime = - OneUptimeDate.fromUnixNano( - datapoint['startTimeUnixNano'] as number - ); + dbMetric.startTime = OneUptimeDate.fromUnixNano( + datapoint['startTimeUnixNano'] as number + ); - dbMetricGauge.timeUnixNano = datapoint[ + dbMetric.timeUnixNano = datapoint[ 'timeUnixNano' ] as number; - dbMetricGauge.time = OneUptimeDate.fromUnixNano( + dbMetric.time = OneUptimeDate.fromUnixNano( datapoint['timeUnixNano'] as number ); - if (Object.keys(datapoint).includes('asDouble')) { - dbMetricGauge.value = datapoint[ + if ( + Object.keys(datapoint).includes('asDouble') + ) { + dbMetric.value = datapoint[ 'asDouble' ] as number; - } else if (Object.keys(datapoint).includes('asInt')) { - dbMetricGauge.value = datapoint[ + } else if ( + Object.keys(datapoint).includes('asInt') + ) { + dbMetric.value = datapoint[ 'asInt' ] as number; } - dbMetricGauge.attributes = - OTelIngestService.getAttributes( - metric['attributes'] as JSONArray - ); - - dbMetrics.push(dbMetricGauge); + dbMetrics.push(dbMetric); } } else if ( metric['histogram'] && (metric['histogram'] as JSONObject)['dataPoints'] && ( (metric['histogram'] as JSONObject)[ - 'dataPoints' + 'dataPoints' ] as JSONArray ).length > 0 ) { for (const datapoint of ( metric['histogram'] as JSONObject )['dataPoints'] as JSONArray) { - const dbMetricHistogram: Metric = new Metric(); - - dbMetricHistogram.projectId = ( - req as TelemetryRequest - ).projectId; - dbMetricHistogram.serviceId = ( - req as TelemetryRequest - ).serviceId; - - dbMetricHistogram.name = metricName; - dbMetricHistogram.description = - metricDescription; - - dbMetricHistogram.startTimeUnixNano = datapoint[ + dbMetric.startTimeUnixNano = datapoint[ 'startTimeUnixNano' ] as number; - dbMetricHistogram.startTime = - OneUptimeDate.fromUnixNano( - datapoint['startTimeUnixNano'] as number - ); + dbMetric.startTime = OneUptimeDate.fromUnixNano( + datapoint['startTimeUnixNano'] as number + ); - dbMetricHistogram.timeUnixNano = datapoint[ + dbMetric.timeUnixNano = datapoint[ 'timeUnixNano' ] as number; - dbMetricHistogram.time = - OneUptimeDate.fromUnixNano( - datapoint['timeUnixNano'] as number - ); + dbMetric.time = OneUptimeDate.fromUnixNano( + datapoint['timeUnixNano'] as number + ); - dbMetricHistogram.count = datapoint[ - 'count' - ] as number; - dbMetricHistogram.sum = datapoint[ - 'sum' - ] as number; + dbMetric.count = datapoint['count'] as number; + dbMetric.sum = datapoint['sum'] as number; - dbMetricHistogram.min = datapoint[ - 'min' - ] as number; - dbMetricHistogram.max = datapoint[ - 'max' - ] as number; + dbMetric.min = datapoint['min'] as number; + dbMetric.max = datapoint['max'] as number; - dbMetricHistogram.bucketCounts = datapoint[ + dbMetric.bucketCounts = datapoint[ 'bucketCounts' ] as Array; - dbMetricHistogram.explicitBounds = datapoint[ + dbMetric.explicitBounds = datapoint[ 'explicitBounds' ] as Array; - // dbMetricHistogram.attributes = - // OTelIngestService.getKeyValues( - // metric['attributes'] as JSONArray - // ); - - dbMetrics.push(dbMetricHistogram); + dbMetrics.push(dbMetric); } } else { logger.warn('Unknown metric type'); diff --git a/Ingestor/Service/OTelIngest.ts b/Ingestor/Service/OTelIngest.ts index c9da28b1f5..27e27a5d1a 100644 --- a/Ingestor/Service/OTelIngest.ts +++ b/Ingestor/Service/OTelIngest.ts @@ -1,4 +1,5 @@ import { JSONArray, JSONObject, JSONValue } from 'Common/Types/JSON'; +import JSONFunctions from 'Common/Types/JSONFunctions'; export default class OTelIngestService { public static getAttributes(items: JSONArray): JSONObject { @@ -22,6 +23,6 @@ export default class OTelIngestService { } } - return finalObj; + return JSONFunctions.flattenObject(finalObj); } } diff --git a/Model/AnalyticsModels/Metric.ts b/Model/AnalyticsModels/Metric.ts index c48fafac08..6d8d16ef3e 100644 --- a/Model/AnalyticsModels/Metric.ts +++ b/Model/AnalyticsModels/Metric.ts @@ -137,6 +137,29 @@ export default class Metric extends AnalyticsBaseModel { }, }), + new AnalyticsTableColumn({ + key: 'unit', + title: 'Unit', + description: 'Unit of the Metric', + required: true, + type: TableColumnType.Text, + accessControl: { + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadTelemetryServiceLog, + ], + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.CreateTelemetryServiceLog, + ], + update: [], + }, + }), + // this is end time. new AnalyticsTableColumn({ key: 'time', @@ -450,6 +473,14 @@ export default class Metric extends AnalyticsBaseModel { this.setColumnValue('description', v); } + public get unit(): string | undefined { + return this.getColumnValue('unit') as string | undefined; + } + + public set unit(v: string | undefined) { + this.setColumnValue('unit', v); + } + public set serviceId(v: ObjectID | undefined) { this.setColumnValue('serviceId', v); }