From 6ef91fd1b731db69ec8b30962c70908488ae83dc Mon Sep 17 00:00:00 2001
From: Simon Larsen <simonlarsen@oneuptime.com>
Date: Tue, 9 Apr 2024 19:24:51 +0100
Subject: [PATCH] Add OpenTelemetry error handling in StartServer.ts

---
 CommonServer/Utils/StartServer.ts | 19 +++++++++++--
 CommonServer/Utils/Telemetry.ts   |  3 +++
 Ingestor/API/OTelIngest.ts        | 44 +++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/CommonServer/Utils/StartServer.ts b/CommonServer/Utils/StartServer.ts
index 9c4b0bef27..93cf2b0550 100644
--- a/CommonServer/Utils/StartServer.ts
+++ b/CommonServer/Utils/StartServer.ts
@@ -118,6 +118,16 @@ app.use((req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
     }
 });
 
+app.use((_req: ExpressRequest, _res: ExpressResponse, next: NextFunction) => {
+    // set span status code to OK by default. If the error occurs, it will be updated in the error handler.
+    const span: api.Span | undefined = api.trace.getSpan(api.context.active());
+    if (span) {
+        span.setStatus({ code: api.SpanStatusCode.OK });
+    }
+
+    next();
+});
+
 type InitFunction = (
     appName: string,
     port?: Port,
@@ -188,10 +198,15 @@ const init: InitFunction = async (
             logger.error(err);
 
             // Mark span as error.
-
             if (err) {
-                const span = api.trace.getSpan(api.context.active());
+                const span: api.Span | undefined = api.trace.getSpan(
+                    api.context.active()
+                );
                 if (span) {
+                    // record exception
+                    span.recordException(err);
+
+                    // set span status code to ERROR
                     span.setStatus({
                         code: api.SpanStatusCode.ERROR,
                         message: err.message,
diff --git a/CommonServer/Utils/Telemetry.ts b/CommonServer/Utils/Telemetry.ts
index 3fa6c8eefc..366b472cc6 100644
--- a/CommonServer/Utils/Telemetry.ts
+++ b/CommonServer/Utils/Telemetry.ts
@@ -19,6 +19,9 @@ import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
 import { Logger, logs } from '@opentelemetry/api-logs';
 import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
 
+// Enable this line to see debug logs
+// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
+
 export default class Telemetry {
     public static sdk: opentelemetry.NodeSDK | null = null;
     public static logger: Logger | null = null;
diff --git a/Ingestor/API/OTelIngest.ts b/Ingestor/API/OTelIngest.ts
index 345e723a56..51e891bd0c 100644
--- a/Ingestor/API/OTelIngest.ts
+++ b/Ingestor/API/OTelIngest.ts
@@ -150,6 +150,50 @@ router.post(
                         dbSpan.endTimeUnixNano = span[
                             'endTimeUnixNano'
                         ] as number;
+
+                        let spanStatusCode: number = 0;
+
+                        if (
+                            span['status'] &&
+                            (span['status'] as JSONObject)?.['code'] &&
+                            typeof (span['status'] as JSONObject)?.['code'] ===
+                                'number'
+                        ) {
+                            spanStatusCode = (span['status'] as JSONObject)?.[
+                                'code'
+                            ] as number;
+                        }
+
+                        if (
+                            span['status'] &&
+                            (span['status'] as JSONObject)?.['code'] &&
+                            typeof (span['status'] as JSONObject)?.['code'] ===
+                                'string'
+                        ) {
+                            if (
+                                (span['status'] as JSONObject)?.['code'] ===
+                                'STATUS_CODE_UNSET'
+                            ) {
+                                spanStatusCode = 0;
+                            } else if (
+                                (span['status'] as JSONObject)?.['code'] ===
+                                'STATUS_CODE_OK'
+                            ) {
+                                spanStatusCode = 1;
+                            } else if (
+                                (span['status'] as JSONObject)?.['code'] ===
+                                'STATUS_CODE_ERROR'
+                            ) {
+                                spanStatusCode = 2;
+                            }
+                        }
+
+                        dbSpan.statusCode = spanStatusCode;
+
+                        dbSpan.statusMessage = (span['status'] as JSONObject)?.[
+                            'message'
+                        ] as string;
+
                         dbSpan.startTime = OneUptimeDate.fromUnixNano(
                             span['startTimeUnixNano'] as number
                         );