get gzip body

.vscode/launch.json

@@ -100,7 +100,7 @@
         {
             "address": "127.0.0.1",
             "localRoot": "${workspaceFolder}/Ingestor",
-            "name": "Probe API: Debug with Docker",
+            "name": "Ingestor: Debug with Docker",
             "port": 9932,
             "remoteRoot": "/usr/src/app",
             "request": "attach",
@@ -125,20 +125,6 @@
             "restart": true,
             "autoAttachChildProcesses": true
         },
-        {
-            "address": "127.0.0.1",
-            "localRoot": "${workspaceFolder}/data-ingestor",
-            "name": "Data Ingestor: Debug with Docker",
-            "port": 9338,
-            "remoteRoot": "/usr/src/app",
-            "request": "attach",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "type": "node",
-            "restart": true,
-            "autoAttachChildProcesses": true
-        },
         {
             "address": "127.0.0.1",
             "localRoot": "${workspaceFolder}/Notification",

CommonServer/Utils/StartServer.ts

@@ -33,6 +33,7 @@ import { DashboardApiRoute } from 'Common/ServiceRoute';
 import HTTPResponse from 'Common/Types/API/HTTPResponse';
 import HTTPErrorResponse from 'Common/Types/API/HTTPErrorResponse';
 import ServerException from 'Common/Types/Exception/ServerException';
+import zlib from 'zlib';
 // import OpenTelemetrySDK from "./OpenTelemetry";
 
 const app: ExpressApplication = Express.getExpressApp();
@@ -41,6 +42,10 @@ app.disable('x-powered-by');
 app.set('port', process.env['PORT']);
 app.set('view engine', 'ejs');
 
+const jsonBodyParserMiddleware = ExpressJson({ limit: '50mb', extended: true }); // 50 MB limit.
+
+const urlEncodedMiddleware = ExpressUrlEncoded({ limit: '50mb', extended: true }); // 50 MB limit.
+
 const logRequest: RequestHandler = (
     req: ExpressRequest,
     _res: ExpressResponse,
@@ -52,17 +57,13 @@ const logRequest: RequestHandler = (
     const method: string = req.method;
     const url: string = req.url;
 
-    const header_info: string = `Request ID: ${
-        (req as OneUptimeRequest).id
-    } -- POD NAME: ${
-        process.env['POD_NAME'] || 'NONE'
-    } -- METHOD: ${method} -- URL: ${url.toString()}`;
+    const header_info: string = `Request ID: ${(req as OneUptimeRequest).id
+        } -- POD NAME: ${process.env['POD_NAME'] || 'NONE'
+        } -- METHOD: ${method} -- URL: ${url.toString()}`;
 
-    const body_info: string = `Request ID: ${
-        (req as OneUptimeRequest).id
-    } -- Request Body: ${
-        req.body ? JSON.stringify(req.body, null, 2) : 'EMPTY'
-    }`;
+    const body_info: string = `Request ID: ${(req as OneUptimeRequest).id
+        } -- Request Body: ${req.body ? JSON.stringify(req.body, null, 2) : 'EMPTY'
+        }`;
 
     logger.info(header_info + '\n ' + body_info);
     next();
@@ -73,6 +74,7 @@ const setDefaultHeaders: RequestHandler = (
     res: ExpressResponse,
     next: NextFunction
 ): void => {
+
     if (typeof req.body === Typeof.String) {
         req.body = JSONFunctions.parse(req.body);
     }
@@ -95,8 +97,34 @@ app.use(setDefaultHeaders);
  * https://stackoverflow.com/questions/19917401/error-request-entity-too-large
  */
 
-app.use(ExpressJson({ limit: '50mb', extended: true }));
-app.use(ExpressUrlEncoded({ limit: '50mb', extended: true }));
+app.use(function (req, res, next) {
+    
+    if (req.headers['content-encoding'] === 'gzip') {
+        var gunzip = zlib.createGunzip();
+        req.pipe(gunzip);
+        var buffer: any = [];
+        gunzip.on('data', function (data) {
+            buffer.push(data.toString());
+        }).on('end', function () {
+            req.body = buffer;
+            next();
+        }).on('error', function (e) {
+            next(e);
+        });
+    } else {
+        jsonBodyParserMiddleware(req, res, next);
+    }
+});
+
+app.use(function (req, res, next) {
+    
+    if (req.headers['content-encoding'] === 'gzip') {
+        next();
+    } else {
+        urlEncodedMiddleware(req, res, next);
+    }
+});
+
 
 app.use(logRequest);
 
@@ -122,10 +150,10 @@ const init: Function = async (
                 const databaseConfig:
                     | HTTPResponse<JSONObject>
                     | HTTPErrorResponse = await API.get<JSONObject>(
-                    URL.fromString(
-                        `http://${DashboardApiHostname}/${DashboardApiRoute}/global-config/vars`
-                    )
-                );
+                        URL.fromString(
+                            `http://${DashboardApiHostname}/${DashboardApiRoute}/global-config/vars`
+                        )
+                    );
 
                 if (databaseConfig instanceof HTTPErrorResponse) {
                     // error getting database config.

Ingestor/API/OTelIngest.ts

@@ -11,7 +11,15 @@ import protobuf from 'protobufjs';
 // Load proto file for OTel
 
 // Create a root namespace
-const root = protobuf.loadSync('CommonServer/ProtoFiles/otel.proto');
+const LogsProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/logs.proto');
+const TracesProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/traces.proto');
+const MetricsProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/metrics.proto');
+
+
+// Lookup the message type
+const LogsData = LogsProto.lookupType('LogsData');
+const TracesData = TracesProto.lookupType('TracesData');
+const MetricsData = MetricsProto.lookupType('MetricsData');
 
 const router: ExpressRouter = Express.getRouter();
 
@@ -27,9 +35,22 @@ router.post(
             logger.info('OTelIngest URL: ', req.url);
 
             if(req.url === '/otel/v1/traces') {
+                const traces = TracesData.decode(req.body);
 
+                logger.info('Traces: ', traces);
             }
 
+            if(req.url === '/otel/v1/logs') {
+                const logs = LogsData.decode(req.body);
+
+                logger.info('Logs: ', logs);
+            }
+
+            if(req.url === '/otel/v1/metrics') {
+                const metrics = MetricsData.decode(req.body);
+
+                logger.info('Metrics: ', metrics);
+            }
 
             // middleware marks the probe as alive.
             // so we don't need to do anything here.

OTelCollector/config.yaml

@@ -11,7 +11,7 @@ receivers:
 exporters:
   otlphttp:
     endpoint: "http://ingestor:3400/otel"
-    headers: {"x-api-key": "****************"}
+    headers: {"x-api-key": "****************", "Content-Type": "application/json"}
 
 service:
   pipelines:

package.json

@@ -58,6 +58,7 @@
         "dev": "npm run config-to-dev && npm run prerun && export $(grep -v '^#' config.env | xargs) && docker compose -f docker-compose.dev.yml up  --remove-orphans  -d $npm_config_services",
         "stop": "export $(grep -v '^#' config.env | xargs) && docker compose down --remove-orphans",
         "down": "npm run stop",
+        "exec": "export $(grep -v '^#' config.env | xargs) && docker compose -f docker-compose.yml exec -it",
         "prune": "docker system prune",
         "remove-all-containers": "docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) || echo 'No running containers'",
         "prepare": "husky install",