diff --git a/.vscode/launch.json b/.vscode/launch.json index 22e5f9f5bd..2bdd3611a0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -100,7 +100,7 @@ { "address": "127.0.0.1", "localRoot": "${workspaceFolder}/Ingestor", - "name": "Probe API: Debug with Docker", + "name": "Ingestor: Debug with Docker", "port": 9932, "remoteRoot": "/usr/src/app", "request": "attach", @@ -125,20 +125,6 @@ "restart": true, "autoAttachChildProcesses": true }, - { - "address": "127.0.0.1", - "localRoot": "${workspaceFolder}/data-ingestor", - "name": "Data Ingestor: Debug with Docker", - "port": 9338, - "remoteRoot": "/usr/src/app", - "request": "attach", - "skipFiles": [ - "/**" - ], - "type": "node", - "restart": true, - "autoAttachChildProcesses": true - }, { "address": "127.0.0.1", "localRoot": "${workspaceFolder}/Notification", diff --git a/CommonServer/Utils/StartServer.ts b/CommonServer/Utils/StartServer.ts index e42a1e1ef0..a7603cad17 100644 --- a/CommonServer/Utils/StartServer.ts +++ b/CommonServer/Utils/StartServer.ts @@ -33,6 +33,7 @@ import { DashboardApiRoute } from 'Common/ServiceRoute'; import HTTPResponse from 'Common/Types/API/HTTPResponse'; import HTTPErrorResponse from 'Common/Types/API/HTTPErrorResponse'; import ServerException from 'Common/Types/Exception/ServerException'; +import zlib from 'zlib'; // import OpenTelemetrySDK from "./OpenTelemetry"; const app: ExpressApplication = Express.getExpressApp(); @@ -41,6 +42,10 @@ app.disable('x-powered-by'); app.set('port', process.env['PORT']); app.set('view engine', 'ejs'); +const jsonBodyParserMiddleware = ExpressJson({ limit: '50mb', extended: true }); // 50 MB limit. + +const urlEncodedMiddleware = ExpressUrlEncoded({ limit: '50mb', extended: true }); // 50 MB limit. + const logRequest: RequestHandler = ( req: ExpressRequest, _res: ExpressResponse, @@ -52,17 +57,13 @@ const logRequest: RequestHandler = ( const method: string = req.method; const url: string = req.url; - const header_info: string = `Request ID: ${ - (req as OneUptimeRequest).id - } -- POD NAME: ${ - process.env['POD_NAME'] || 'NONE' - } -- METHOD: ${method} -- URL: ${url.toString()}`; + const header_info: string = `Request ID: ${(req as OneUptimeRequest).id + } -- POD NAME: ${process.env['POD_NAME'] || 'NONE' + } -- METHOD: ${method} -- URL: ${url.toString()}`; - const body_info: string = `Request ID: ${ - (req as OneUptimeRequest).id - } -- Request Body: ${ - req.body ? JSON.stringify(req.body, null, 2) : 'EMPTY' - }`; + const body_info: string = `Request ID: ${(req as OneUptimeRequest).id + } -- Request Body: ${req.body ? JSON.stringify(req.body, null, 2) : 'EMPTY' + }`; logger.info(header_info + '\n ' + body_info); next(); @@ -73,6 +74,7 @@ const setDefaultHeaders: RequestHandler = ( res: ExpressResponse, next: NextFunction ): void => { + if (typeof req.body === Typeof.String) { req.body = JSONFunctions.parse(req.body); } @@ -95,8 +97,34 @@ app.use(setDefaultHeaders); * https://stackoverflow.com/questions/19917401/error-request-entity-too-large */ -app.use(ExpressJson({ limit: '50mb', extended: true })); -app.use(ExpressUrlEncoded({ limit: '50mb', extended: true })); +app.use(function (req, res, next) { + + if (req.headers['content-encoding'] === 'gzip') { + var gunzip = zlib.createGunzip(); + req.pipe(gunzip); + var buffer: any = []; + gunzip.on('data', function (data) { + buffer.push(data.toString()); + }).on('end', function () { + req.body = buffer; + next(); + }).on('error', function (e) { + next(e); + }); + } else { + jsonBodyParserMiddleware(req, res, next); + } +}); + +app.use(function (req, res, next) { + + if (req.headers['content-encoding'] === 'gzip') { + next(); + } else { + urlEncodedMiddleware(req, res, next); + } +}); + app.use(logRequest); @@ -122,10 +150,10 @@ const init: Function = async ( const databaseConfig: | HTTPResponse | HTTPErrorResponse = await API.get( - URL.fromString( - `http://${DashboardApiHostname}/${DashboardApiRoute}/global-config/vars` - ) - ); + URL.fromString( + `http://${DashboardApiHostname}/${DashboardApiRoute}/global-config/vars` + ) + ); if (databaseConfig instanceof HTTPErrorResponse) { // error getting database config. diff --git a/Ingestor/API/OTelIngest.ts b/Ingestor/API/OTelIngest.ts index 44f396de27..aca86fb70e 100644 --- a/Ingestor/API/OTelIngest.ts +++ b/Ingestor/API/OTelIngest.ts @@ -11,7 +11,15 @@ import protobuf from 'protobufjs'; // Load proto file for OTel // Create a root namespace -const root = protobuf.loadSync('CommonServer/ProtoFiles/otel.proto'); +const LogsProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/logs.proto'); +const TracesProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/traces.proto'); +const MetricsProto = protobuf.loadSync('/usr/src/app/ProtoFiles/Otel/v1/metrics.proto'); + + +// Lookup the message type +const LogsData = LogsProto.lookupType('LogsData'); +const TracesData = TracesProto.lookupType('TracesData'); +const MetricsData = MetricsProto.lookupType('MetricsData'); const router: ExpressRouter = Express.getRouter(); @@ -27,9 +35,22 @@ router.post( logger.info('OTelIngest URL: ', req.url); if(req.url === '/otel/v1/traces') { + const traces = TracesData.decode(req.body); + logger.info('Traces: ', traces); } + if(req.url === '/otel/v1/logs') { + const logs = LogsData.decode(req.body); + + logger.info('Logs: ', logs); + } + + if(req.url === '/otel/v1/metrics') { + const metrics = MetricsData.decode(req.body); + + logger.info('Metrics: ', metrics); + } // middleware marks the probe as alive. // so we don't need to do anything here. diff --git a/OTelCollector/config.yaml b/OTelCollector/config.yaml index 1df00c99ae..0018989214 100644 --- a/OTelCollector/config.yaml +++ b/OTelCollector/config.yaml @@ -11,7 +11,7 @@ receivers: exporters: otlphttp: endpoint: "http://ingestor:3400/otel" - headers: {"x-api-key": "****************"} + headers: {"x-api-key": "****************", "Content-Type": "application/json"} service: pipelines: diff --git a/package.json b/package.json index f3ccb86a8f..2ad1a507a0 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "dev": "npm run config-to-dev && npm run prerun && export $(grep -v '^#' config.env | xargs) && docker compose -f docker-compose.dev.yml up --remove-orphans -d $npm_config_services", "stop": "export $(grep -v '^#' config.env | xargs) && docker compose down --remove-orphans", "down": "npm run stop", + "exec": "export $(grep -v '^#' config.env | xargs) && docker compose -f docker-compose.yml exec -it", "prune": "docker system prune", "remove-all-containers": "docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) || echo 'No running containers'", "prepare": "husky install",