initial data-ingestor project setup

data-ingestor/.dockerignore

@@ -0,0 +1,11 @@
+node_modules/
+.vscode/
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+yarn.lock

data-ingestor/.env

@@ -0,0 +1,6 @@
+MONGO_URL=mongodb://localhost:27017/fyipedb
+CLUSTER_KEY=f414c23b4cdf4e84a6a66ecfd528eff2
+SERVER_URL=http://localhost:3002
+SLACK_ERROR_LOG_WEBHOOK=https://hooks.slack.com/services/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
+SLACK_ERROR_LOG_CHANNEL=fyipe-engineering
+PORT=3200

data-ingestor/.gitignore

@@ -0,0 +1,19 @@
+# See https://help.github.com/ignore-files/ for more about ignoring files.
+
+# dependencies
+#/backend/node_modules
+/kubernetes
+/node_modules
+.idea
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+yarn.lock

data-ingestor/index.js

@@ -0,0 +1,85 @@
+/* eslint-disable no-console */
+const { NODE_ENV } = process.env;
+if (!NODE_ENV || NODE_ENV === 'development') {
+    // Load env vars from /data-ingestor/.env
+    require('custom-env').env();
+}
+
+process.on('exit', () => {
+    // eslint-disable-next-line no-console
+    console.log('Server Shutting Shutdown');
+});
+
+process.on('unhandledRejection', err => {
+    // eslint-disable-next-line no-console
+    console.error('Unhandled rejection in server process occurred');
+    // eslint-disable-next-line no-console
+    console.error(err);
+});
+
+process.on('uncaughtException', err => {
+    // eslint-disable-next-line no-console
+    console.error('Uncaught exception in server process occurred');
+    // eslint-disable-next-line no-console
+    console.error(err);
+});
+
+const express = require('express');
+const app = express();
+const http = require('http').createServer(app);
+const cors = require('cors');
+const { mongoUrl } = require('./utils/config');
+const MongoClient = require('mongodb').MongoClient;
+
+// mongodb
+function getMongoClient() {
+    return new MongoClient(mongoUrl, {
+        useNewUrlParser: true,
+        useUnifiedTopology: true,
+    });
+}
+// setup mongodb connection
+const client = getMongoClient();
+(async function() {
+    try {
+        console.log('connecting to db');
+        await client.connect();
+        console.log('connected to db');
+    } catch (error) {
+        console.log('connection error: ', error);
+    }
+})();
+
+// attach the client to global object
+global.client = client;
+
+app.use(cors());
+
+app.use(function(req, res, next) {
+    if (typeof req.body === 'string') {
+        req.body = JSON.parse(req.body);
+    }
+    res.header('Access-Control-Allow-Credentials', true);
+    res.header('Access-Control-Allow-Origin', req.headers.origin);
+    res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS');
+    res.header(
+        'Access-Control-Allow-Headers',
+        'X-Requested-With, X-HTTP-Method-Override, Content-Type, Accept,Authorization'
+    );
+
+    return next();
+});
+
+// Add limit of 10 MB to avoid "Request Entity too large error"
+// https://stackoverflow.com/questions/19917401/error-request-entity-too-large
+app.use(express.urlencoded({ limit: '10mb', extended: true }));
+app.use(express.json({ limit: '10mb' }));
+
+app.set('port', process.env.PORT || 3200);
+
+http.listen(app.get('port'), function() {
+    // eslint-disable-next-line
+    console.log('data-ingestor server started on port ' + app.get('port'));
+});
+
+module.exports = app;

data-ingestor/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "data-ingestor",
+    "version": "3.0.0",
+    "description": "A project to handle all probe related processing",
+    "main": "index.js",
+    "scripts": {
+        "start": "node --max-old-space-size=4096 index.js",
+        "dev": "cross-env NODE_ENV=development nodemon index.js"
+    },
+    "keywords": [],
+    "author": "",
+    "license": "ISC",
+    "dependencies": {
+        "cors": "^2.8.5",
+        "cross-env": "^7.0.3",
+        "custom-env": "^2.0.1",
+        "express": "^4.17.1",
+        "mongodb": "^4.0.0"
+    },
+    "devDependencies": {
+        "nodemon": "^2.0.12"
+    }
+}

data-ingestor/utils/api.js

@@ -0,0 +1,107 @@
+const axios = require('axios');
+const { clusterKey, serverUrl, dataIngestorVersion } = require('./config');
+
+const _this = {
+    getHeaders: () => {
+        return {
+            'Access-Control-Allow-Origin': '*',
+            Accept: 'application/json',
+            'Content-Type': 'application/json;charset=UTF-8',
+            clusterKey,
+            dataIngestorVersion,
+        };
+    },
+    postApi: (url, data) => {
+        const headers = _this.getHeaders();
+
+        return new Promise((resolve, reject) => {
+            axios({
+                method: 'POST',
+                url: `${serverUrl}/${url}`,
+                headers,
+                data,
+            })
+                .then(function(response) {
+                    resolve(response.data);
+                })
+                .catch(function(error) {
+                    if (error && error.response && error.response.data)
+                        error = error.response.data;
+                    if (error && error.data) {
+                        error = error.data;
+                    }
+                    reject(error);
+                });
+        });
+    },
+
+    getApi: url => {
+        const headers = _this.getHeaders();
+        return new Promise((resolve, reject) => {
+            axios({
+                method: 'GET',
+                url: `${serverUrl}/${url}`,
+                headers,
+            })
+                .then(function(response) {
+                    resolve(response.data);
+                })
+                .catch(function(error) {
+                    if (error && error.response && error.response.data)
+                        error = error.response.data;
+                    if (error && error.data) {
+                        error = error.data;
+                    }
+                    reject(error);
+                });
+        });
+    },
+
+    putApi: (url, data) => {
+        const headers = _this.getHeaders();
+        return new Promise((resolve, reject) => {
+            axios({
+                method: 'PUT',
+                url: `${serverUrl}/${url}`,
+                headers,
+                data,
+            })
+                .then(function(response) {
+                    resolve(response.data);
+                })
+                .catch(function(error) {
+                    if (error && error.response && error.response.data)
+                        error = error.response.data;
+                    if (error && error.data) {
+                        error = error.data;
+                    }
+                    reject(error);
+                });
+        });
+    },
+
+    deleteApi: (url, data) => {
+        const headers = _this.getHeaders();
+        return new Promise((resolve, reject) => {
+            axios({
+                method: 'DELETE',
+                url: `${serverUrl}/${url}`,
+                headers,
+                data,
+            })
+                .then(function(response) {
+                    resolve(response.data);
+                })
+                .catch(function(error) {
+                    if (error && error.response && error.response.data)
+                        error = error.response.data;
+                    if (error && error.data) {
+                        error = error.data;
+                    }
+                    reject(error);
+                });
+        });
+    },
+};
+
+module.exports = _this;

data-ingestor/utils/config.js

@@ -0,0 +1,8 @@
+const packageJson = require('../package.json');
+
+module.exports = {
+    serverUrl: process.env['SERVER_URL'],
+    clusterKey: process.env['CLUSTER_KEY'],
+    dataIngestorVersion: packageJson.version,
+    mongoUrl: process.env['MONGO_URL'] || 'mongodb://localhost:27017/fyipedb',
+};