diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d68b6dbe94..030fd746e6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -103,6 +103,20 @@ services: # https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder - /usr/src/app/node_modules + fetch-resources: + ports: + - '3400:3400' + build: + context: ./fetch-resources + dockerfile: ./Dockerfile.dev + env_file: + - ./fetch-resources/.env + environment: + - CLUSTER_KEY=test + - PORT=3400 + - MONGO_URL=mongodb://mongo:27017/fyipedb + - REALTIME_URL=http://realtime:3300 + # There can only be one probe in developer docker compose. probe: ports: @@ -121,6 +135,7 @@ services: - PROBE_KEY=test-key - IS_SAAS_SERVICE=${IS_SAAS_SERVICE} - CLUSTER_KEY=test + - FETCH_RESOURCES_URL=http://fetch-resources:3400 volumes: - ./probe:/usr/src/app # Use node modules of the container and not host system. diff --git a/docker-compose.yml b/docker-compose.yml index 649e9fb292..0f4399ad73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -135,6 +135,17 @@ services: environment: - CLUSTER_KEY=test - PORT=3300 + fetch-resources: + ports: + - '3400:3400' + build: ./fetch-resources + env_file: + - ./fetch-resources/.env + environment: + - CLUSTER_KEY=test + - PORT=3400 + - MONGO_URL=mongodb://mongo:27017/fyipedb + - REALTIME_URL=http://realtime:3300 probe1: ports: - '3024:3024' @@ -149,6 +160,7 @@ services: - IS_SAAS_SERVICE=${IS_SAAS_SERVICE} - CLUSTER_KEY=test - DATA_INGESTOR_URL=http://data-ingestor:3200 + - FETCH_RESOURCES_URL=http://fetch-resources:3400 depends_on: - backend - data-ingestor diff --git a/fetch-resources/Dockerfile b/fetch-resources/Dockerfile index d9a869f669..47725e2ce9 100755 --- a/fetch-resources/Dockerfile +++ b/fetch-resources/Dockerfile @@ -14,24 +14,11 @@ SHELL ["/bin/bash", "-c"] #SET ENV Variables ENV PRODUCTION=true -RUN OS_ARCHITECTURE="amd64" -RUN if [[ "$(uname -m)" -eq "aarch64" ]] ; then OS_ARCHITECTURE="arm64" ; fi -RUN if [[ "$(uname -m)" -eq "arm64" ]] ; then OS_ARCHITECTURE="arm64" ; fi - RUN mkdir /usr/src RUN mkdir /usr/src/app WORKDIR /usr/src/app -# Install kubectl for kubernetes monitor scanning - -RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(OS_ARCHITECTURE)/kubectl" -RUN chmod +x ./kubectl -RUN mv ./kubectl /usr/local/bin/kubectl && \ - chown root: /usr/local/bin/kubectl - - - # Install app dependencies COPY package*.json /usr/src/app/ RUN npm ci --only=production diff --git a/fetch-resources/Dockerfile.dev b/fetch-resources/Dockerfile.dev index 36c3765027..9d00c0c781 100644 --- a/fetch-resources/Dockerfile.dev +++ b/fetch-resources/Dockerfile.dev @@ -13,18 +13,6 @@ SHELL ["/bin/bash", "-c"] WORKDIR /usr/src/app -RUN OS_ARCHITECTURE="amd64" -RUN if [[ "$(uname -m)" -eq "aarch64" ]] ; then OS_ARCHITECTURE="arm64" ; fi -RUN if [[ "$(uname -m)" -eq "arm64" ]] ; then OS_ARCHITECTURE="arm64" ; fi - -# Install kubectl for kubernetes monitor scanning -RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(OS_ARCHITECTURE)/kubectl" -RUN chmod +x ./kubectl -RUN mv ./kubectl /usr/local/bin/kubectl && \ - chown root: /usr/local/bin/kubectl - - - # Install app dependencies RUN cd /usr/src/app diff --git a/package.json b/package.json index 21f03148dc..7dd89b1d5b 100644 --- a/package.json +++ b/package.json @@ -37,9 +37,9 @@ "build": "docker-compose build", "build-dev": "docker-compose -f docker-compose.dev.yml build $npm_config_services", "prod": "docker-compose up -d", - "dev": "npm run build-dev $npm_config_services && npm run run-enterprise-dev $npm_config_services", + "dev": "npm run build-dev $npm_config_services && npm run start-enterprise-dev $npm_config_services", "saas-dev": "export IS_SAAS_SERVICE=true && npm run dev $npm_config_services", - "run-enterprise-dev": "DOCKER_CLIENT_TIMEOUT=1000 && COMPOSE_HTTP_TIMEOUT=1000 && docker-compose -f docker-compose.dev.yml up -d $npm_config_services", + "start-enterprise-dev": "export DOCKER_CLIENT_TIMEOUT=1000 && export COMPOSE_HTTP_TIMEOUT=1000 && docker-compose -f docker-compose.dev.yml up -d $npm_config_services", "stop-dev": "docker-compose -f docker-compose.dev.yml down --remove-orphans", "staging-test": "cd tests && npm run-script staging-test", "remove-all-containers": "docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) || echo 'No running containers'", diff --git a/probe/index.js b/probe/index.js index 2f2b269894..6b92d0d90a 100755 --- a/probe/index.js +++ b/probe/index.js @@ -116,12 +116,9 @@ app.get(['/probe/version', '/version'], function(req, res) { app.use(Sentry.Handlers.errorHandler()); global.Sentry = Sentry; -// This cron runs every second minute. -cron.schedule('*/2 * * * *', () => { - setTimeout(() => { - Main.runJob(monitorStore); - }, cronMinuteStartTime * 1000); -}); +setTimeout(() => { + Main.runJob(monitorStore); +}, cronMinuteStartTime * 1000); http.listen(app.get('port'), function() { // eslint-disable-next-line diff --git a/probe/package-lock.json b/probe/package-lock.json index b46a8e3098..48ee4d0790 100644 --- a/probe/package-lock.json +++ b/probe/package-lock.json @@ -11,6 +11,7 @@ "license": "ISC", "dependencies": { "@sentry/node": "^6.16.1", + "await-sleep": "^0.0.1", "axios": "^0.25.0", "chrome-launcher": "^0.15.0", "cors": "^2.8.5", @@ -773,6 +774,11 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "node_modules/await-sleep": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/await-sleep/-/await-sleep-0.0.1.tgz", + "integrity": "sha512-H3X3eAxwGpeNIk/yvFOs8g7500Q1YvzrxjSC9TNgLGtjrMFxPwhDdcT34QNs2iGWpZ+5WKkMJdjDoYs+Sw+TaA==" + }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -5273,6 +5279,11 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "await-sleep": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/await-sleep/-/await-sleep-0.0.1.tgz", + "integrity": "sha512-H3X3eAxwGpeNIk/yvFOs8g7500Q1YvzrxjSC9TNgLGtjrMFxPwhDdcT34QNs2iGWpZ+5WKkMJdjDoYs+Sw+TaA==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", diff --git a/probe/package.json b/probe/package.json index 2187c62080..03f46abe6f 100755 --- a/probe/package.json +++ b/probe/package.json @@ -23,6 +23,7 @@ "homepage": "https://gitlab.com/oneuptime/probe#readme", "dependencies": { "@sentry/node": "^6.16.1", + "await-sleep": "^0.0.1", "axios": "^0.25.0", "chrome-launcher": "^0.15.0", "cors": "^2.8.5", diff --git a/probe/workers/main.js b/probe/workers/main.js index c81f8f08f1..951fbb5fc3 100755 --- a/probe/workers/main.js +++ b/probe/workers/main.js @@ -12,7 +12,7 @@ const ErrorService = require('../utils/errorService'); const IncomingHttpRequestMonitors = require('./incomingHttpRequestMonitors'); const KubernetesMonitors = require('./kubernetesMonitors'); const limit = process.env.RESOURCES_LIMIT; -// const ApiService = require('../utils/apiService'); +const asyncSleep = require('await-sleep'); /** * @@ -28,10 +28,24 @@ const limit = process.env.RESOURCES_LIMIT; module.exports = { runJob: async function(monitorStore) { + + monitorStore = {}; + try { + + console.log(`Getting a list of ${limit} monitors`); + let monitors = await getApi('probe/monitors', limit); monitors = JSON.parse(monitors.data); // parse the stringified data + console.log(`Number of Monitors fetched - ${monitors.length} monitors`); + + if(monitors.length === 0){ + // there are no monitors to monitor. Sleep for 30 seconds and then wake up. + console.log("No monitors to monitor. Sleeping for 30 seconds."); + await asyncSleep(30 * 1000); + } + // add monitor to store monitors.forEach(monitor => { if (!monitorStore[monitor._id]) { @@ -39,13 +53,11 @@ module.exports = { } }); - // update all monitors to have scanning set to true - // const monitorIds = monitors.map(monitor => monitor._id); - // await ApiService.addProbeScan(monitorIds); - + // loop over the monitor for (const [key, monitor] of Object.entries(monitorStore)) { try { + console.log(`Currently monitoring: Monitor ID ${key}`); if (monitor.type === 'api') { await ApiMonitors.ping({ monitor }); } else if (monitor.type === 'url') { @@ -71,11 +83,15 @@ module.exports = { } } - // update all monitor scan status to false - // await ApiService.removeProbeScan(monitorIds); + // this is a recursive function. + runJob(monitorStore); + } catch (error) { ErrorService.log('getApi', error); global.Sentry.captureException(error); + + // This is a recursive function. + runJob(monitorStore); } }, };