mirrors/oneuptime
1
0
Fork 0
mirror of https://github.com/OneUptime/oneuptime synced 2024-11-21 22:59:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

add fetch-resources to docker-compose

Browse Source
This commit is contained in:
Nawaz Dhandala 2022-01-18 16:38:06 +00:00
parent 7ecffc4d94
commit 8b60338201
9 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docker-compose.dev.yml
View File

@ -103,6 +103,20 @@ services:
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
- /usr/src/app/node_modules
fetch-resources:
ports:
- '3400:3400'
build:
context: ./fetch-resources
dockerfile: ./Dockerfile.dev
env_file:
- ./fetch-resources/.env
environment:
- CLUSTER_KEY=test
- PORT=3400
- MONGO_URL=mongodb://mongo:27017/fyipedb
- REALTIME_URL=http://realtime:3300
# There can only be one probe in developer docker compose.
probe:
ports:
@ -121,6 +135,7 @@ services:
- PROBE_KEY=test-key
- IS_SAAS_SERVICE=${IS_SAAS_SERVICE}
- CLUSTER_KEY=test
- FETCH_RESOURCES_URL=http://fetch-resources:3400
volumes:
- ./probe:/usr/src/app
# Use node modules of the container and not host system.

12
docker-compose.yml
View File

@ -135,6 +135,17 @@ services:
environment:
- CLUSTER_KEY=test
- PORT=3300
fetch-resources:
ports:
- '3400:3400'
build: ./fetch-resources
env_file:
- ./fetch-resources/.env
environment:
- CLUSTER_KEY=test
- PORT=3400
- MONGO_URL=mongodb://mongo:27017/fyipedb
- REALTIME_URL=http://realtime:3300
probe1:
ports:
- '3024:3024'
@ -149,6 +160,7 @@ services:
- IS_SAAS_SERVICE=${IS_SAAS_SERVICE}
- CLUSTER_KEY=test
- DATA_INGESTOR_URL=http://data-ingestor:3200
- FETCH_RESOURCES_URL=http://fetch-resources:3400
depends_on:
- backend
- data-ingestor

13
fetch-resources/Dockerfile
View File

@ -14,24 +14,11 @@ SHELL ["/bin/bash", "-c"]
#SET ENV Variables
ENV PRODUCTION=true
RUN OS_ARCHITECTURE="amd64"
RUN if [[ "$(uname -m)" -eq "aarch64" ]] ; then OS_ARCHITECTURE="arm64" ; fi
RUN if [[ "$(uname -m)" -eq "arm64" ]] ; then OS_ARCHITECTURE="arm64" ; fi
RUN mkdir /usr/src
RUN mkdir /usr/src/app
WORKDIR /usr/src/app
# Install kubectl for kubernetes monitor scanning
RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(OS_ARCHITECTURE)/kubectl"
RUN chmod +x ./kubectl
RUN mv ./kubectl /usr/local/bin/kubectl && \
chown root: /usr/local/bin/kubectl
# Install app dependencies
COPY package*.json /usr/src/app/
RUN npm ci --only=production

12
fetch-resources/Dockerfile.dev
View File

@ -13,18 +13,6 @@ SHELL ["/bin/bash", "-c"]
WORKDIR /usr/src/app
RUN OS_ARCHITECTURE="amd64"
RUN if [[ "$(uname -m)" -eq "aarch64" ]] ; then OS_ARCHITECTURE="arm64" ; fi
RUN if [[ "$(uname -m)" -eq "arm64" ]] ; then OS_ARCHITECTURE="arm64" ; fi
# Install kubectl for kubernetes monitor scanning
RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(OS_ARCHITECTURE)/kubectl"
RUN chmod +x ./kubectl
RUN mv ./kubectl /usr/local/bin/kubectl && \
chown root: /usr/local/bin/kubectl
# Install app dependencies
RUN cd /usr/src/app

4
package.json
View File

@ -37,9 +37,9 @@
"build": "docker-compose build",
"build-dev": "docker-compose -f docker-compose.dev.yml build $npm_config_services",
"prod": "docker-compose up -d",
"dev": "npm run build-dev $npm_config_services && npm run run-enterprise-dev $npm_config_services",
"dev": "npm run build-dev $npm_config_services && npm run start-enterprise-dev $npm_config_services",
"saas-dev": "export IS_SAAS_SERVICE=true && npm run dev $npm_config_services",
"run-enterprise-dev": "DOCKER_CLIENT_TIMEOUT=1000 && COMPOSE_HTTP_TIMEOUT=1000 && docker-compose -f docker-compose.dev.yml up -d $npm_config_services",
"start-enterprise-dev": "export DOCKER_CLIENT_TIMEOUT=1000 && export COMPOSE_HTTP_TIMEOUT=1000 && docker-compose -f docker-compose.dev.yml up -d $npm_config_services",
"stop-dev": "docker-compose -f docker-compose.dev.yml down --remove-orphans",
"staging-test": "cd tests && npm run-script staging-test",
"remove-all-containers": "docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) || echo 'No running containers'",

9
probe/index.js
View File

@ -116,12 +116,9 @@ app.get(['/probe/version', '/version'], function(req, res) {
app.use(Sentry.Handlers.errorHandler());
global.Sentry = Sentry;
// This cron runs every second minute.
cron.schedule('*/2 * * * *', () => {
setTimeout(() => {
Main.runJob(monitorStore);
}, cronMinuteStartTime * 1000);
});
setTimeout(() => {
Main.runJob(monitorStore);
}, cronMinuteStartTime * 1000);
http.listen(app.get('port'), function() {
// eslint-disable-next-line

11
probe/package-lock.json generated
View File

@ -11,6 +11,7 @@
"license": "ISC",
"dependencies": {
"@sentry/node": "^6.16.1",
"await-sleep": "^0.0.1",
"axios": "^0.25.0",
"chrome-launcher": "^0.15.0",
"cors": "^2.8.5",
@ -773,6 +774,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"node_modules/await-sleep": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/await-sleep/-/await-sleep-0.0.1.tgz",
"integrity": "sha512-H3X3eAxwGpeNIk/yvFOs8g7500Q1YvzrxjSC9TNgLGtjrMFxPwhDdcT34QNs2iGWpZ+5WKkMJdjDoYs+Sw+TaA=="
},
"node_modules/aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@ -5273,6 +5279,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"await-sleep": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/await-sleep/-/await-sleep-0.0.1.tgz",
"integrity": "sha512-H3X3eAxwGpeNIk/yvFOs8g7500Q1YvzrxjSC9TNgLGtjrMFxPwhDdcT34QNs2iGWpZ+5WKkMJdjDoYs+Sw+TaA=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",

1
probe/package.json
View File

@ -23,6 +23,7 @@
"homepage": "https://gitlab.com/oneuptime/probe#readme",
"dependencies": {
"@sentry/node": "^6.16.1",
"await-sleep": "^0.0.1",
"axios": "^0.25.0",
"chrome-launcher": "^0.15.0",
"cors": "^2.8.5",

30
probe/workers/main.js
View File

@ -12,7 +12,7 @@ const ErrorService = require('../utils/errorService');
const IncomingHttpRequestMonitors = require('./incomingHttpRequestMonitors');
const KubernetesMonitors = require('./kubernetesMonitors');
const limit = process.env.RESOURCES_LIMIT;
// const ApiService = require('../utils/apiService');
const asyncSleep = require('await-sleep');
/**
*
@ -28,10 +28,24 @@ const limit = process.env.RESOURCES_LIMIT;
module.exports = {
runJob: async function(monitorStore) {
monitorStore = {};
try {
console.log(`Getting a list of ${limit} monitors`);
let monitors = await getApi('probe/monitors', limit);
monitors = JSON.parse(monitors.data); // parse the stringified data
console.log(`Number of Monitors fetched - ${monitors.length} monitors`);
if(monitors.length === 0){
// there are no monitors to monitor. Sleep for 30 seconds and then wake up.
console.log("No monitors to monitor. Sleeping for 30 seconds.");
await asyncSleep(30 * 1000);
}
// add monitor to store
monitors.forEach(monitor => {
if (!monitorStore[monitor._id]) {
@ -39,13 +53,11 @@ module.exports = {
}
});
// update all monitors to have scanning set to true
// const monitorIds = monitors.map(monitor => monitor._id);
// await ApiService.addProbeScan(monitorIds);
// loop over the monitor
for (const [key, monitor] of Object.entries(monitorStore)) {
try {
console.log(`Currently monitoring: Monitor ID ${key}`);
if (monitor.type === 'api') {
await ApiMonitors.ping({ monitor });
} else if (monitor.type === 'url') {
@ -71,11 +83,15 @@ module.exports = {
}
}
// update all monitor scan status to false
// await ApiService.removeProbeScan(monitorIds);
// this is a recursive function.
runJob(monitorStore);
} catch (error) {
ErrorService.log('getApi', error);
global.Sentry.captureException(error);
// This is a recursive function.
runJob(monitorStore);
}
},
};