diff --git a/.vscode/launch.json b/.vscode/launch.json index f7574d1e46..ed2277c318 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -71,7 +71,7 @@ }, { "address": "0.0.0.0", - "localRoot": "${workspaceFolder}/realtime", + "localRoot": "${workspaceFolder}/Realtime", "name": "Realtime: Debug with Docker", "port": 9250, "remoteRoot": "/usr/src/app", @@ -83,6 +83,20 @@ "restart": true, "autoAttachChildProcesses": true }, + { + "address": "0.0.0.0", + "localRoot": "${workspaceFolder}/Workers", + "name": "Workers: Debug with Docker", + "port": 9654, + "remoteRoot": "/usr/src/app", + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node", + "restart": true, + "autoAttachChildProcesses": true + }, { "address": "0.0.0.0", "localRoot": "${workspaceFolder}/ProbeAPI", diff --git a/Nginx/default.conf b/Nginx/default.conf index b65f1359d8..fd29efe671 100644 --- a/Nginx/default.conf +++ b/Nginx/default.conf @@ -26,6 +26,10 @@ upstream home { server home:1444 weight=10 max_fails=3 fail_timeout=30s; } +upstream workers { + server workers:3452 weight=10 max_fails=3 fail_timeout=30s; +} + # upstream admin { # server AdminDashboard:3100; # } @@ -164,4 +168,17 @@ server { proxy_set_header Connection "upgrade"; proxy_pass http://dashboard-api/; } + + location /workers { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # enable WebSockets (for ws://sockjs not connected error in the accounts source: https://stackoverflow.com/questions/41381444/websocket-connection-failed-error-during-websocket-handshake-unexpected-respon) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://workers/; + } } \ No newline at end of file diff --git a/Workers/.dockerignore b/Workers/.dockerignore new file mode 100755 index 0000000000..b22a8ad1d0 --- /dev/null +++ b/Workers/.dockerignore @@ -0,0 +1,55 @@ +.git + +node_modules +# See https://help.github.com/ignore-files/ for more about ignoring files. + +# dependencies +/node_modules + +.idea +# testing +/coverage + +# production +/build + +# misc +.DS_Store + +env.js + +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +yarn.lock +Untitled-1 +*.local.sh +*.local.yaml +run +stop + +nohup.out* + +encrypted-credentials.tar +encrypted-credentials/ + +_README.md + +# Important Add production values to gitignore. +values-saas-production.yaml +kubernetes/values-saas-production.yaml + +/private + +/tls_cert.pem +/tls_key.pem +/keys + +temp_readme.md + +tests/coverage + +settings.json + +GoSDK/tester/ \ No newline at end of file diff --git a/Workers/.env b/Workers/.env new file mode 100755 index 0000000000..3c65c8708f --- /dev/null +++ b/Workers/.env @@ -0,0 +1 @@ +PORT=3002 \ No newline at end of file diff --git a/Workers/.gitattributes b/Workers/.gitattributes new file mode 100644 index 0000000000..30ddbbb6cb --- /dev/null +++ b/Workers/.gitattributes @@ -0,0 +1 @@ +*.js text eol=lf \ No newline at end of file diff --git a/Workers/.gitignore b/Workers/.gitignore new file mode 100755 index 0000000000..9ff7be318d --- /dev/null +++ b/Workers/.gitignore @@ -0,0 +1,29 @@ +# See https://help.github.com/ignore-files/ for more about ignoring files. + +# dependencies +#/backend/node_modules +/kubernetes +/node_modules +.idea + +# misc +.DS_Store + + +npm-debug.log* +yarn-debug.log* +yarn-error.log* +yarn.lock + +**/*/paymentService.test.js +apiTest.rest + +application_security_dir +container_security_dir + +# coverage +/coverage +/.nyc_output + +/greenlock.d/config.json +/greenlock.d/config.json.bak \ No newline at end of file diff --git a/Workers/Dockerfile b/Workers/Dockerfile new file mode 100755 index 0000000000..97464ba3c9 --- /dev/null +++ b/Workers/Dockerfile @@ -0,0 +1,68 @@ +# +# OneUptime-Workers Dockerfile +# + +# Pull base image nodejs image. +FROM node:alpine + +# Install bash. +RUN apk update && apk add bash && apk add curl + + +# Install python +RUN apk update && apk add --no-cache --virtual .gyp python3 make g++ + +#Use bash shell by default +SHELL ["/bin/bash", "-c"] +RUN npm install typescript -g + +RUN mkdir /usr/src + +# Install common +RUN mkdir /usr/src/Common +WORKDIR /usr/src/Common +COPY ./Common/package*.json /usr/src/Common/ +RUN npm install +COPY ./Common /usr/src/Common +RUN npm run compile + +# Install Model +RUN mkdir /usr/src/Model +WORKDIR /usr/src/Model +COPY ./Model/package*.json /usr/src/Model/ +RUN npm install +COPY ./Model /usr/src/Model +RUN npm run compile + +# Install CommonServer +RUN mkdir /usr/src/CommonServer +WORKDIR /usr/src/CommonServer +COPY ./CommonServer/package*.json /usr/src/CommonServer/ +RUN npm install +COPY ./CommonServer /usr/src/CommonServer +RUN npm run compile + + +#SET ENV Variables +ENV PRODUCTION=true + +RUN mkdir /usr/src/app + +WORKDIR /usr/src/app + +# Install app dependencies +COPY ./Workers/package*.json /usr/src/app/ +RUN npm install +RUN npm install -g ts-node +RUN npm install -g ts-node-dev + +# Bundle app source +COPY ./Workers /usr/src/app + +# Expose ports. +# - 3452: OneUptime-worker +EXPOSE 3452 + +#Run the app +RUN npm run compile +CMD [ "npm", "start"] diff --git a/Workers/Dockerfile.dev b/Workers/Dockerfile.dev new file mode 100644 index 0000000000..788f4c546a --- /dev/null +++ b/Workers/Dockerfile.dev @@ -0,0 +1,64 @@ +# +# OneUptime-Workers Dockerfile +# + +# Pull base image nodejs image. +FROM node:alpine + +# Install bash. +RUN apk update && apk add bash && apk add curl + + +# Install python +RUN apk update && apk add --no-cache --virtual .gyp python3 make g++ + +#Use bash shell by default +SHELL ["/bin/bash", "-c"] +RUN npm install typescript -g +RUN npm install nodemon -g + +RUN mkdir /usr/src + +# Install common +RUN mkdir /usr/src/Common +WORKDIR /usr/src/Common +COPY ./Common/package*.json /usr/src/Common/ +RUN npm install +COPY ./Common /usr/src/Common +RUN npm run compile + +# Install Model +RUN mkdir /usr/src/Model +WORKDIR /usr/src/Model +COPY ./Model/package*.json /usr/src/Model/ +RUN npm install +COPY ./Model /usr/src/Model +RUN npm run compile + +# Install CommonServer +RUN mkdir /usr/src/CommonServer +WORKDIR /usr/src/CommonServer +COPY ./CommonServer/package*.json /usr/src/CommonServer/ +RUN npm install +COPY ./CommonServer /usr/src/CommonServer +RUN npm run compile + + +WORKDIR /usr/src/app/ +# Install app dependencies +COPY ./Workers/package*.json /usr/src/app/ +RUN npm install +RUN npm install -g ts-node + + +# Bundle app source +COPY ./Workers /usr/src/app + +# Expose ports. +# - 3452: OneUptime-Workers +EXPOSE 3452 +EXPOSE 9229 + +#Run the app +RUN npm run compile +CMD [ "npm", "run", "dev"] diff --git a/Workers/Index.ts b/Workers/Index.ts index e69de29bb2..a25abca533 100644 --- a/Workers/Index.ts +++ b/Workers/Index.ts @@ -0,0 +1,25 @@ +import { PostgresAppInstance } from 'CommonServer/Infrastructure/PostgresDatabase'; +import Redis from 'CommonServer/Infrastructure/Redis'; +import logger from 'CommonServer/Utils/Logger'; +import App from 'CommonServer/Utils/StartServer'; + +const APP_NAME: string = 'workers'; + +const init: Function = async (): Promise => { + try { + // init the app + await App(APP_NAME); + // connect to the database. + await PostgresAppInstance.connect( + PostgresAppInstance.getDatasourceOptions() + ); + + // connect redis + await Redis.connect(); + } catch (err) { + logger.error('App Init Failed:'); + logger.error(err); + } +}; + +init(); \ No newline at end of file diff --git a/Workers/Jobs/ScheduledMaintenance/ChangeStateToOngoing.tsx b/Workers/Jobs/ScheduledMaintenance/ChangeStateToOngoing.tsx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Workers/nodemon.json b/Workers/nodemon.json new file mode 100644 index 0000000000..2885214620 --- /dev/null +++ b/Workers/nodemon.json @@ -0,0 +1,5 @@ +{ + "watch": ["./","../Common", "../CommonServer", "../Model"], + "ext": "ts,json,tsx,env,js,jsx", + "exec": "node --inspect=0.0.0.0:9229 --require ts-node/register Index.ts" +} \ No newline at end of file diff --git a/Workers/package-lock.json b/Workers/package-lock.json index 4fd0fb8642..453088ca78 100644 --- a/Workers/package-lock.json +++ b/Workers/package-lock.json @@ -7,7 +7,219 @@ "": { "name": "workers", "version": "1.0.0", - "license": "MIT" + "license": "MIT", + "dependencies": { + "Common": "file:../Common", + "CommonServer": "file:../CommonServer", + "Model": "file:../Model", + "node-cron": "^3.0.2" + } + }, + "../Common": { + "name": "common", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@types/crypto-js": "^4.1.1", + "@types/nanoid-dictionary": "^4.2.0", + "@types/uuid": "^8.3.4", + "axios": "^0.26.1", + "crypto-js": "^4.1.1", + "moment": "^2.29.2", + "nanoid": "^3.3.2", + "nanoid-dictionary": "^4.3.0", + "process": "^0.11.10", + "reflect-metadata": "^0.1.13", + "slugify": "^1.6.5", + "typeorm": "^0.3.6", + "uuid": "^8.3.2" + }, + "devDependencies": { + "@faker-js/faker": "^6.3.1", + "@types/jest": "^27.5.2", + "@types/node": "^17.0.22", + "jest": "^27.5.1", + "ts-jest": "^27.1.4" + } + }, + "../CommonServer": { + "name": "common-server", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@elastic/elasticsearch": "^8.1.0", + "@opentelemetry/api": "^1.1.0", + "@opentelemetry/auto-instrumentations-node": "^0.31.0", + "@opentelemetry/sdk-node": "^0.30.0", + "@types/ejs": "^3.1.1", + "@types/gridfs-stream": "^0.5.35", + "@types/json2csv": "^5.0.3", + "airtable": "^0.11.3", + "Common": "file:../Common", + "cors": "^2.8.5", + "dotenv": "^16.0.0", + "ejs": "^3.1.8", + "express": "^4.17.3", + "gridfs-stream": "^1.1.1", + "handlebars": "^4.7.7", + "json2csv": "^5.0.7", + "jsonwebtoken": "^8.5.1", + "Model": "file:../Model", + "node-device-detector": "^2.0.0", + "nodemailer": "^6.7.3", + "nodemailer-express-handlebars": "^5.0.0", + "pg": "^8.7.3", + "redis": "^4.2.0", + "socket.io": "^4.4.1", + "typeorm": "^0.3.6", + "typeorm-extension": "^2.1.0", + "winston": "^3.6.0" + }, + "devDependencies": { + "@faker-js/faker": "^6.3.1", + "@types/cors": "^2.8.12", + "@types/express": "^4.17.13", + "@types/jest": "^27.4.1", + "@types/jsonwebtoken": "^8.5.8", + "@types/node": "^17.0.22", + "jest": "^27.5.1", + "ts-jest": "^27.1.4" + } + }, + "../Model": { + "name": "model", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "Common": "file:../Common", + "typeorm": "^0.3.7" + }, + "devDependencies": { + "@types/jest": "^27.4.1", + "@types/node": "^17.0.22", + "jest": "^27.5.1", + "ts-jest": "^27.1.4" + } + }, + "node_modules/Common": { + "resolved": "../Common", + "link": true + }, + "node_modules/CommonServer": { + "resolved": "../CommonServer", + "link": true + }, + "node_modules/Model": { + "resolved": "../Model", + "link": true + }, + "node_modules/node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-iP8l0yGlNpE0e6q1o185yOApANRe47UPbLf4YxfbiNHt/RU5eBcGB/e0oudruheSf+LQeDMezqC5BVAb5wwRcQ==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + } + }, + "dependencies": { + "Common": { + "version": "file:../Common", + "requires": { + "@faker-js/faker": "^6.3.1", + "@types/crypto-js": "^4.1.1", + "@types/jest": "^27.5.2", + "@types/nanoid-dictionary": "^4.2.0", + "@types/node": "^17.0.22", + "@types/uuid": "^8.3.4", + "axios": "^0.26.1", + "crypto-js": "^4.1.1", + "jest": "^27.5.1", + "moment": "^2.29.2", + "nanoid": "^3.3.2", + "nanoid-dictionary": "^4.3.0", + "process": "^0.11.10", + "reflect-metadata": "^0.1.13", + "slugify": "^1.6.5", + "ts-jest": "^27.1.4", + "typeorm": "^0.3.6", + "uuid": "^8.3.2" + } + }, + "CommonServer": { + "version": "file:../CommonServer", + "requires": { + "@elastic/elasticsearch": "^8.1.0", + "@faker-js/faker": "^6.3.1", + "@opentelemetry/api": "^1.1.0", + "@opentelemetry/auto-instrumentations-node": "^0.31.0", + "@opentelemetry/sdk-node": "^0.30.0", + "@types/cors": "^2.8.12", + "@types/ejs": "^3.1.1", + "@types/express": "^4.17.13", + "@types/gridfs-stream": "^0.5.35", + "@types/jest": "^27.4.1", + "@types/json2csv": "^5.0.3", + "@types/jsonwebtoken": "^8.5.8", + "@types/node": "^17.0.22", + "airtable": "^0.11.3", + "Common": "file:../Common", + "cors": "^2.8.5", + "dotenv": "^16.0.0", + "ejs": "^3.1.8", + "express": "^4.17.3", + "gridfs-stream": "^1.1.1", + "handlebars": "^4.7.7", + "jest": "^27.5.1", + "json2csv": "^5.0.7", + "jsonwebtoken": "^8.5.1", + "Model": "file:../Model", + "node-device-detector": "^2.0.0", + "nodemailer": "^6.7.3", + "nodemailer-express-handlebars": "^5.0.0", + "pg": "^8.7.3", + "redis": "^4.2.0", + "socket.io": "^4.4.1", + "ts-jest": "^27.1.4", + "typeorm": "^0.3.6", + "typeorm-extension": "^2.1.0", + "winston": "^3.6.0" + } + }, + "Model": { + "version": "file:../Model", + "requires": { + "@types/jest": "^27.4.1", + "@types/node": "^17.0.22", + "Common": "file:../Common", + "jest": "^27.5.1", + "ts-jest": "^27.1.4", + "typeorm": "^0.3.7" + } + }, + "node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-iP8l0yGlNpE0e6q1o185yOApANRe47UPbLf4YxfbiNHt/RU5eBcGB/e0oudruheSf+LQeDMezqC5BVAb5wwRcQ==", + "requires": { + "uuid": "8.3.2" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" } } } diff --git a/Workers/package.json b/Workers/package.json index 91d839c4b7..92c5b02777 100644 --- a/Workers/package.json +++ b/Workers/package.json @@ -7,5 +7,11 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", - "license": "MIT" + "license": "MIT", + "dependencies": { + "node-cron": "^3.0.2", + "Common": "file:../Common", + "Model": "file:../Model", + "CommonServer": "file:../CommonServer" + } } diff --git a/Workers/tsconfig.json b/Workers/tsconfig.json index 2af2643bb3..42e2ba9754 100644 --- a/Workers/tsconfig.json +++ b/Workers/tsconfig.json @@ -25,7 +25,7 @@ /* Modules */ // "module": "es2022" /* Specify what module code is generated. */, - "rootDir": "", /* Specify the root folder within your source files. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ @@ -49,7 +49,7 @@ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ - "outDir": "build/dist", /* Specify an output folder for all emitted files. */ + "outDir": "./build/dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d586740147..545809d8f4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -202,6 +202,43 @@ services: - /usr/src/Model/node_modules/ + + workers: + ports: + - '3452:3452' + - '9654:9229' # Debugging port. + build: + network: host + context: . + dockerfile: ./Workers/Dockerfile.dev + env_file: + - ./Common/.env + - ./CommonServer/.env + - ./Workers/.env + environment: *common-variables + depends_on: + - redis + - postgres + - mail + links: + - redis + - postgres + - mail + volumes: + - ./Workers:/usr/src/app + # Use node modules of the container and not host system. + # https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder + - /usr/src/app/node_modules/ + - ./Common:/usr/src/Common + - ./Model:/usr/src/Model + - ./CommonServer:/usr/src/CommonServer + - ./CommonUI:/usr/src/CommonUI + - /usr/src/Common/node_modules/ + - /usr/src/CommonUI/node_modules/ + - /usr/src/CommonServer/node_modules/ + - /usr/src/Model/node_modules/ + +