diff --git a/Common/Models/DatabaseModels/MonitorTest.ts b/Common/Models/DatabaseModels/MonitorTest.ts index 0ceab4b1e8..43cbe8e4be 100644 --- a/Common/Models/DatabaseModels/MonitorTest.ts +++ b/Common/Models/DatabaseModels/MonitorTest.ts @@ -440,5 +440,38 @@ export default class MonitorTest extends BaseModel { nullable: true, unique: false, }) - public lastMonitoringLog?: MonitorStepProbeResponse = undefined; + public monitorStepProbeResponse?: MonitorStepProbeResponse = undefined; + + @ColumnAccessControl({ + create: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.CreateProjectMonitor, + ], + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadProjectMonitor, + ], + update: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.EditProjectMonitor, + ], + }) + @TableColumn({ + isDefaultValueColumn: false, + required: true, + type: TableColumnType.Boolean, + }) + @Column({ + type: ColumnType.Boolean, + nullable: true, + unique: false, + default: true, + }) + public isInQueue?: boolean = undefined; } diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts index bd4190d35c..0d24c4202a 100644 --- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts @@ -157,5 +157,5 @@ export default [ MigrationName1728472625805, MigrationName1729682875503, MigrationName1730117995642, - MigrationName1730209089495 + MigrationName1730209089495, ]; diff --git a/Common/Server/Services/MonitorTestService.ts b/Common/Server/Services/MonitorTestService.ts new file mode 100644 index 0000000000..d49a12e985 --- /dev/null +++ b/Common/Server/Services/MonitorTestService.ts @@ -0,0 +1,10 @@ +import DatabaseService from "./DatabaseService"; +import MonitorTest from "Common/Models/DatabaseModels/MonitorTest"; + +export class MonitorTestService extends DatabaseService { + public constructor() { + super(MonitorTest); + } +} + +export default new MonitorTestService(); diff --git a/Dashboard/src/Components/Form/Monitor/MonitorStep.tsx b/Dashboard/src/Components/Form/Monitor/MonitorStep.tsx index f0d675c139..e8c2325eb5 100644 --- a/Dashboard/src/Components/Form/Monitor/MonitorStep.tsx +++ b/Dashboard/src/Components/Form/Monitor/MonitorStep.tsx @@ -56,6 +56,9 @@ import MonitorStepTraceMonitor, { MonitorStepTraceMonitorUtil, } from "Common/Types/Monitor/MonitorStepTraceMonitor"; import CheckboxElement from "Common/UI/Components/Checkbox/Checkbox"; +import MonitorTestForm from "./MonitorTest"; +import MonitorSteps from "Common/Types/Monitor/MonitorSteps"; +import Probe from "Common/Models/DatabaseModels/Probe"; export interface ComponentProps { monitorStatusDropdownOptions: Array; @@ -66,6 +69,8 @@ export interface ComponentProps { onChange?: undefined | ((value: MonitorStep) => void); // onDelete?: undefined | (() => void); monitorType: MonitorType; + allMonitorSteps: MonitorSteps; + probes: Array; } const MonitorStepElement: FunctionComponent = ( @@ -702,6 +707,16 @@ const MonitorStepElement: FunctionComponent = ( )} + {/** Monitor Test Form */} + + + + {/** Monitoring Critera Form */} +
{props.monitorType !== MonitorType.IncomingRequest && ( <> diff --git a/Dashboard/src/Components/Form/Monitor/MonitorTest.tsx b/Dashboard/src/Components/Form/Monitor/MonitorTest.tsx new file mode 100644 index 0000000000..43074253b0 --- /dev/null +++ b/Dashboard/src/Components/Form/Monitor/MonitorTest.tsx @@ -0,0 +1,198 @@ +import React, { FunctionComponent, ReactElement, useState } from "react"; +import MonitorType, { + MonitorTypeHelper, +} from "Common/Types/Monitor/MonitorType"; +import Probe from "Common/Models/DatabaseModels/Probe"; +import Button, { + ButtonSize, + ButtonStyleType, +} from "Common/UI/Components/Button/Button"; +import IconProp from "Common/Types/Icon/IconProp"; +import Modal from "Common/UI/Components/Modal/Modal"; +import BasicFormModal from "Common/UI/Components/FormModal/BasicFormModal"; +import { JSONObject } from "Common/Types/JSON"; +import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType"; +import DropdownUtil from "Common/UI/Utils/Dropdown"; +import ObjectID from "Common/Types/ObjectID"; +import MonitorTest from "Common/Models/DatabaseModels/MonitorTest"; +import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI"; +import MonitorSteps from "Common/Types/Monitor/MonitorSteps"; +import HTTPResponse from "Common/Types/API/HTTPResponse"; +import API from "Common/UI/Utils/API/API"; +import ButtonType from "Common/UI/Components/Button/ButtonTypes"; +import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage"; +import Loader, { LoaderType } from "Common/UI/Components/Loader/Loader"; +import { MonitorStepProbeResponse } from "Common/Models/DatabaseModels/MonitorProbe"; +import SummaryInfo from "../../Monitor/SummaryView/SummaryInfo"; + +export interface ComponentProps { + monitorSteps: MonitorSteps; + monitorType: MonitorType; + probes: Array; +} + +const MonitorTestForm: FunctionComponent = ( + props: ComponentProps, +): ReactElement => { + // only show this monitor if this monitor is probeable. + + const isProbeable: boolean = MonitorTypeHelper.isProbableMonitor( + props.monitorType, + ); + + if (!isProbeable) { + return <>; + } + + const [showTestModal, setShowTestModal] = useState(false); + const [showResultModal, setShowResultModal] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [monitorStepProbeResponse, setMonitorStepProbeResponse] = + useState(null); + + type ProcessResultFunction = (probeId: ObjectID) => Promise; + const processResult: ProcessResultFunction = async ( + probeId: ObjectID, + ): Promise => { + try { + setShowTestModal(false); + setShowResultModal(true); + + // now we need to run the probe and get the result. + + // save the monitor step to the database. + const monitorTestObj: MonitorTest = new MonitorTest(); + monitorTestObj.monitorSteps = props.monitorSteps; + monitorTestObj.probeId = probeId; + monitorTestObj.monitorType = props.monitorType; + + // save the monitor test to the database. + + const monitorTest: HTTPResponse = (await ModelAPI.create({ + model: monitorTestObj, + modelType: MonitorTest, + })) as HTTPResponse; + + // now we need to fetch the result of this result every 15 seconds. + + const monitorTestId: ObjectID = monitorTest.data.id!; + + let attempts: number = 0; + + const interval: NodeJS.Timer = setInterval(async () => { + const result: MonitorTest | null = (await ModelAPI.getItem({ + modelType: MonitorTest, + id: monitorTestId, + select: { + monitorStepProbeResponse: true, + }, + })) as MonitorTest | null; + + if (result?.monitorStepProbeResponse) { + //set the response and clear the interval. + + setMonitorStepProbeResponse(result.monitorStepProbeResponse); + clearInterval(interval); + setIsLoading(false); + setError(null); + } + + // if we have tried 10 times, then we should stop trying. + + attempts++; + + if (attempts > 10) { + clearInterval(interval); + setIsLoading(false); + setError( + "Monitor Test took too long to complete. Please try again later.", + ); + } + }, 15000); // 15 seconds. + } catch (err) { + setError(API.getFriendlyErrorMessage(err as Error)); + } + }; + + return ( +
+
+ ); +}; + +export default MonitorTestForm; diff --git a/Ingestor/API/Monitor.ts b/Ingestor/API/Monitor.ts index 390e119843..00d7ca08c4 100644 --- a/Ingestor/API/Monitor.ts +++ b/Ingestor/API/Monitor.ts @@ -31,6 +31,8 @@ import MonitorProbe from "Common/Models/DatabaseModels/MonitorProbe"; import MonitorService from "Common/Server/Services/MonitorService"; import ProjectService from "Common/Server/Services/ProjectService"; import MonitorType from "Common/Types/Monitor/MonitorType"; +import MonitorTest from "Common/Models/DatabaseModels/MonitorTest"; +import MonitorTestService from "Common/Server/Services/MonitorTestService"; const router: ExpressRouter = Express.getRouter(); @@ -460,4 +462,137 @@ router.post( }, ); +router.post( + "/monitor-test/list", + ProbeAuthorization.isAuthorizedServiceMiddleware, + async ( + req: ExpressRequest, + res: ExpressResponse, + next: NextFunction, + ): Promise => { + let mutex: SemaphoreMutex | null = null; + + logger.debug("Monitor list API called"); + + try { + const data: JSONObject = req.body; + const limit: number = (data["limit"] as number) || 100; + + logger.debug("Monitor list API called with limit: " + limit); + logger.debug("Data:"); + logger.debug(data); + + if ( + !(req as ProbeExpressRequest).probe || + !(req as ProbeExpressRequest).probe?.id + ) { + logger.error("Probe not found"); + + return Response.sendErrorResponse( + req, + res, + new BadDataException("Probe not found"), + ); + } + + const probeId: ObjectID = (req as ProbeExpressRequest).probe!.id!; + + if (!probeId) { + logger.error("Probe not found"); + + return Response.sendErrorResponse( + req, + res, + new BadDataException("Probe not found"), + ); + } + + try { + mutex = await Semaphore.lock({ + key: probeId.toString(), + namespace: "MonitorAPI.monitor-test-list", + }); + } catch (err) { + logger.error(err); + } + + //get list of monitors to be monitored + + logger.debug("Fetching monitor list"); + + const monitorTests: Array = await MonitorTestService.findBy({ + query: { + monitorStepProbeResponse: QueryHelper.isNull(), + probeId: probeId, + isInQueue: true, // only get the tests which are in queue + }, + sort: { + createdAt: SortOrder.Ascending, + }, + skip: 0, + limit: limit, + select: { + monitorType: true, + monitorSteps: true, + _id: true, + }, + props: { + isRoot: true, + }, + }); + + logger.debug("Fetched monitor tests"); + logger.debug(monitorTests); + + // update the lastMonitoredAt field of the monitors + + const updatePromises: Array> = []; + + for (const monitorTest of monitorTests) { + updatePromises.push( + MonitorTestService.updateOneById({ + id: monitorTest.id!, + data: { + isInQueue: false, // in progress now + }, + props: { + isRoot: true, + }, + }), + ); + } + + await Promise.all(updatePromises); + + if (mutex) { + try { + await Semaphore.release(mutex); + } catch (err) { + logger.error(err); + } + } + + logger.debug("Sending response"); + + return Response.sendEntityArrayResponse( + req, + res, + monitorTests, + new PositiveNumber(monitorTests.length), + MonitorTest, + ); + } catch (err) { + try { + if (mutex) { + await Semaphore.release(mutex); + } + } catch (err) { + logger.error(err); + } + + return next(err); + } + }, +); + export default router; diff --git a/Ingestor/API/Probe.ts b/Ingestor/API/Probe.ts index e2c60b7ff0..45d01c7949 100644 --- a/Ingestor/API/Probe.ts +++ b/Ingestor/API/Probe.ts @@ -25,6 +25,8 @@ import Response from "Common/Server/Utils/Response"; import GlobalConfig from "Common/Models/DatabaseModels/GlobalConfig"; import Probe from "Common/Models/DatabaseModels/Probe"; import User from "Common/Models/DatabaseModels/User"; +import MonitorTestService from "Common/Server/Services/MonitorTestService"; +import OneUptimeDate from "Common/Types/Date"; const router: ExpressRouter = Express.getRouter(); @@ -258,4 +260,61 @@ router.post( }, ); +router.post( + "/probe/response/monitor-test-ingest/:testId", + ProbeAuthorization.isAuthorizedServiceMiddleware, + async ( + req: ExpressRequest, + res: ExpressResponse, + next: NextFunction, + ): Promise => { + try { + const probeResponse: ProbeMonitorResponse = JSONFunctions.deserialize( + req.body["probeMonitorResponse"], + ) as any; + + const testId: ObjectID = new ObjectID(req.params["testId"] as string); + + if (!testId) { + return Response.sendErrorResponse( + req, + res, + new BadDataException("TestId not found"), + ); + } + + if (!probeResponse) { + return Response.sendErrorResponse( + req, + res, + new BadDataException("ProbeMonitorResponse not found"), + ); + } + + // save the probe response to the monitor test. + + await MonitorTestService.updateOneById({ + id: testId, + data: { + monitorStepProbeResponse: { + [probeResponse.monitorStepId.toString()]: { + ...JSON.parse(JSON.stringify(probeResponse)), + monitoredAt: OneUptimeDate.getCurrentDate(), + }, + } as any, + }, + props: { + isRoot: true, + }, + }); + + // send success response. + + return Response.sendEmptySuccessResponse(req, res); + } catch (err) { + return next(err); + } + }, +); + export default router; diff --git a/Probe/Jobs/Monitor/FetchMonitorTest.ts b/Probe/Jobs/Monitor/FetchMonitorTest.ts new file mode 100644 index 0000000000..c9128bace4 --- /dev/null +++ b/Probe/Jobs/Monitor/FetchMonitorTest.ts @@ -0,0 +1,116 @@ +import { INGESTOR_URL, PROBE_MONITOR_FETCH_LIMIT } from "../../Config"; +import MonitorUtil from "../../Utils/Monitors/Monitor"; +import ProbeAPIRequest from "../../Utils/ProbeAPIRequest"; +import BaseModel from "Common/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel"; +import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse"; +import HTTPMethod from "Common/Types/API/HTTPMethod"; +import HTTPResponse from "Common/Types/API/HTTPResponse"; +import URL from "Common/Types/API/URL"; +import MonitorTest from "Common/Models/DatabaseModels/MonitorTest"; +import APIException from "Common/Types/Exception/ApiException"; +import { JSONArray } from "Common/Types/JSON"; +import ProbeMonitorResponse from "Common/Types/Probe/ProbeMonitorResponse"; +import Sleep from "Common/Types/Sleep"; +import API from "Common/Utils/API"; +import logger from "Common/Server/Utils/Logger"; + +export default class FetchMonitorTestAndProbe { + private workerName: string = ""; + + public constructor(workerName: string) { + this.workerName = workerName; + } + + public async run(): Promise { + logger.debug(`Running worker ${this.workerName}`); + + // eslint-disable-next-line no-constant-condition + while (true) { + try { + logger.debug(`Probing monitors ${this.workerName}`); + + await this.fetchListAndProbe(); + + logger.debug(`Probing monitors ${this.workerName} complete`); + + // sleep for 15 seconds + + await Sleep.sleep(15000); + } catch (err) { + logger.error(`Error in worker ${this.workerName}`); + logger.error(err); + await Sleep.sleep(2000); + } + } + } + + private async fetchListAndProbe(): Promise { + try { + logger.debug("Fetching monitor list"); + + const monitorListUrl: URL = URL.fromString( + INGESTOR_URL.toString(), + ).addRoute("/monitor-test/list"); + + const result: HTTPResponse | HTTPErrorResponse = + await API.fetch( + HTTPMethod.POST, + monitorListUrl, + { + ...ProbeAPIRequest.getDefaultRequestBody(), + limit: PROBE_MONITOR_FETCH_LIMIT || 100, + }, + {}, + {}, + ); + + logger.debug("Fetched monitor test list"); + logger.debug(result); + + const monitorTests: Array = BaseModel.fromJSONArray( + result.data as JSONArray, + MonitorTest, + ); + + const probeMonitorPromises: Array< + Promise> + > = []; // Array of promises to probe monitors + + for (const monitorTest of monitorTests) { + probeMonitorPromises.push(MonitorUtil.probeMonitorTest(monitorTest)); + } + + // all settled + // eslint-disable-next-line no-undef + const results: PromiseSettledResult<(ProbeMonitorResponse | null)[]>[] = + await Promise.allSettled(probeMonitorPromises); + + let resultIndex: number = 0; + + for (const result of results) { + if (monitorTests && monitorTests[resultIndex]) { + logger.debug("Monitor Test:"); + logger.debug(monitorTests[resultIndex]); + } + + if (result.status === "rejected") { + logger.error("Error in probing monitor:"); + logger.error(result.reason); + } else { + logger.debug("Probed monitor: "); + logger.debug(result.value); + } + + resultIndex++; + } + } catch (err) { + logger.error("Error in fetching monitor list"); + logger.error(err); + + if (err instanceof APIException) { + logger.error("API Exception Error"); + logger.error(JSON.stringify(err.error, null, 2)); + } + } + } +} diff --git a/Probe/Utils/Monitors/Monitor.ts b/Probe/Utils/Monitors/Monitor.ts index df474da6ec..e5ba4d8e53 100644 --- a/Probe/Utils/Monitors/Monitor.ts +++ b/Probe/Utils/Monitors/Monitor.ts @@ -30,8 +30,58 @@ import LocalCache from "Common/Server/Infrastructure/LocalCache"; import logger from "Common/Server/Utils/Logger"; import Monitor from "Common/Models/DatabaseModels/Monitor"; import PositiveNumber from "Common/Types/PositiveNumber"; +import ObjectID from "Common/Types/ObjectID"; +import MonitorTest from "Common/Models/DatabaseModels/MonitorTest"; export default class MonitorUtil { + public static async probeMonitorTest( + monitorTest: MonitorTest, + ): Promise> { + const results: Array = []; + + if ( + !monitorTest.monitorSteps || + monitorTest.monitorSteps.data?.monitorStepsInstanceArray.length === 0 + ) { + logger.debug("No monitor steps found"); + return []; + } + + for (const monitorStep of monitorTest.monitorSteps.data + ?.monitorStepsInstanceArray || []) { + if (!monitorStep) { + continue; + } + + const result: ProbeMonitorResponse | null = await this.probeMonitorStep({ + monitorType: monitorTest.monitorType!, + monitorId: monitorTest.id!, + monitorStep: monitorStep, + }); + + if (result) { + // report this back to Probe API. + + await API.fetch( + HTTPMethod.POST, + URL.fromString(INGESTOR_URL.toString()).addRoute( + "/probe/response/monitor-test-ingest/" + monitorTest.id?.toString(), + ), + { + ...ProbeAPIRequest.getDefaultRequestBody(), + probeMonitorResponse: result as any, + }, + {}, + {}, + ); + } + + results.push(result); + } + + return results; + } + public static async probeMonitor( monitor: Monitor, ): Promise> { @@ -51,10 +101,11 @@ export default class MonitorUtil { continue; } - const result: ProbeMonitorResponse | null = await this.probeMonitorStep( - monitorStep, - monitor, - ); + const result: ProbeMonitorResponse | null = await this.probeMonitorStep({ + monitorType: monitor.monitorType!, + monitorId: monitor.id!, + monitorStep: monitorStep, + }); if (result) { // report this back to Probe API. @@ -117,13 +168,18 @@ export default class MonitorUtil { return true; } - public static async probeMonitorStep( - monitorStep: MonitorStep, - monitor: Monitor, - ): Promise { + public static async probeMonitorStep(data: { + monitorStep: MonitorStep; + monitorType: MonitorType; + monitorId: ObjectID; + }): Promise { + const monitorStep: MonitorStep = data.monitorStep; + const monitorType: MonitorType = data.monitorType; + const monitorId: ObjectID = data.monitorId; + const result: ProbeMonitorResponse = { monitorStepId: monitorStep.id, - monitorId: monitor.id!, + monitorId: monitorId!, probeId: ProbeUtil.getProbeId(), failureCause: "", monitoredAt: OneUptimeDate.getCurrentDate(), @@ -133,10 +189,7 @@ export default class MonitorUtil { return result; } - if ( - monitor.monitorType === MonitorType.Ping || - monitor.monitorType === MonitorType.IP - ) { + if (monitorType === MonitorType.Ping || monitorType === MonitorType.IP) { if (!monitorStep.data?.monitorDestination) { return result; } @@ -151,7 +204,7 @@ export default class MonitorUtil { new Port(80), // use port 80 by default. { retry: 10, - monitorId: monitor.id!, + monitorId: monitorId, timeout: new PositiveNumber(60000), // 60 seconds }, ); @@ -168,7 +221,7 @@ export default class MonitorUtil { monitorStep.data?.monitorDestination, { retry: 10, - monitorId: monitor.id!, + monitorId: monitorId, timeout: new PositiveNumber(60000), // 60 seconds }, ); @@ -183,7 +236,7 @@ export default class MonitorUtil { } } - if (monitor.monitorType === MonitorType.Port) { + if (monitorType === MonitorType.Port) { if (!monitorStep.data?.monitorDestination) { return result; } @@ -205,7 +258,7 @@ export default class MonitorUtil { monitorStep.data.monitorDestinationPort, { retry: 10, - monitorId: monitor.id!, + monitorId: monitorId, timeout: new PositiveNumber(60000), // 60 seconds }, ); @@ -219,7 +272,7 @@ export default class MonitorUtil { result.failureCause = response.failureCause; } - if (monitor.monitorType === MonitorType.SyntheticMonitor) { + if (monitorType === MonitorType.SyntheticMonitor) { if (!monitorStep.data?.customCode) { result.failureCause = "Code not specified. Please add playwright script."; @@ -229,7 +282,7 @@ export default class MonitorUtil { const response: Array | null = await SyntheticMonitor.execute({ script: monitorStep.data.customCode, - monitorId: monitor.id!, + monitorId: monitorId, screenSizeTypes: monitorStep.data .screenSizeTypes as Array, browserTypes: monitorStep.data.browserTypes as Array, @@ -242,7 +295,7 @@ export default class MonitorUtil { result.syntheticMonitorResponse = response; } - if (monitor.monitorType === MonitorType.CustomJavaScriptCode) { + if (monitorType === MonitorType.CustomJavaScriptCode) { if (!monitorStep.data?.customCode) { result.failureCause = "Code not specified. Please add playwright script."; @@ -252,7 +305,7 @@ export default class MonitorUtil { const response: CustomCodeMonitorResponse | null = await CustomCodeMonitor.execute({ script: monitorStep.data.customCode, - monitorId: monitor.id!, + monitorId: monitorId, }); if (!response) { @@ -262,7 +315,7 @@ export default class MonitorUtil { result.customCodeMonitorResponse = response; } - if (monitor.monitorType === MonitorType.SSLCertificate) { + if (monitorType === MonitorType.SSLCertificate) { if (!monitorStep.data?.monitorDestination) { return result; } @@ -281,7 +334,7 @@ export default class MonitorUtil { monitorStep.data?.monitorDestination as URL, { retry: 10, - monitorId: monitor.id!, + monitorId: monitorId, timeout: new PositiveNumber(60000), // 60 seconds }, ); @@ -297,7 +350,7 @@ export default class MonitorUtil { }; } - if (monitor.monitorType === MonitorType.Website) { + if (monitorType === MonitorType.Website) { if (!monitorStep.data?.monitorDestination) { return result; } @@ -308,7 +361,7 @@ export default class MonitorUtil { monitorStep.data?.monitorDestination as URL, { isHeadRequest: MonitorUtil.isHeadRequest(monitorStep), - monitorId: monitor.id!, + monitorId: monitorId, retry: 10, timeout: new PositiveNumber(60000), // 60 seconds doNotFollowRedirects: monitorStep.data?.doNotFollowRedirects || false, @@ -327,7 +380,7 @@ export default class MonitorUtil { result.failureCause = response.failureCause; } - if (monitor.monitorType === MonitorType.API) { + if (monitorType === MonitorType.API) { if (!monitorStep.data?.monitorDestination) { return result; } @@ -349,7 +402,7 @@ export default class MonitorUtil { { requestHeaders: monitorStep.data?.requestHeaders || {}, requestBody: requestBody || undefined, - monitorId: monitor.id!, + monitorId: monitorId, requestType: monitorStep.data?.requestType || HTTPMethod.GET, retry: 10, timeout: new PositiveNumber(60000), // 60 seconds