Refactor MonitorTestService and MonitorTest model

This commit is contained in:
Simon Larsen 2024-10-29 16:59:07 +00:00
parent 25ba824d79
commit 9b4ef72682
No known key found for this signature in database
GPG Key ID: 96C5DCA24769DBCA
9 changed files with 648 additions and 29 deletions

View File

@ -440,5 +440,38 @@ export default class MonitorTest extends BaseModel {
nullable: true,
unique: false,
})
public lastMonitoringLog?: MonitorStepProbeResponse = undefined;
public monitorStepProbeResponse?: MonitorStepProbeResponse = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.CreateProjectMonitor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectMonitor,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.EditProjectMonitor,
],
})
@TableColumn({
isDefaultValueColumn: false,
required: true,
type: TableColumnType.Boolean,
})
@Column({
type: ColumnType.Boolean,
nullable: true,
unique: false,
default: true,
})
public isInQueue?: boolean = undefined;
}

View File

@ -157,5 +157,5 @@ export default [
MigrationName1728472625805,
MigrationName1729682875503,
MigrationName1730117995642,
MigrationName1730209089495
MigrationName1730209089495,
];

View File

@ -0,0 +1,10 @@
import DatabaseService from "./DatabaseService";
import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
export class MonitorTestService extends DatabaseService<MonitorTest> {
public constructor() {
super(MonitorTest);
}
}
export default new MonitorTestService();

View File

@ -56,6 +56,9 @@ import MonitorStepTraceMonitor, {
MonitorStepTraceMonitorUtil,
} from "Common/Types/Monitor/MonitorStepTraceMonitor";
import CheckboxElement from "Common/UI/Components/Checkbox/Checkbox";
import MonitorTestForm from "./MonitorTest";
import MonitorSteps from "Common/Types/Monitor/MonitorSteps";
import Probe from "Common/Models/DatabaseModels/Probe";
export interface ComponentProps {
monitorStatusDropdownOptions: Array<DropdownOption>;
@ -66,6 +69,8 @@ export interface ComponentProps {
onChange?: undefined | ((value: MonitorStep) => void);
// onDelete?: undefined | (() => void);
monitorType: MonitorType;
allMonitorSteps: MonitorSteps;
probes: Array<Probe>;
}
const MonitorStepElement: FunctionComponent<ComponentProps> = (
@ -702,6 +707,16 @@ const MonitorStepElement: FunctionComponent<ComponentProps> = (
</div>
)}
{/** Monitor Test Form */}
<MonitorTestForm
monitorSteps={props.allMonitorSteps}
monitorType={props.monitorType}
probes={props.probes}
/>
{/** Monitoring Critera Form */}
<div className="mt-5">
{props.monitorType !== MonitorType.IncomingRequest && (
<>

View File

@ -0,0 +1,198 @@
import React, { FunctionComponent, ReactElement, useState } from "react";
import MonitorType, {
MonitorTypeHelper,
} from "Common/Types/Monitor/MonitorType";
import Probe from "Common/Models/DatabaseModels/Probe";
import Button, {
ButtonSize,
ButtonStyleType,
} from "Common/UI/Components/Button/Button";
import IconProp from "Common/Types/Icon/IconProp";
import Modal from "Common/UI/Components/Modal/Modal";
import BasicFormModal from "Common/UI/Components/FormModal/BasicFormModal";
import { JSONObject } from "Common/Types/JSON";
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
import DropdownUtil from "Common/UI/Utils/Dropdown";
import ObjectID from "Common/Types/ObjectID";
import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI";
import MonitorSteps from "Common/Types/Monitor/MonitorSteps";
import HTTPResponse from "Common/Types/API/HTTPResponse";
import API from "Common/UI/Utils/API/API";
import ButtonType from "Common/UI/Components/Button/ButtonTypes";
import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage";
import Loader, { LoaderType } from "Common/UI/Components/Loader/Loader";
import { MonitorStepProbeResponse } from "Common/Models/DatabaseModels/MonitorProbe";
import SummaryInfo from "../../Monitor/SummaryView/SummaryInfo";
export interface ComponentProps {
monitorSteps: MonitorSteps;
monitorType: MonitorType;
probes: Array<Probe>;
}
const MonitorTestForm: FunctionComponent<ComponentProps> = (
props: ComponentProps,
): ReactElement => {
// only show this monitor if this monitor is probeable.
const isProbeable: boolean = MonitorTypeHelper.isProbableMonitor(
props.monitorType,
);
if (!isProbeable) {
return <></>;
}
const [showTestModal, setShowTestModal] = useState<boolean>(false);
const [showResultModal, setShowResultModal] = useState<boolean>(false);
const [isLoading, setIsLoading] = useState<boolean>(false);
const [error, setError] = useState<string | null>(null);
const [monitorStepProbeResponse, setMonitorStepProbeResponse] =
useState<MonitorStepProbeResponse | null>(null);
type ProcessResultFunction = (probeId: ObjectID) => Promise<void>;
const processResult: ProcessResultFunction = async (
probeId: ObjectID,
): Promise<void> => {
try {
setShowTestModal(false);
setShowResultModal(true);
// now we need to run the probe and get the result.
// save the monitor step to the database.
const monitorTestObj: MonitorTest = new MonitorTest();
monitorTestObj.monitorSteps = props.monitorSteps;
monitorTestObj.probeId = probeId;
monitorTestObj.monitorType = props.monitorType;
// save the monitor test to the database.
const monitorTest: HTTPResponse<MonitorTest> = (await ModelAPI.create({
model: monitorTestObj,
modelType: MonitorTest,
})) as HTTPResponse<MonitorTest>;
// now we need to fetch the result of this result every 15 seconds.
const monitorTestId: ObjectID = monitorTest.data.id!;
let attempts: number = 0;
const interval: NodeJS.Timer = setInterval(async () => {
const result: MonitorTest | null = (await ModelAPI.getItem({
modelType: MonitorTest,
id: monitorTestId,
select: {
monitorStepProbeResponse: true,
},
})) as MonitorTest | null;
if (result?.monitorStepProbeResponse) {
//set the response and clear the interval.
setMonitorStepProbeResponse(result.monitorStepProbeResponse);
clearInterval(interval);
setIsLoading(false);
setError(null);
}
// if we have tried 10 times, then we should stop trying.
attempts++;
if (attempts > 10) {
clearInterval(interval);
setIsLoading(false);
setError(
"Monitor Test took too long to complete. Please try again later.",
);
}
}, 15000); // 15 seconds.
} catch (err) {
setError(API.getFriendlyErrorMessage(err as Error));
}
};
return (
<div>
<Button
buttonStyle={ButtonStyleType.NORMAL}
buttonSize={ButtonSize.Small}
title="Test"
icon={IconProp.Play}
onClick={() => {
setShowTestModal(true);
}}
/>
{showTestModal && (
<BasicFormModal
title={"Test Monitor"}
onClose={() => {
return setShowTestModal(false);
}}
onSubmit={async (data: JSONObject) => {
await processResult(data["probe"] as ObjectID);
}}
formProps={{
initialValues: {},
fields: [
{
field: {
probe: true,
},
title: "Select Probe",
fieldType: FormFieldSchemaType.Dropdown,
dropdownOptions: DropdownUtil.getDropdownOptionsFromEntityArray(
{
array: props.probes,
labelField: "name",
valueField: "_id",
},
),
required: true,
placeholder: "",
},
],
}}
/>
)}
{showResultModal && (
<Modal
title="Monitor Test Result"
submitButtonText="Close"
submitButtonType={ButtonType.Button}
onSubmit={() => {
setShowResultModal(false);
}}
>
<div>
{error && <ErrorMessage error={error} />}
{isLoading && <Loader loaderType={LoaderType.Bar} />}
{isLoading && (
<div>
Running Monitor Test. This usually takes a minute or two to
complete.
</div>
)}
{monitorStepProbeResponse && (
<div>
<SummaryInfo
monitorType={props.monitorType}
probeMonitorResponses={Object.values(
monitorStepProbeResponse,
)}
/>
</div>
)}
</div>
</Modal>
)}
</div>
);
};
export default MonitorTestForm;

View File

@ -31,6 +31,8 @@ import MonitorProbe from "Common/Models/DatabaseModels/MonitorProbe";
import MonitorService from "Common/Server/Services/MonitorService";
import ProjectService from "Common/Server/Services/ProjectService";
import MonitorType from "Common/Types/Monitor/MonitorType";
import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
import MonitorTestService from "Common/Server/Services/MonitorTestService";
const router: ExpressRouter = Express.getRouter();
@ -460,4 +462,137 @@ router.post(
},
);
router.post(
"/monitor-test/list",
ProbeAuthorization.isAuthorizedServiceMiddleware,
async (
req: ExpressRequest,
res: ExpressResponse,
next: NextFunction,
): Promise<void> => {
let mutex: SemaphoreMutex | null = null;
logger.debug("Monitor list API called");
try {
const data: JSONObject = req.body;
const limit: number = (data["limit"] as number) || 100;
logger.debug("Monitor list API called with limit: " + limit);
logger.debug("Data:");
logger.debug(data);
if (
!(req as ProbeExpressRequest).probe ||
!(req as ProbeExpressRequest).probe?.id
) {
logger.error("Probe not found");
return Response.sendErrorResponse(
req,
res,
new BadDataException("Probe not found"),
);
}
const probeId: ObjectID = (req as ProbeExpressRequest).probe!.id!;
if (!probeId) {
logger.error("Probe not found");
return Response.sendErrorResponse(
req,
res,
new BadDataException("Probe not found"),
);
}
try {
mutex = await Semaphore.lock({
key: probeId.toString(),
namespace: "MonitorAPI.monitor-test-list",
});
} catch (err) {
logger.error(err);
}
//get list of monitors to be monitored
logger.debug("Fetching monitor list");
const monitorTests: Array<MonitorTest> = await MonitorTestService.findBy({
query: {
monitorStepProbeResponse: QueryHelper.isNull(),
probeId: probeId,
isInQueue: true, // only get the tests which are in queue
},
sort: {
createdAt: SortOrder.Ascending,
},
skip: 0,
limit: limit,
select: {
monitorType: true,
monitorSteps: true,
_id: true,
},
props: {
isRoot: true,
},
});
logger.debug("Fetched monitor tests");
logger.debug(monitorTests);
// update the lastMonitoredAt field of the monitors
const updatePromises: Array<Promise<void>> = [];
for (const monitorTest of monitorTests) {
updatePromises.push(
MonitorTestService.updateOneById({
id: monitorTest.id!,
data: {
isInQueue: false, // in progress now
},
props: {
isRoot: true,
},
}),
);
}
await Promise.all(updatePromises);
if (mutex) {
try {
await Semaphore.release(mutex);
} catch (err) {
logger.error(err);
}
}
logger.debug("Sending response");
return Response.sendEntityArrayResponse(
req,
res,
monitorTests,
new PositiveNumber(monitorTests.length),
MonitorTest,
);
} catch (err) {
try {
if (mutex) {
await Semaphore.release(mutex);
}
} catch (err) {
logger.error(err);
}
return next(err);
}
},
);
export default router;

View File

@ -25,6 +25,8 @@ import Response from "Common/Server/Utils/Response";
import GlobalConfig from "Common/Models/DatabaseModels/GlobalConfig";
import Probe from "Common/Models/DatabaseModels/Probe";
import User from "Common/Models/DatabaseModels/User";
import MonitorTestService from "Common/Server/Services/MonitorTestService";
import OneUptimeDate from "Common/Types/Date";
const router: ExpressRouter = Express.getRouter();
@ -258,4 +260,61 @@ router.post(
},
);
router.post(
"/probe/response/monitor-test-ingest/:testId",
ProbeAuthorization.isAuthorizedServiceMiddleware,
async (
req: ExpressRequest,
res: ExpressResponse,
next: NextFunction,
): Promise<void> => {
try {
const probeResponse: ProbeMonitorResponse = JSONFunctions.deserialize(
req.body["probeMonitorResponse"],
) as any;
const testId: ObjectID = new ObjectID(req.params["testId"] as string);
if (!testId) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("TestId not found"),
);
}
if (!probeResponse) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("ProbeMonitorResponse not found"),
);
}
// save the probe response to the monitor test.
await MonitorTestService.updateOneById({
id: testId,
data: {
monitorStepProbeResponse: {
[probeResponse.monitorStepId.toString()]: {
...JSON.parse(JSON.stringify(probeResponse)),
monitoredAt: OneUptimeDate.getCurrentDate(),
},
} as any,
},
props: {
isRoot: true,
},
});
// send success response.
return Response.sendEmptySuccessResponse(req, res);
} catch (err) {
return next(err);
}
},
);
export default router;

View File

@ -0,0 +1,116 @@
import { INGESTOR_URL, PROBE_MONITOR_FETCH_LIMIT } from "../../Config";
import MonitorUtil from "../../Utils/Monitors/Monitor";
import ProbeAPIRequest from "../../Utils/ProbeAPIRequest";
import BaseModel from "Common/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse";
import HTTPMethod from "Common/Types/API/HTTPMethod";
import HTTPResponse from "Common/Types/API/HTTPResponse";
import URL from "Common/Types/API/URL";
import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
import APIException from "Common/Types/Exception/ApiException";
import { JSONArray } from "Common/Types/JSON";
import ProbeMonitorResponse from "Common/Types/Probe/ProbeMonitorResponse";
import Sleep from "Common/Types/Sleep";
import API from "Common/Utils/API";
import logger from "Common/Server/Utils/Logger";
export default class FetchMonitorTestAndProbe {
private workerName: string = "";
public constructor(workerName: string) {
this.workerName = workerName;
}
public async run(): Promise<void> {
logger.debug(`Running worker ${this.workerName}`);
// eslint-disable-next-line no-constant-condition
while (true) {
try {
logger.debug(`Probing monitors ${this.workerName}`);
await this.fetchListAndProbe();
logger.debug(`Probing monitors ${this.workerName} complete`);
// sleep for 15 seconds
await Sleep.sleep(15000);
} catch (err) {
logger.error(`Error in worker ${this.workerName}`);
logger.error(err);
await Sleep.sleep(2000);
}
}
}
private async fetchListAndProbe(): Promise<void> {
try {
logger.debug("Fetching monitor list");
const monitorListUrl: URL = URL.fromString(
INGESTOR_URL.toString(),
).addRoute("/monitor-test/list");
const result: HTTPResponse<JSONArray> | HTTPErrorResponse =
await API.fetch<JSONArray>(
HTTPMethod.POST,
monitorListUrl,
{
...ProbeAPIRequest.getDefaultRequestBody(),
limit: PROBE_MONITOR_FETCH_LIMIT || 100,
},
{},
{},
);
logger.debug("Fetched monitor test list");
logger.debug(result);
const monitorTests: Array<MonitorTest> = BaseModel.fromJSONArray(
result.data as JSONArray,
MonitorTest,
);
const probeMonitorPromises: Array<
Promise<Array<ProbeMonitorResponse | null>>
> = []; // Array of promises to probe monitors
for (const monitorTest of monitorTests) {
probeMonitorPromises.push(MonitorUtil.probeMonitorTest(monitorTest));
}
// all settled
// eslint-disable-next-line no-undef
const results: PromiseSettledResult<(ProbeMonitorResponse | null)[]>[] =
await Promise.allSettled(probeMonitorPromises);
let resultIndex: number = 0;
for (const result of results) {
if (monitorTests && monitorTests[resultIndex]) {
logger.debug("Monitor Test:");
logger.debug(monitorTests[resultIndex]);
}
if (result.status === "rejected") {
logger.error("Error in probing monitor:");
logger.error(result.reason);
} else {
logger.debug("Probed monitor: ");
logger.debug(result.value);
}
resultIndex++;
}
} catch (err) {
logger.error("Error in fetching monitor list");
logger.error(err);
if (err instanceof APIException) {
logger.error("API Exception Error");
logger.error(JSON.stringify(err.error, null, 2));
}
}
}
}

View File

@ -30,8 +30,58 @@ import LocalCache from "Common/Server/Infrastructure/LocalCache";
import logger from "Common/Server/Utils/Logger";
import Monitor from "Common/Models/DatabaseModels/Monitor";
import PositiveNumber from "Common/Types/PositiveNumber";
import ObjectID from "Common/Types/ObjectID";
import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
export default class MonitorUtil {
public static async probeMonitorTest(
monitorTest: MonitorTest,
): Promise<Array<ProbeMonitorResponse | null>> {
const results: Array<ProbeMonitorResponse | null> = [];
if (
!monitorTest.monitorSteps ||
monitorTest.monitorSteps.data?.monitorStepsInstanceArray.length === 0
) {
logger.debug("No monitor steps found");
return [];
}
for (const monitorStep of monitorTest.monitorSteps.data
?.monitorStepsInstanceArray || []) {
if (!monitorStep) {
continue;
}
const result: ProbeMonitorResponse | null = await this.probeMonitorStep({
monitorType: monitorTest.monitorType!,
monitorId: monitorTest.id!,
monitorStep: monitorStep,
});
if (result) {
// report this back to Probe API.
await API.fetch<JSONObject>(
HTTPMethod.POST,
URL.fromString(INGESTOR_URL.toString()).addRoute(
"/probe/response/monitor-test-ingest/" + monitorTest.id?.toString(),
),
{
...ProbeAPIRequest.getDefaultRequestBody(),
probeMonitorResponse: result as any,
},
{},
{},
);
}
results.push(result);
}
return results;
}
public static async probeMonitor(
monitor: Monitor,
): Promise<Array<ProbeMonitorResponse | null>> {
@ -51,10 +101,11 @@ export default class MonitorUtil {
continue;
}
const result: ProbeMonitorResponse | null = await this.probeMonitorStep(
monitorStep,
monitor,
);
const result: ProbeMonitorResponse | null = await this.probeMonitorStep({
monitorType: monitor.monitorType!,
monitorId: monitor.id!,
monitorStep: monitorStep,
});
if (result) {
// report this back to Probe API.
@ -117,13 +168,18 @@ export default class MonitorUtil {
return true;
}
public static async probeMonitorStep(
monitorStep: MonitorStep,
monitor: Monitor,
): Promise<ProbeMonitorResponse | null> {
public static async probeMonitorStep(data: {
monitorStep: MonitorStep;
monitorType: MonitorType;
monitorId: ObjectID;
}): Promise<ProbeMonitorResponse | null> {
const monitorStep: MonitorStep = data.monitorStep;
const monitorType: MonitorType = data.monitorType;
const monitorId: ObjectID = data.monitorId;
const result: ProbeMonitorResponse = {
monitorStepId: monitorStep.id,
monitorId: monitor.id!,
monitorId: monitorId!,
probeId: ProbeUtil.getProbeId(),
failureCause: "",
monitoredAt: OneUptimeDate.getCurrentDate(),
@ -133,10 +189,7 @@ export default class MonitorUtil {
return result;
}
if (
monitor.monitorType === MonitorType.Ping ||
monitor.monitorType === MonitorType.IP
) {
if (monitorType === MonitorType.Ping || monitorType === MonitorType.IP) {
if (!monitorStep.data?.monitorDestination) {
return result;
}
@ -151,7 +204,7 @@ export default class MonitorUtil {
new Port(80), // use port 80 by default.
{
retry: 10,
monitorId: monitor.id!,
monitorId: monitorId,
timeout: new PositiveNumber(60000), // 60 seconds
},
);
@ -168,7 +221,7 @@ export default class MonitorUtil {
monitorStep.data?.monitorDestination,
{
retry: 10,
monitorId: monitor.id!,
monitorId: monitorId,
timeout: new PositiveNumber(60000), // 60 seconds
},
);
@ -183,7 +236,7 @@ export default class MonitorUtil {
}
}
if (monitor.monitorType === MonitorType.Port) {
if (monitorType === MonitorType.Port) {
if (!monitorStep.data?.monitorDestination) {
return result;
}
@ -205,7 +258,7 @@ export default class MonitorUtil {
monitorStep.data.monitorDestinationPort,
{
retry: 10,
monitorId: monitor.id!,
monitorId: monitorId,
timeout: new PositiveNumber(60000), // 60 seconds
},
);
@ -219,7 +272,7 @@ export default class MonitorUtil {
result.failureCause = response.failureCause;
}
if (monitor.monitorType === MonitorType.SyntheticMonitor) {
if (monitorType === MonitorType.SyntheticMonitor) {
if (!monitorStep.data?.customCode) {
result.failureCause =
"Code not specified. Please add playwright script.";
@ -229,7 +282,7 @@ export default class MonitorUtil {
const response: Array<SyntheticMonitorResponse> | null =
await SyntheticMonitor.execute({
script: monitorStep.data.customCode,
monitorId: monitor.id!,
monitorId: monitorId,
screenSizeTypes: monitorStep.data
.screenSizeTypes as Array<ScreenSizeType>,
browserTypes: monitorStep.data.browserTypes as Array<BrowserType>,
@ -242,7 +295,7 @@ export default class MonitorUtil {
result.syntheticMonitorResponse = response;
}
if (monitor.monitorType === MonitorType.CustomJavaScriptCode) {
if (monitorType === MonitorType.CustomJavaScriptCode) {
if (!monitorStep.data?.customCode) {
result.failureCause =
"Code not specified. Please add playwright script.";
@ -252,7 +305,7 @@ export default class MonitorUtil {
const response: CustomCodeMonitorResponse | null =
await CustomCodeMonitor.execute({
script: monitorStep.data.customCode,
monitorId: monitor.id!,
monitorId: monitorId,
});
if (!response) {
@ -262,7 +315,7 @@ export default class MonitorUtil {
result.customCodeMonitorResponse = response;
}
if (monitor.monitorType === MonitorType.SSLCertificate) {
if (monitorType === MonitorType.SSLCertificate) {
if (!monitorStep.data?.monitorDestination) {
return result;
}
@ -281,7 +334,7 @@ export default class MonitorUtil {
monitorStep.data?.monitorDestination as URL,
{
retry: 10,
monitorId: monitor.id!,
monitorId: monitorId,
timeout: new PositiveNumber(60000), // 60 seconds
},
);
@ -297,7 +350,7 @@ export default class MonitorUtil {
};
}
if (monitor.monitorType === MonitorType.Website) {
if (monitorType === MonitorType.Website) {
if (!monitorStep.data?.monitorDestination) {
return result;
}
@ -308,7 +361,7 @@ export default class MonitorUtil {
monitorStep.data?.monitorDestination as URL,
{
isHeadRequest: MonitorUtil.isHeadRequest(monitorStep),
monitorId: monitor.id!,
monitorId: monitorId,
retry: 10,
timeout: new PositiveNumber(60000), // 60 seconds
doNotFollowRedirects: monitorStep.data?.doNotFollowRedirects || false,
@ -327,7 +380,7 @@ export default class MonitorUtil {
result.failureCause = response.failureCause;
}
if (monitor.monitorType === MonitorType.API) {
if (monitorType === MonitorType.API) {
if (!monitorStep.data?.monitorDestination) {
return result;
}
@ -349,7 +402,7 @@ export default class MonitorUtil {
{
requestHeaders: monitorStep.data?.requestHeaders || {},
requestBody: requestBody || undefined,
monitorId: monitor.id!,
monitorId: monitorId,
requestType: monitorStep.data?.requestType || HTTPMethod.GET,
retry: 10,
timeout: new PositiveNumber(60000), // 60 seconds