From 736ebb6f28b13497c298d9878adf4b3a2be79c5d Mon Sep 17 00:00:00 2001
From: KernelDeimos <eric.alex.dube@gmail.com>
Date: Thu, 25 Apr 2024 19:39:01 -0400
Subject: [PATCH] Improve server health service

---
 .../runtime-analysis/ServerHealthService.js   | 24 ++++++++++++++++---
 .../thumbnails/HTTPThumbnailService.js        | 15 ++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/packages/backend/src/services/runtime-analysis/ServerHealthService.js b/packages/backend/src/services/runtime-analysis/ServerHealthService.js
index 31ca1e9c..c166d2b7 100644
--- a/packages/backend/src/services/runtime-analysis/ServerHealthService.js
+++ b/packages/backend/src/services/runtime-analysis/ServerHealthService.js
@@ -79,8 +79,9 @@ class ServerHealthService extends BaseService {
     init_service_checks_ () {
         const svc_alarm = this.services.get('alarm');
         asyncSafeSetInterval(async () => {
+            this.log.tick('service checks');
             const check_failures = [];
-            for ( const { name, fn } of this.checks_ ) {
+            for ( const { name, fn, chainable } of this.checks_ ) {
                 const p_timeout = new TeePromise();
                 const timeout = setTimeout(() => {
                     p_timeout.reject(new Error('Health check timed out'));
@@ -93,7 +94,7 @@ class ServerHealthService extends BaseService {
                     clearTimeout(timeout);
                 } catch ( err ) {
                     // Trigger an alarm if this check isn't already in the failure list
-                    
+
                     if ( this.failures_.some(v => v.name === name) ) {
                         return;
                     }
@@ -104,6 +105,15 @@ class ServerHealthService extends BaseService {
                         { error: err }
                     );
                     check_failures.push({ name });
+
+                    // Run the on_fail handlers
+                    for ( const fn of chainable.on_fail_ ) {
+                        try {
+                            await fn(err);
+                        } catch ( e ) {
+                            this.log.error(`Error in on_fail handler for ${name}`, e);
+                        }
+                    }
                 }
             }
 
@@ -124,7 +134,15 @@ class ServerHealthService extends BaseService {
     }
 
     add_check (name, fn) {
-        this.checks_.push({ name, fn });
+        const chainable = {
+            on_fail_: [],
+        };
+        chainable.on_fail = (fn) => {
+            chainable.on_fail_.push(fn);
+            return chainable;
+        };
+        this.checks_.push({ name, fn, chainable });
+        return chainable;
     }
 
     get_status () {
diff --git a/packages/backend/src/services/thumbnails/HTTPThumbnailService.js b/packages/backend/src/services/thumbnails/HTTPThumbnailService.js
index 17438d80..805d124e 100644
--- a/packages/backend/src/services/thumbnails/HTTPThumbnailService.js
+++ b/packages/backend/src/services/thumbnails/HTTPThumbnailService.js
@@ -104,6 +104,21 @@ class HTTPThumbnailService extends BaseService {
         }
     }
 
+    async _init () {
+        const services = this.services;
+        const svc_serverHealth = services.get('server-health');
+
+        svc_serverHealth.add_check('thumbnail-ping', async () => {
+            this.log.noticeme('THUMBNAIL PING');
+            await axios.request(
+                {
+                    method: 'get',
+                    url: `${this.host_}/ping`,
+                }
+            );
+        });
+    }
+
     get host_ () {
         return this.config.host || 'http://127.0.0.1:3101';
     }