feat: Add configurable response data storage for notifications (#6684)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Frank Elsinga <frank@elsinga.de>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

server/model/heartbeat.js

@@ -1,4 +1,7 @@
 const { BeanModel } = require("redbean-node/dist/bean-model");
+const zlib = require("node:zlib");
+const { promisify } = require("node:util");
+const gunzip = promisify(zlib.gunzip);
 
 /**
  * status:
@@ -36,8 +39,46 @@ class Heartbeat extends BeanModel {
             important: this._important,
             duration: this._duration,
             retries: this._retries,
+            response: this._response,
         };
     }
+
+    /**
+     * Return an object that ready to parse to JSON
+     * @param {{ decodeResponse?: boolean }} opts Options for JSON serialization
+     * @returns {Promise<object>} Object ready to parse
+     */
+    async toJSONAsync(opts) {
+        return {
+            monitorID: this._monitorId,
+            status: this._status,
+            time: this._time,
+            msg: this._msg,
+            ping: this._ping,
+            important: this._important,
+            duration: this._duration,
+            retries: this._retries,
+            response: opts?.decodeResponse ? await Heartbeat.decodeResponseValue(this._response) : undefined,
+        };
+    }
+
+    /**
+     * Decode compressed response payload stored in database.
+     * @param {string|null} response Encoded response payload.
+     * @returns {string|null} Decoded response payload.
+     */
+    static async decodeResponseValue(response) {
+        if (!response) {
+            return response;
+        }
+
+        try {
+            // Offload gzip decode from main event loop to libuv thread pool
+            return (await gunzip(Buffer.from(response, "base64"))).toString("utf8");
+        } catch (error) {
+            return response;
+        }
+    }
 }
 
 module.exports = Heartbeat;

server/model/monitor.js

@@ -24,6 +24,8 @@ const {
     PING_PER_REQUEST_TIMEOUT_MIN,
     PING_PER_REQUEST_TIMEOUT_MAX,
     PING_PER_REQUEST_TIMEOUT_DEFAULT,
+    RESPONSE_BODY_LENGTH_DEFAULT,
+    RESPONSE_BODY_LENGTH_MAX,
 } = require("../../src/util");
 const {
     ping,
@@ -56,6 +58,9 @@ const { CookieJar } = require("tough-cookie");
 const { HttpsCookieAgent } = require("http-cookie-agent/http");
 const https = require("https");
 const http = require("http");
+const zlib = require("node:zlib");
+const { promisify } = require("node:util");
+const gzip = promisify(zlib.gzip);
 const DomainExpiry = require("./domain_expiry");
 
 const rootCertificates = rootCertificatesFingerprints();
@@ -203,6 +208,11 @@ class Monitor extends BeanModel {
             ping_numeric: this.isPingNumeric(),
             ping_count: this.ping_count,
             ping_per_request_timeout: this.ping_per_request_timeout,
+
+            // response saving options
+            saveResponse: this.getSaveResponse(),
+            saveErrorResponse: this.getSaveErrorResponse(),
+            responseMaxLength: this.response_max_length ?? RESPONSE_BODY_LENGTH_DEFAULT,
         };
 
         if (includeSensitiveData) {
@@ -386,6 +396,22 @@ class Monitor extends BeanModel {
         return Boolean(this.kafkaProducerAllowAutoTopicCreation);
     }
 
+    /**
+     * Parse to boolean
+     * @returns {boolean} Should save response data on success?
+     */
+    getSaveResponse() {
+        return Boolean(this.save_response);
+    }
+
+    /**
+     * Parse to boolean
+     * @returns {boolean} Should save response data on error?
+     */
+    getSaveErrorResponse() {
+        return Boolean(this.save_error_response);
+    }
+
     /**
      * Start monitor
      * @param {Server} io Socket server instance
@@ -620,6 +646,11 @@ class Monitor extends BeanModel {
                     bean.msg = `${res.status} - ${res.statusText}`;
                     bean.ping = dayjs().valueOf() - startTime;
 
+                    // in the frontend, the save response is only shown if the saveErrorResponse is set
+                    if (this.getSaveResponse() && this.getSaveErrorResponse()) {
+                        await this.saveResponseData(bean, res.data);
+                    }
+
                     // fallback for if kelog event is not emitted, but we may still have tlsInfo,
                     // e.g. if the connection is made through a proxy
                     if (this.getUrl()?.protocol === "https:" && tlsInfo.valid === undefined) {
@@ -931,6 +962,10 @@ class Monitor extends BeanModel {
                     bean.msg = error.message;
                 }
 
+                if (this.getSaveErrorResponse() && error?.response?.data !== undefined) {
+                    await this.saveResponseData(bean, error.response.data);
+                }
+
                 // If UP come in here, it must be upside down mode
                 // Just reset the retries
                 if (this.isUpsideDown() && bean.status === UP) {
@@ -1114,6 +1149,35 @@ class Monitor extends BeanModel {
         }
     }
 
+    /**
+     * Save response body to a heartbeat if response saving is enabled.
+     * @param {import("redbean-node").Bean} bean Heartbeat bean to populate.
+     * @param {unknown} data Response payload.
+     * @returns {void}
+     */
+    async saveResponseData(bean, data) {
+        if (data === undefined) {
+            return;
+        }
+
+        let responseData = data;
+        if (typeof responseData !== "string") {
+            try {
+                responseData = JSON.stringify(responseData);
+            } catch (error) {
+                responseData = String(responseData);
+            }
+        }
+
+        const maxSize = this.response_max_length ?? RESPONSE_BODY_LENGTH_DEFAULT;
+        if (responseData.length > maxSize) {
+            responseData = responseData.substring(0, maxSize) + "... (truncated)";
+        }
+
+        // Offload gzip compression from main event loop to libuv thread pool
+        bean.response = (await gzip(Buffer.from(responseData, "utf8"))).toString("base64");
+    }
+
     /**
      * Make a request using axios
      * @param {object} options Options for Axios
@@ -1417,7 +1481,7 @@ class Monitor extends BeanModel {
      * Send a notification about a monitor
      * @param {boolean} isFirstBeat Is this beat the first of this monitor?
      * @param {Monitor} monitor The monitor to send a notification about
-     * @param {Bean} bean Status information about monitor
+     * @param {import("./heartbeat")} bean Status information about monitor
      * @returns {Promise<void>}
      */
     static async sendNotification(isFirstBeat, monitor, bean) {
@@ -1435,7 +1499,7 @@ class Monitor extends BeanModel {
 
             for (let notification of notificationList) {
                 try {
-                    const heartbeatJSON = bean.toJSON();
+                    const heartbeatJSON = await bean.toJSONAsync({ decodeResponse: true });
                     const monitorData = [{ id: monitor.id, active: monitor.active, name: monitor.name }];
                     const preloadData = await Monitor.preparePreloadData(monitorData);
                     // Prevent if the msg is undefined, notifications such as Discord cannot send out.
@@ -1642,6 +1706,16 @@ class Monitor extends BeanModel {
             throw new Error(`Retry interval cannot be less than ${MIN_INTERVAL_SECOND} seconds`);
         }
 
+        if (this.response_max_length !== undefined) {
+            if (this.response_max_length < 0) {
+                throw new Error(`Response max length cannot be less than 0`);
+            }
+
+            if (this.response_max_length > RESPONSE_BODY_LENGTH_MAX) {
+                throw new Error(`Response max length cannot be more than ${RESPONSE_BODY_LENGTH_MAX} bytes`);
+            }
+        }
+
         if (this.type === "ping") {
             // ping parameters validation
             if (this.packetSize && (this.packetSize < PING_PACKET_SIZE_MIN || this.packetSize > PING_PACKET_SIZE_MAX)) {