Add gzip support to fetch helper

library/agent/api/fetchBlockedIPAddresses.ts

@@ -8,6 +8,7 @@ export async function fetchBlockedIPAddresses(token: Token): Promise<string[]> {
     url: new URL(`${baseUrl.toString()}api/runtime/firewall/lists`),
     method: "GET",
     headers: {
+      // We need to set the Accept-Encoding header to "gzip" to receive the response in gzip format
       "Accept-Encoding": "gzip",
       Authorization: token.asString(),
     },

library/helpers/fetch.test.ts

@@ -1,6 +1,7 @@
 import * as t from "tap";
 import { createServer, Server } from "http";
 import { fetch } from "./fetch";
+import { gzip } from "zlib";
 
 let server: Server;
 
@@ -10,17 +11,38 @@ t.beforeEach(async () => {
     const chunks: Buffer[] = [];
     req.on("data", (chunk: Buffer) => chunks.push(chunk));
     req.on("end", () => {
-      const body = Buffer.concat(chunks).toString();
-      res.writeHead(200, { "Content-Type": "application/json" });
-      res.end(JSON.stringify({ method: req.method, body }));
+      const body = JSON.stringify({
+        method: req.method,
+        body: Buffer.concat(chunks).toString(),
+      });
+      if (req.headers["accept-encoding"] === "gzip") {
+        res.setHeader("Content-Encoding", "gzip");
+        gzip(body, (err, result) => {
+          if (err) {
+            res.writeHead(500);
+            res.end(err.message);
+            return;
+          }
+          res.writeHead(200, { "Content-Type": "application/json" });
+          res.end(result);
+        });
+      } else {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(body);
+      }
     });
   });
-  await new Promise((resolve) => server.listen(0, resolve)); // Start the server asynchronously
+  await new Promise<void>((resolve) => server.listen(0, resolve));
 });
 
 // Stop the server after running tests
 t.afterEach(async () => {
-  await new Promise((resolve) => server.close(resolve)); // Stop the server asynchronously
+  await new Promise<void>((resolve, reject) =>
+    server.close((err) => {
+      if (err) reject(err);
+      else resolve();
+    })
+  );
 });
 
 t.test("should make a GET request", async (t) => {
@@ -31,6 +53,19 @@ t.test("should make a GET request", async (t) => {
   t.same(JSON.parse(response.body), { method: "GET", body: "" });
 });
 
+t.test("should make a GET request with gzip", async (t) => {
+  const url = new URL(`http://localhost:${(server.address() as any).port}`);
+  const response = await fetch({
+    url,
+    headers: {
+      "Accept-Encoding": "gzip",
+    },
+  });
+
+  t.equal(response.statusCode, 200);
+  t.same(JSON.parse(response.body), { method: "GET", body: "" });
+});
+
 t.test("should make a POST request with body", async (t) => {
   const url = new URL(`http://localhost:${(server.address() as any).port}`);
   const response = await fetch({
@@ -46,3 +81,22 @@ t.test("should make a POST request with body", async (t) => {
     body: '{"key":"value"}',
   });
 });
+
+t.test("should make a POST request with body and gzip", async (t) => {
+  const url = new URL(`http://localhost:${(server.address() as any).port}`);
+  const response = await fetch({
+    url,
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Accept-Encoding": "gzip",
+    },
+    body: JSON.stringify({ key: "value" }),
+  });
+
+  t.equal(response.statusCode, 200);
+  t.same(JSON.parse(response.body), {
+    method: "POST",
+    body: '{"key":"value"}',
+  });
+});

library/helpers/fetch.ts

@@ -1,5 +1,7 @@
-import { request as requestHttp } from "http";
+import { IncomingMessage, request as requestHttp } from "http";
 import { request as requestHttps } from "https";
+import { type Readable } from "stream";
+import { createGunzip } from "zlib";
 
 async function request({
   url,
@@ -25,11 +27,18 @@ async function request({
         signal,
       },
       (response) => {
+        let stream: Readable = response;
+        if (response.headers["content-encoding"] === "gzip") {
+          const gunzip = createGunzip();
+          stream = response.pipe(gunzip);
+        }
+
         let data = "";
-        response.on("data", (chunk) => {
+        stream.on("data", (chunk) => {
           data += chunk;
         });
-        response.on("end", () => {
+
+        stream.on("end", () => {
           // We don't throw errors unless the request times out, is aborted or fails for low level reasons
           // Error objects are annoying to work with
           // That's why we use `resolve` instead of `reject`
@@ -50,6 +59,36 @@ async function request({
   });
 }
 
+async function handleResponse(response: IncomingMessage) {
+  return new Promise<{
+    body: string;
+    statusCode: number;
+  }>((resolve, reject) => {
+    let data = "";
+    let responseStream: Readable = response;
+
+    // Handle gzip-encoded response
+    if (response.headers && response.headers["content-encoding"] === "gzip") {
+      const gunzip = createGunzip();
+      responseStream = response.pipe(gunzip);
+    }
+
+    responseStream.on("data", (chunk) => {
+      data += chunk.toString();
+    });
+
+    responseStream.on("end", () => {
+      // We don't throw errors unless the request times out, is aborted or fails for low level reasons
+      // Error objects are annoying to work with
+      // That's why we use `resolve` instead of `reject`
+      resolve({
+        body: data,
+        statusCode: response.statusCode || 0,
+      });
+    });
+  });
+}
+
 export async function fetch({
   url,
   method = "GET",