Full implementation of xhr.responseType, rigorous test and a perf imp…

lib/XMLHttpRequest.js

@@ -134,6 +134,17 @@ function XMLHttpRequest(opts) {
   this.status = null;
   this.statusText = null;
 
+  // xhr.responseType is supported:
+  //   When responseType is 'text' or '', self.responseText will be utf8 decoded text.
+  //   When responseType is 'json', self.responseText initially will be utf8 decoded text,
+  //   which is then JSON parsed into self.response.
+  //   When responseType is 'document', self.responseText initially will be utf8 decoded text,
+  //   which is then parsed by npmjs package '@xmldom/xmldom' into a DOM object self.responseXML.
+  //   When responseType is 'arraybuffer', self.response is an ArrayBuffer.
+  //   When responseType is 'blob', self.response is a Blob.
+  // cf. section 3.6, subsections 8,9,10,11 of https://xhr.spec.whatwg.org/#the-response-attribute
+  this.responseType = ""; /* 'arraybuffer' or 'text' or '' or 'json' or 'blob' or 'document' */
+
   /**
    * Private methods
    */
@@ -158,6 +169,89 @@ function XMLHttpRequest(opts) {
     return (method && forbiddenRequestMethods.indexOf(method) === -1);
   };
 
+  /**
+   * When xhr.responseType === 'arraybuffer', xhr.response must have type ArrayBuffer according
+   * to section 3.6.9 of https://xhr.spec.whatwg.org/#the-response-attribute .
+   * However, bufTotal = Buffer.concat(...) often has byteOffset > 0, so bufTotal.buffer is larger
+   * than the useable region in bufTotal. This means that a new copy of bufTotal would need to be
+   * created to get the correct ArrayBuffer. Instead, do the concat by hand to create the right
+   * sized ArrayBuffer in the first place.
+   *
+   * The return type is Uint8Array,
+   * because often Buffer will have Buffer.length < Buffer.buffer.byteLength.
+   *
+   * @param {Array<Buffer>} bufferArray
+   * @returns {Uint8Array}
+   */
+  var concat = function(bufferArray) {
+    let length = 0, offset = 0;
+    for (let k = 0; k < bufferArray.length; k++)
+      length += bufferArray[k].length;
+    const result = new Uint8Array(length);
+    for (let k = 0; k < bufferArray.length; k++)
+    {
+      result.set(bufferArray[k], offset);
+      offset += bufferArray[k].length;
+    }
+    return result;
+  };
+
+  /**
+   * When xhr.responseType === 'arraybuffer', xhr.response must have type ArrayBuffer according
+   * to section 3.6.9 of https://xhr.spec.whatwg.org/#the-response-attribute .
+   * However, buf = Buffer.from(str) often has byteOffset > 0, so buf.buffer is larger than the
+   * usable region in buf. This means that a new copy of buf would need to be created to get the
+   * correct arrayBuffer. Instead, do it by hand to create the right sized ArrayBuffer in the
+   * first place.
+   *
+   * @param {string} str
+   * @returns {Buffer}
+   */
+  var stringToBuffer = function(str) {
+    const ab = new ArrayBuffer(str.length)
+    const buf = Buffer.from(ab);
+    for (let k = 0; k < str.length; k++)
+      buf[k] = Number(str.charCodeAt(k));
+    return buf;
+  }
+
+  /**
+   * Given a Buffer buf, check whether buf.buffer.byteLength > buf.length and if so,
+   * create a new ArrayBuffer whose byteLength is buf.length, containing the bytes.
+   * of buf. This function shouldn't usually be needed, unless there's a future
+   * behavior change where buf.buffer.byteLength > buf.length unexpectedly.
+   *
+   * @param {Buffer} buf
+   * @returns {ArrayBuffer}
+   */
+  var checkAndShrinkBuffer = function(buf) {
+    if (buf.length === buf.buffer.byteLength)
+      return buf.buffer;
+    const ab = new ArrayBuffer(buf.length);
+    const result = Buffer.from(ab);
+    for (let k = 0; k < buf.length; k++)
+      result[k] = buf[k];
+    return ab;
+  }
+
+  /**
+   * Parse HTML string and return a DOM.
+   * When self.responseType is 'document', users are reuired to install the npmjs module '@xmldom/xmldom'.
+   * cf. 3.6.11 of https://xhr.spec.whatwg.org/#the-response-attribute .
+   *
+   * @param {string} - HTML to be parsed.
+   * @return {string} - DOM representation.
+   */
+  var html2dom = function(html) {
+    const DOMParser = require('@xmldom/xmldom').DOMParser;
+    try {
+      return new DOMParser().parseFromString(html, 'text/xml');
+    } catch(err) {
+      /** @todo throw appropriate DOMException. */
+    }
+    return '';
+  };
+
   /**
    * Public methods
    */
@@ -328,16 +422,17 @@ function XMLHttpRequest(opts) {
             self.handleError(error, error.errno || -1);
           } else {
             self.status = 200;
-            self.responseText = data.toString('utf8');
-            self.response = data;
+            // Use self.responseType to create the correct self.responseType, self.response, self.XMLDocument.
+            self.createFileOrSyncResponse(data);
             setState(self.DONE);
           }
         });
       } else {
         try {
-          this.response = fs.readFileSync(unescape(url.pathname));
-          this.responseText = this.response.toString('utf8');
           this.status = 200;
+          const syncData = fs.readFileSync(unescape(url.pathname));
+          // Use self.responseType to create the correct self.responseType, self.response, self.XMLDocument.
+          this.createFileOrSyncResponse(syncData);
           setState(self.DONE);
         } catch(e) {
           this.handleError(e, e.errno || -1);
@@ -421,6 +516,8 @@ function XMLHttpRequest(opts) {
         // Set response var to the response we got back
         // This is so it remains accessable outside this scope
         response = resp;
+        // Collect buffers and concatenate once.
+        const buffers = [];
         // Check for redirect
         // @TODO Prevent looped redirects
         if (response.statusCode === 302 || response.statusCode === 303 || response.statusCode === 307) {
@@ -456,13 +553,34 @@ function XMLHttpRequest(opts) {
         }
 
         setState(self.HEADERS_RECEIVED);
+
+        // When responseType is 'text' or '', self.responseText will be utf8 decoded text.
+        // When responseType is 'json', self.responseText initially will be utf8 decoded text,
+        // which is then JSON parsed into self.response.
+        // When responseType is 'document', self.responseText initially will be utf8 decoded text,
+        // which is then parsed by npmjs package '@xmldom/xmldom'into a DOM object self.responseXML.
+        // When responseType is 'arraybuffer', self.response is an ArrayBuffer.
+        // When responseType is 'blob', self.response is a Blob.
+        // cf. section 3.6, subsections 8,9,10,11 of https://xhr.spec.whatwg.org/#the-response-attribute
+        const isUtf8 = self.responseType === "" || self.responseType === "text"
+          || self.responseType === "json" || self.responseType === "document";
+        if (isUtf8 && response.setEncoding) {
+          response.setEncoding("utf8");
+        }
+
         self.status = response.statusCode;
 
         response.on('data', function(chunk) {
           // Make sure there's some data
           if (chunk) {
-            var data = Buffer.from(chunk);
-            self.response = Buffer.concat([self.response, data]);
+            if (isUtf8) {
+              // When responseType is 'text', '', 'json' or 'document',
+              //   then each chunk is already utf8 decoded.
+              self.responseText += chunk;
+            } else {
+              // Otherwise collect the chunk buffers.
+              buffers.push(chunk);
+            }
           }
           // Don't emit state changes if the connection has been aborted.
           if (sendFlag) {
@@ -475,10 +593,10 @@ function XMLHttpRequest(opts) {
             // The sendFlag needs to be set before setState is called.  Otherwise if we are chaining callbacks
             // there can be a timing issue (the callback is called and a new call is made before the flag is reset).
             sendFlag = false;
+            // Create the correct response for responseType.
+            self.createResponse(buffers);
             // Discard the 'end' event if the connection has been aborted
             setState(self.DONE);
-            // Construct responseText from response
-            self.responseText = self.response.toString('utf8');
           }
         });
 
@@ -516,33 +634,43 @@ function XMLHttpRequest(opts) {
       fs.writeFileSync(syncFile, "", "utf8");
       // The async request the other Node process executes
       var execString = "var http = require('http'), https = require('https'), fs = require('fs');"
+        + "function concat(bufferArray) {"
+        + "  let length = 0, offset = 0;"
+        + "  for (let k = 0; k < bufferArray.length; k++)"
+        + "    length += bufferArray[k].length;"
+        + "  const result = Buffer.alloc(length);"
+        + "  for (let k = 0; k < bufferArray.length; k++)"
+        + "  {"
+        + "    result.set(bufferArray[k], offset);"
+        + "    offset += bufferArray[k].length;"
+        + "  }"
+        + "  return result;"
+        + "};"
         + "var doRequest = http" + (ssl ? "s" : "") + ".request;"
         + "var options = " + JSON.stringify(options) + ";"
-        + "var responseText = '';"
         + "var responseData = Buffer.alloc(0);"
+        + "var buffers = [];"
         + "var req = doRequest(options, function(response) {"
-        + "response.on('data', function(chunk) {"
-        + "  var data = Buffer.from(chunk);"
-        + "  responseText += data.toString('utf8');"
-        + "  responseData = Buffer.concat([responseData, data]);"
-        + "});"
-        + "response.on('end', function() {"
-        + "fs.writeFileSync('" + contentFile + "', JSON.stringify({err: null, data: {statusCode: response.statusCode, headers: response.headers, text: responseText, data: responseData.toString('base64')}}), 'utf8');"
-        + "fs.unlinkSync('" + syncFile + "');"
-        + "});"
-        + "response.on('error', function(error) {"
-        + "fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
-        + "fs.unlinkSync('" + syncFile + "');"
-        + "});"
+        + "  response.on('data', function(chunk) {"
+        + "    buffers.push(chunk);"
+        + "  });"
+        + "  response.on('end', function() {"
+        + "    responseData = concat(buffers);"
+        + "    fs.writeFileSync('" + contentFile + "', JSON.stringify({err: null, data: {statusCode: response.statusCode, headers: response.headers, data: responseData.toString('utf8')}}), 'utf8');"
+        + "    fs.unlinkSync('" + syncFile + "');"
+        + "  });"
+        + "  response.on('error', function(error) {"
+        + "    fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+        + "    fs.unlinkSync('" + syncFile + "');"
+        + "  });"
         + "}).on('error', function(error) {"
-        + "fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
-        + "fs.unlinkSync('" + syncFile + "');"
+        + "  fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+        + "  fs.unlinkSync('" + syncFile + "');"
         + "});"
         + (data ? "req.write('" + JSON.stringify(data).slice(1,-1).replace(/'/g, "\\'") + "');":"")
         + "req.end();";
       // Start the other Node Process, executing this string
       var syncProc = spawn(process.argv[0], ["-e", execString]);
-      var statusText;
       while(fs.existsSync(syncFile)) {
         // Wait while the sync file is empty
       }
@@ -557,16 +685,19 @@ function XMLHttpRequest(opts) {
         self.handleError(errorObj, 503);
       } else {
         // If the file returned okay, parse its data and move to the DONE state
-        self.status = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, "$1");
-        var resp = JSON.parse(self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, "$1"));
+        const resp = JSON.parse(self.responseText);
+        self.status = resp.data.statusCode;
+        self.response = stringToBuffer(resp.data.data);
+        // Use self.responseType to create the correct self.responseType, self.response, self.responseXML.
+        self.createFileOrSyncResponse(self.response);
+        // Set up response correctly.
         response = {
           statusCode: self.status,
           headers: resp.data.headers
         };
-        self.responseText = resp.data.text;
-        self.response = Buffer.from(resp.data.data, 'base64');
-        setState(self.DONE, true);
+        setState(self.DONE);
       }
+
     }
   };
 
@@ -578,6 +709,8 @@ function XMLHttpRequest(opts) {
     this.status = status || 0;
     this.statusText = error;
     this.responseText = error.stack;
+    this.responseXML = "";
+    this.response = Buffer.alloc(0);
     errorFlag = true;
     setState(this.DONE);
   };
@@ -650,6 +783,87 @@ function XMLHttpRequest(opts) {
     }
   };
 
+  /**
+   * Construct the correct form of response, given responseType when in non-file based, asynchronous mode.
+   *
+   * When self.responseType is "", "text", "json", "document", self.responseText is a utf8 string.
+   * When self.responseType is "arraybuffer", "blob", the response is in the buffers parameter,
+   * an Array of Buffers. Then concat(buffers) is Uint8Array, from which checkAndShrinkBuffer
+   * extracts the correct sized ArrayBuffer.
+   *
+   * @param {Array<Buffer>} buffers
+   */
+  this.createResponse = function(buffers) {
+    self.responseXML = '';
+    switch (self.responseType) {
+      case "":
+      case "text":
+        self.response = Buffer.alloc(0);
+        break;
+      case 'json':
+        self.response = JSON.parse(self.responseText);
+        self.responseText = '';
+        break;
+      case "document":
+        const xml = JSON.parse(self.responseText);
+        self.responseXML = html2dom(xml);
+        self.responseText = '';
+        self.response = Buffer.alloc(0);
+        break;
+      default:
+        self.responseText = '';
+        const totalResponse = concat(buffers);
+        // When self.responseType === 'arraybuffer', self.response is an ArrayBuffer.
+        // Get the correct sized ArrayBuffer.
+        self.response = checkAndShrinkBuffer(totalResponse);
+        if (self.responseType === 'blob' && typeof Blob === 'function') {
+          // Construct the Blob object that contains response.
+          self.response = new Blob([self.response]);
+        }
+        break;
+    }
+  }
+
+  /**
+   * Construct the correct form of response, given responseType when in synchronous mode or file based.
+   *
+   * The input is the response parameter which is a Buffer.
+   * When self.responseType is "", "text", "json", "document",
+   *   the input is further refined to be: response.toString('utf8').
+   * When self.responseType is "arraybuffer", "blob",
+   *   the input is further refined to be: checkAndShrinkBuffer(response).
+   *
+   * @param {Buffer} response
+   */
+  this.createFileOrSyncResponse = function(response) {
+    self.responseText = '';
+    self.responseXML = '';
+    switch (self.responseType) {
+      case "":
+      case "text":
+        self.responseText = response.toString('utf8');
+        self.response = Buffer.alloc(0);
+        break;
+      case 'json':
+        self.response = JSON.parse(response.toString('utf8'));
+        break;
+      case "document":
+        const xml = JSON.parse(response.toString('utf8'));
+        self.responseXML = html2dom(xml);
+        self.response = Buffer.alloc(0);
+        break;
+      default:
+        // When self.responseType === 'arraybuffer', self.response is an ArrayBuffer.
+        // Get the correct sized ArrayBuffer.
+        self.response = checkAndShrinkBuffer(response);
+        if (self.responseType === 'blob' && typeof Blob === 'function') {
+          // Construct the Blob object that contains response.
+          self.response = new Blob([self.response]);
+        }
+        break;
+    }
+  }
+
   /**
    * Changes readyState and calls onreadystatechange.
    *

tests/server.js

@@ -1,3 +1,4 @@
+
 var http = require("http");
 
 var server = http.createServer(function (req, res) {
@@ -14,10 +15,18 @@ var server = http.createServer(function (req, res) {
             return res
                 .writeHead(200, {"Content-Type": "application/json"})
                 .end(JSON.stringify({ foo: "bar" }));
-        case "/binary":
+        case "/binary1":
             return res
                 .writeHead(200, {"Content-Type": "application/octet-stream"})
                 .end(Buffer.from("Hello world!"));
+        case "/binary2": {
+            const ta = new Float32Array([1, 5, 6, 7]);
+            const buf = Buffer.from(ta.buffer);
+            const str = buf.toString('binary');
+            return res
+                .writeHead(200, {"Content-Type": "application/octet-stream"})
+                .end(str);
+	}
         default:
             return res
                 .writeHead(404, {"Content-Type": "text/plain"})