initial commit

Signed-off-by: jace-roell <[email protected]>

packages/imperative/src/io/__tests__/IO.unit.test.ts

@@ -288,6 +288,22 @@ describe("IO tests", () => {
         expect(error).toBeUndefined();
     });
 
+    it("processNewlines should replace LF line endings with CRLF on Windows when input is a Buffer", () => {
+        jest.spyOn(os, "platform").mockReturnValueOnce(IO.OS_WIN32);
+        const originalBuffer = Buffer.from("\nabc\ndef\n");
+        const processedBuffer = IO.processNewlines(originalBuffer);
+        const expectedBuffer = Buffer.from("\r\nabc\r\ndef\r\n");
+        expect(processedBuffer.equals(expectedBuffer)).toBe(true);
+    });
+
+    it("processNewlines should not replace LF line ending when last byte is CR and input is a Buffer", () => {
+        jest.spyOn(os, "platform").mockReturnValueOnce(IO.OS_WIN32);
+        const originalBuffer = Buffer.from("\nabc\ndef\n");
+        const processedBuffer = IO.processNewlines(originalBuffer, "\r".charCodeAt(0));
+        const expectedBuffer = Buffer.from("\nabc\r\ndef\r\n");
+        expect(processedBuffer.equals(expectedBuffer)).toBe(true);
+    });
+
     it("should throw imperative error when getting an IO error writing asynchronously", async () => {
 
         // mock fs.writeFile

packages/imperative/src/io/src/IO.ts

@@ -254,17 +254,38 @@ export class IO {
      * @returns {string} - input with removed newlines
      * @memberof IO
      */
-    public static processNewlines(original: string, lastByte?: number): string {
+    public static processNewlines<T extends string | Buffer>(original: T, lastByte?: number): T {
         ImperativeExpect.toNotBeNullOrUndefined(original, "Required parameter 'original' must not be null or undefined");
+
         if (os.platform() !== IO.OS_WIN32) {
             return original;
         }
-        // otherwise, we're on windows
-        const processed = original.replace(/([^\r])\n/g, "$1\r\n");
-        if ((lastByte == null || lastByte !== "\r".charCodeAt(0)) && original.startsWith("\n")) {
-            return "\r" + processed;
+
+        let processed: T;
+
+        if (typeof original === "string") {
+            processed = original.replace(/([^\r])\n/g, "$1\r\n") as T;
+            if ((lastByte == null || lastByte !== "\r".charCodeAt(0)) && original.startsWith("\n")) {
+                return ("\r" + processed) as T;
+            }
+            return processed;
+        } else {
+            const bufferList: number[] = [];
+            let prevByte = lastByte;
+            for (let i = 0; i < original.length; i++) {
+                const currentByte = original[i];
+                //Check if previous byte is not Carriage Return (13) and if current byte is Line Feed (10)
+                // eslint-disable-next-line @typescript-eslint/no-magic-numbers
+                if (currentByte === 10 && prevByte !== 13) {
+                    // eslint-disable-next-line @typescript-eslint/no-magic-numbers
+                    bufferList.push(13);
+                }
+                bufferList.push(currentByte);
+                prevByte = currentByte;
+            }
+            processed = Buffer.from(bufferList) as T;
+            return processed;
         }
-        return processed;
     }
 
     /**

packages/imperative/src/rest/src/client/AbstractRestClient.ts

@@ -770,7 +770,7 @@ export abstract class AbstractRestClient {
             this.log.debug("Streaming data chunk of length " + respData.length + " to response stream");
             if (this.mNormalizeResponseNewlines && this.mContentEncoding == null) {
                 this.log.debug("Normalizing new lines in data chunk to operating system appropriate line endings");
-                respData = Buffer.from(IO.processNewlines(respData.toString(), this.lastByteReceived));
+                respData = IO.processNewlines(respData, this.lastByteReceived);
             }
             if (this.mTask != null) {
                 // update the progress task if provided by the requester

packages/imperative/src/rest/src/client/CompressionUtils.ts

@@ -102,7 +102,7 @@ export class CompressionUtils {
         let lastByteReceived: number = 0;
         return new Transform({
             transform(chunk, _, callback) {
-                this.push(Buffer.from(IO.processNewlines(chunk.toString(), lastByteReceived)));
+                this.push(IO.processNewlines(chunk, lastByteReceived));
                 lastByteReceived = chunk[chunk.byteLength - 1];
                 callback();
             }

packages/zosfiles/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
+## Recent Changes
+
+- BugFix: Resolved issue where special characters could be corrupted when downloading a large file.
+
 ## `8.8.0`
 
 - Enhancement: Allows for passing a `.zosattributues` file path for the download encoding format via the `attributes` option on the `Download.ussFile` method. [#2322](https://github.com/zowe/zowe-cli/issues/2322)

packages/zosfiles/__tests__/__system__/methods/download/Download.system.test.ts

@@ -42,6 +42,7 @@ import { runCliScript } from "@zowe/cli-test-utils";
 const rimraf = require("rimraf").sync;
 const delayTime = 2000;
 const testData = "abcdefghijklmnopqrstuvwxyz";
+const specialCharTestData = fs.readFileSync(__dirname+"/__resources__/testfiles/specialCharTestData.txt");
 
 let REAL_SESSION: Session;
 let testEnvironment: ITestEnvironment<ITestPropertiesSchema>;
@@ -579,6 +580,40 @@ describe.each([false, true])("Download Data Set - Encoded: %s", (encoded: boolea
                 const regex = /\./gi;
                 file = dsname.replace(regex, "/") + "/member.dat";
             });
+            it("should download a data set member full of special characters to test buffer chunk concatenation", async () => {
+                let error;
+                let response: IZosFilesResponse;
+
+                // upload data to the newly created data set
+                await Upload.bufferToDataSet(REAL_SESSION, Buffer.from(specialCharTestData), dsname + "(member)");
+                await wait(delayTime);
+
+                try {
+                    response = await Download.allMembers(REAL_SESSION, dsname);
+                    Imperative.console.info("Response: " + inspect(response));
+                } catch (err) {
+                    error = err;
+                    Imperative.console.info("Error: " + inspect(error));
+                }
+                expect(error).toBeFalsy();
+                expect(response).toBeTruthy();
+                expect(response.success).toBeTruthy();
+                expect(response.commandResponse).toContain(
+                    ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1));
+
+                // Convert the data set name to use as a path/file
+                const regex = /\./gi;
+                file = dsname.replace(regex, "/");
+                file = file.toLowerCase();
+
+                // Compare the downloaded contents to those uploaded
+                const fileContents = stripNewLines(fs.readFileSync(`${file}/member.txt`).toString());
+                const uniqueFileChars = Array.from(new Set(fileContents)).join('');
+                const expectedUniqueChars = "àèéìòùÀÈÉÌÒÙ° "; // Expected unique characters
+
+                // Ensure the file only contains the unique characters and nothing else
+                expect(uniqueFileChars).toEqual(expectedUniqueChars);
+            });
         });
 
         describe("Data sets matching - all data sets - PO", () => {