Merge pull request #2137 from keboola/petr-hosek-PSGO-859

feat: Add option to download sliced file from staging storage without decompression

internal/pkg/service/cli/cmd/remote/file/download/cmd.go

@@ -15,10 +15,11 @@ import (
 )
 
 type Flags struct {
-	StorageAPIHost  configmap.Value[string] `configKey:"storage-api-host" configShorthand:"H" configUsage:"storage API host, eg. \"connection.keboola.com\""`
-	StorageAPIToken configmap.Value[string] `configKey:"storage-api-token" configShorthand:"t" configUsage:"storage API token from your project"`
-	Output          configmap.Value[string] `configKey:"output" configShorthand:"o" configUsage:"path to the destination file or directory"`
-	AllowSliced     configmap.Value[bool]   `configKey:"allow-sliced" configUsage:"output sliced files as a directory containing slices as individual files"`
+	StorageAPIHost    configmap.Value[string] `configKey:"storage-api-host" configShorthand:"H" configUsage:"storage API host, eg. \"connection.keboola.com\""`
+	StorageAPIToken   configmap.Value[string] `configKey:"storage-api-token" configShorthand:"t" configUsage:"storage API token from your project"`
+	Output            configmap.Value[string] `configKey:"output" configShorthand:"o" configUsage:"path to the destination file or directory"`
+	AllowSliced       configmap.Value[bool]   `configKey:"allow-sliced" configUsage:"output sliced files as a directory containing slices as individual files"`
+	WithoutDecompress configmap.Value[bool]   `configKey:"without-decompress" configUsage:"do not decompress the downloaded files or sliced files."`
 }
 
 func DefaultFlags() Flags {
@@ -83,9 +84,10 @@ func Command(p dependencies.Provider) *cobra.Command {
 			defer d.EventSender().SendCmdEvent(cmd.Context(), time.Now(), &cmdErr, "remote-file-download")
 
 			opts := download.Options{
-				File:        file,
-				Output:      output,
-				AllowSliced: f.AllowSliced.Value,
+				File:              file,
+				Output:            output,
+				AllowSliced:       f.AllowSliced.Value,
+				WithOutDecompress: f.WithoutDecompress,
 			}
 
 			return download.Run(cmd.Context(), opts, d)

pkg/lib/operation/project/remote/file/download/operation.go

@@ -168,16 +168,18 @@ func (d *downloader) readSliceTo(ctx context.Context, slice string, writer io.Wr
 	}
 
 	// Add decompression reader
-	if strings.HasSuffix(slice, GZIPFileExt) || (slice == "" && strings.HasSuffix(d.options.File.Name, GZIPFileExt)) {
-		if gzipReader, err := pgzip.NewReader(reader); err == nil {
-			defer func() {
-				if closeErr := gzipReader.Close(); returnErr == nil && closeErr != nil {
-					returnErr = closeErr
-				}
-			}()
-			reader = gzipReader
-		} else {
-			return errors.Errorf(`cannot create gzip reader: %w`, err)
+	if !d.options.WithOutDecompress.IsSet() {
+		if strings.HasSuffix(slice, GZIPFileExt) || (slice == "" && strings.HasSuffix(d.options.File.Name, GZIPFileExt)) {
+			if gzipReader, err := pgzip.NewReader(reader); err == nil {
+				defer func() {
+					if closeErr := gzipReader.Close(); returnErr == nil && closeErr != nil {
+						returnErr = closeErr
+					}
+				}()
+				reader = gzipReader
+			} else {
+				return errors.Errorf(`cannot create gzip reader: %w`, err)
+			}
 		}
 	}
 

pkg/lib/operation/project/remote/file/download/options.go

@@ -9,11 +9,12 @@ import (
 )
 
 type Options struct {
-	File        *keboola.FileDownloadCredentials
-	Output      string
-	AllowSliced bool
-	Columns     []string
-	Header      configmap.Value[bool]
+	File              *keboola.FileDownloadCredentials
+	Output            string
+	AllowSliced       bool
+	Columns           []string
+	Header            configmap.Value[bool]
+	WithOutDecompress configmap.Value[bool]
 }
 
 func (o *Options) ToStdout() bool {

test/cli/help/remote-file-download/args

@@ -0,0 +1 @@
+remote file download --help

test/cli/help/remote-file-download/expected-code

@@ -0,0 +1 @@
+0

test/cli/help/remote-file-download/expected-stdout

@@ -0,0 +1,16 @@
+Command "remote file download"
+
+Download a file from Keboola staging storage and save it to a local file or output to stdout.
+
+Usage:
+  kbc remote file download [file] [flags]
+
+Flags:
+      --allow-sliced               output sliced files as a directory containing slices as individual files
+  -o, --output string              path to the destination file or directory
+  -H, --storage-api-host string    storage API host, eg. "connection.keboola.com"
+  -t, --storage-api-token string   storage API token from your project
+      --without-decompress         do not decompress the downloaded files or sliced files.
+
+Global Flags:
+%A

test/cli/remote-file/download-file-disk-without-decompress/args

@@ -0,0 +1 @@
+remote file download %%TEST_FILE_TEST1_ID%% --output file.csv.gz --without-decompress --storage-api-host %%TEST_KBC_STORAGE_API_HOST%% --storage-api-token %%TEST_KBC_STORAGE_API_TOKEN%%

test/cli/remote-file/download-file-disk-without-decompress/expected-code

@@ -0,0 +1 @@
+0

test/cli/remote-file/download-file-disk-without-decompress/expected-stderr

@@ -0,0 +1 @@
+%ADownloading%A

test/cli/remote-file/download-file-disk-without-decompress/expected-stdout

@@ -0,0 +1 @@
+File "%%TEST_FILE_TEST1_ID%%" downloaded to "file.csv.gz".

test/cli/remote-file/download-file-disk-without-decompress/initial-state.json

@@ -0,0 +1,18 @@
+{
+  "branches": [
+    {
+      "branch": {
+        "name": "Main",
+        "description": "",
+        "isDefault": true
+      },
+      "configs": []
+    }
+  ],
+  "files": [
+    {
+      "name": "test1",
+      "content": "H4sIAAAAAAAAA0vOzzHUSc7PMeIqSwSygASYZQximQAAVQ0b1R0AAAA="
+    }
+  ]
+}

test/cli/remote-file/download-file-disk-without-decompress/out/file.csv.gz

@@ -0,0 +1 @@
+H4sIAAAAAAAAA0vOzzHUSc7PMeIqSwSygASYZQximQAAVQ0b1R0AAAA=