Merge #38521

38521: importccl: ignore query params when detecting file compression from suffix r=dt a=dt This changes the detection of the compression scheme to, if it does not find the suffix it is looking for, run again after parsing the filename as a URI and only using the Path component. Fixes #32100. Release note (sql change): enable automatic detection of compression schemes on filenames in IMPORT that have parameters after the path. Co-authored-by: David Taylor <[email protected]>
cockroachdb · Jun 28, 2019 · 537767a · 537767a
2 parents d964db8 + 4071154
commit 537767a
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/pkg/ccl/importccl/import_stmt_test.go b/pkg/ccl/importccl/import_stmt_test.go
@@ -928,7 +928,7 @@ func TestImportCSVStmt(t *testing.T) {
 
 	gzip := make([]string, len(files))
 	for i := range files {
-		gzip[i] = strings.TrimPrefix(gzipFile(t, filepath.Join(dir, files[i])), dir)
+		gzip[i] = strings.TrimPrefix(gzipFile(t, filepath.Join(dir, files[i])), dir) + "?param=value"
 	}
 	gzip = nodelocalPrefix(gzip)
 
@@ -1164,7 +1164,7 @@ func TestImportCSVStmt(t *testing.T) {
 
 			if err := jobutils.VerifySystemJob(t, sqlDB, testNum, jobspb.TypeImport, jobs.StatusSucceeded, jobs.Record{
 				Username:    security.RootUser,
-				Description: fmt.Sprintf(jobPrefix+` CSV DATA (%s)`+tc.jobOpts, strings.Join(tc.files, ", ")),
+				Description: fmt.Sprintf(jobPrefix+` CSV DATA (%s)`+tc.jobOpts, strings.ReplaceAll(strings.Join(tc.files, ", "), "?param=value", "")),
 			}); err != nil {
 				t.Fatal(err)
 			}

diff --git a/pkg/ccl/importccl/read_import_proc.go b/pkg/ccl/importccl/read_import_proc.go
@@ -15,6 +15,7 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"net/url"
 	"strings"
 	"time"
 
@@ -174,6 +175,9 @@ func guessCompressionFromName(
 	case strings.HasSuffix(name, ".bz2") || strings.HasSuffix(name, ".bz"):
 		return roachpb.IOFileFormat_Bzip
 	default:
+		if parsed, err := url.Parse(name); err == nil && parsed.Path != name {
+			return guessCompressionFromName(parsed.Path, hint)
+		}
 		return roachpb.IOFileFormat_None
 	}
 }