Add tests for tar

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/windows v1.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect

go.sum

@@ -33,6 +33,9 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -77,8 +80,10 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
 github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -106,6 +111,7 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

tar.go

@@ -62,7 +62,7 @@ func ExtractTarZ(xFile *XFile) (int64, []string, error) {
 
 	zipStream, err := lzw.NewReader(compressedFile)
 	if err != nil {
-		return 0, nil, fmt.Errorf("xz.NewReader: %w", err)
+		return 0, nil, fmt.Errorf("lzw.NewReader: %w", err)
 	}
 
 	return xFile.untar(tar.NewReader(zipStream))

tar_test.go

@@ -0,0 +1,258 @@
+package xtractr_test
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/dsnet/compress/bzip2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/ulikunitz/xz"
+	"golift.io/xtractr"
+)
+
+type compressor interface {
+	Compress(t *testing.T, sourceDir string, destFileBase string) error
+}
+
+type (
+	tarCompressor     struct{}
+	tarZCompressor    struct{}
+	tarBzipCompressor struct{}
+	tarXZCompressor   struct{}
+	tarGzipCompressor struct{}
+)
+
+func TestTar(t *testing.T) {
+	t.Parallel()
+	const (
+		loremIpsum = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, 
+sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip
+ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit 
+esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.`
+		testDataSize     = 1544
+		testFileCount    = 5
+		testArchiveCount = 1
+	)
+
+	randomDigits := []uint8{
+		157, 242, 143, 106, 163, 159, 194, 141, 32, 22, 249, 78,
+		225, 206, 190, 199, 99, 146, 53, 149, 239, 179, 72, 2, 197, 196, 91, 81, 192,
+		241, 69, 166, 213, 172, 111, 117, 210, 51, 136, 185, 130, 109, 139, 57, 150, 63,
+		85, 86, 204, 10, 26, 1, 186, 234, 96, 187, 205, 138, 224, 77, 114, 226, 16, 222,
+		151, 175, 200, 116, 36, 198, 173, 168, 230, 4, 18, 245, 31, 214, 158, 105, 171,
+		123, 195, 137, 40, 93, 83, 215, 6, 118, 161, 223, 43, 167, 7, 3, 113, 148, 201,
+		125,
+	}
+
+	testFiles := []string{
+		"README.txt",
+		"level1/",
+		"level1/level1.txt",
+		"level1/level1.bin",
+		"level1/level2/",
+		"level1/level2/level2.txt",
+		"level1/level2/level2.bin",
+	}
+
+	testDataDir, err := os.MkdirTemp(".", "xtractr_test_*_data")
+	require.NoError(t, err, "creating temp directory failed")
+	t.Cleanup(func() {
+		os.RemoveAll(testDataDir)
+	})
+
+	sourceFilesDir := filepath.Join(testDataDir, "sources")
+	err = os.MkdirAll(sourceFilesDir, 0o700)
+	require.NoError(t, err)
+
+	var destFilesDir string
+	for _, file := range testFiles {
+		fullPath := filepath.Join(sourceFilesDir, file)
+		var err error
+		switch {
+		case file[len(file)-1] == '/':
+			err = os.MkdirAll(fullPath, 0o700)
+		case filepath.Ext(file) == ".txt":
+			err = os.WriteFile(fullPath, []byte(loremIpsum), 0o600)
+		default:
+			err = os.WriteFile(fullPath, randomDigits, 0o600)
+		}
+		require.NoError(t, err)
+
+		destFilesDir = filepath.Join(testDataDir, "out")
+		err = os.MkdirAll(destFilesDir, 0o700)
+		require.NoError(t, err)
+	}
+
+	tests := []struct {
+		name        string
+		compressor  compressor
+		extension   string
+		skipWindows bool
+	}{
+		{"tar", &tarCompressor{}, "tar", false},
+		{"tarZ", &tarZCompressor{}, "tar.z", true},
+		{"tarBzip", &tarBzipCompressor{}, "tar.bz2", false},
+		{"tarXZ", &tarXZCompressor{}, "tar.xz", false},
+		{"tarGzip", &tarGzipCompressor{}, "tar.gz", false},
+	}
+
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			if runtime.GOOS == "windows" && test.skipWindows {
+				t.Log("skipping test on windows")
+			}
+
+			archiveBase := filepath.Join(destFilesDir, "archive")
+			err := test.compressor.Compress(t, sourceFilesDir, archiveBase)
+			require.NoError(t, err)
+
+			extractDir := filepath.Join(destFilesDir, test.name)
+			err = os.Mkdir(extractDir, 0o700)
+			require.NoError(t, err)
+
+			size, files, archives, err := xtractr.ExtractFile(&xtractr.XFile{
+				FilePath:  archiveBase + "." + test.extension,
+				OutputDir: filepath.Clean(extractDir),
+				FileMode:  0o600,
+				DirMode:   0o700,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, int64(testDataSize), size)
+			assert.Len(t, files, testFileCount)
+			assert.Len(t, archives, testArchiveCount)
+		})
+	}
+}
+
+func safeCloser(t *testing.T, c io.Closer) {
+	t.Helper()
+	err := c.Close()
+	require.NoError(t, err)
+}
+
+func writeTar(sourceDir string, destWriter io.Writer) error {
+	tarWriter := tar.NewWriter(destWriter)
+	outErr := filepath.Walk(sourceDir, func(path string, info os.FileInfo, _ error) error {
+		if info.Mode().IsDir() {
+			return nil
+		}
+		relativePath := path[len(sourceDir):]
+		if len(relativePath) == 0 {
+			return nil
+		}
+		fileReader, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("failed to open file: %w", err)
+		}
+		defer fileReader.Close()
+
+		h, err := tar.FileInfoHeader(info, relativePath)
+		if err != nil {
+			return fmt.Errorf("failed to create tar header: %w", err)
+		}
+		h.Name = relativePath
+		if err = tarWriter.WriteHeader(h); err != nil {
+			return fmt.Errorf("failed to write tar header: %w", err)
+		}
+		if _, err := io.Copy(tarWriter, fileReader); err != nil {
+			return fmt.Errorf("failed to copy file (%s) into tar header: %w", fileReader.Name(), err)
+		}
+		return nil
+	})
+
+	if outErr == nil {
+		return nil
+	}
+	return fmt.Errorf("failed to walk source directory: %w", outErr)
+}
+
+func (c *tarCompressor) Compress(t *testing.T, sourceDir string, destBase string) error {
+	t.Helper()
+	tarFile, err := os.Create(destBase + ".tar")
+	defer safeCloser(t, tarFile)
+	require.NoError(t, err)
+
+	return writeTar(sourceDir, tarFile)
+}
+
+func (c *tarZCompressor) Compress(t *testing.T, _ string, destBase string) error {
+	t.Helper()
+
+	// No native Go library for .tar.Z and compress Unix utility is not available on
+	// Windows. So, we use a pre-compressed file from test_data.
+	tarZFilename := destBase + ".tar.z"
+	tarZDestFile, err := os.Create(tarZFilename)
+	require.NoError(t, err)
+
+	tarZTestFile, err := os.Open("test_data/archive.tar.Z")
+	require.NoError(t, err)
+
+	written, err := io.Copy(tarZDestFile, tarZTestFile)
+	assert.Greater(t, written, int64(0))
+	require.NoError(t, err)
+
+	return nil
+}
+
+func (c *tarBzipCompressor) Compress(t *testing.T, sourceDir string, destBase string) error {
+	t.Helper()
+	tarBZ2Filename := destBase + ".tar.bz2"
+
+	tarBZ2File, err := os.Create(tarBZ2Filename)
+	require.NoError(t, err)
+
+	bzip2Writer, err := bzip2.NewWriter(tarBZ2File, &bzip2.WriterConfig{Level: bzip2.BestSpeed})
+	defer safeCloser(t, bzip2Writer)
+	require.NoError(t, err)
+
+	err = writeTar(sourceDir, bzip2Writer)
+	require.NoError(t, err)
+
+	return nil
+}
+
+func (c *tarXZCompressor) Compress(t *testing.T, sourceDir string, destBase string) error {
+	t.Helper()
+	tarXZFilename := destBase + ".tar.xz"
+
+	tarXZFile, err := os.Create(tarXZFilename)
+	require.NoError(t, err)
+
+	xzWriter, err := xz.NewWriter(tarXZFile)
+	defer safeCloser(t, xzWriter)
+	require.NoError(t, err)
+
+	err = writeTar(sourceDir, xzWriter)
+	require.NoError(t, err)
+
+	return nil
+}
+
+func (c *tarGzipCompressor) Compress(t *testing.T, sourceDir string, destBase string) error {
+	t.Helper()
+	tarGZFilename := destBase + ".tar.gz"
+
+	tarGZFile, err := os.Create(tarGZFilename)
+	require.NoError(t, err)
+
+	gzipWriter := gzip.NewWriter(tarGZFile)
+	defer gzipWriter.Close()
+	require.NoError(t, err)
+
+	err = writeTar(sourceDir, gzipWriter)
+	require.NoError(t, err)
+
+	return nil
+}