feat: Add tarball download + unarchiving to updater

.gitignore

@@ -10,7 +10,7 @@ cosign.*
 gpg.*
 *.msi
 *.exe
-*.zip
+windows/*.zip
 windows/**/wix.dynamic.json
 windows/**/wix
 windows/config.yaml

updater/cmd/updater/main.go

@@ -16,14 +16,20 @@ package main
 
 import (
 	"fmt"
+	"log"
+	"os"
 
+	"github.com/observiq/observiq-otel-collector/updater/internal/download"
 	"github.com/observiq/observiq-otel-collector/updater/internal/version"
 	"github.com/spf13/pflag"
 )
 
 // Unimplemented
 func main() {
-	var showVersion = pflag.BoolP("version", "v", false, "prints the version of the collector")
+	var showVersion = pflag.BoolP("version", "v", false, "Prints the version of the collector and exits, if specified.")
+	var downloadURL = pflag.String("url", "", "URL to download the update archive from.")
+	var tmpDir = pflag.String("tmpdir", "", "Temporary directory for artifacts. Parent of the 'rollback' directory.")
+	var contentHash = pflag.String("content-hash", "", "Hex encoded hash of the content at the specified URL.")
 	pflag.Parse()
 
 	if *showVersion {
@@ -32,4 +38,27 @@ func main() {
 		fmt.Println("built at:", version.Date())
 		return
 	}
+
+	if *downloadURL == "" {
+		log.Println("The --url flag must be specified!")
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if *tmpDir == "" {
+		log.Println("The --tmpdir flag must be specified!")
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if *contentHash == "" {
+		log.Println("The --content-hash flag must be specified!")
+		pflag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if err := download.FetchAndExtractArchive(*downloadURL, *tmpDir, *contentHash); err != nil {
+		log.Fatalf("Failed to download and verify update: %s", err)
+	}
+
 }

updater/go.mod

@@ -3,12 +3,22 @@ module github.com/observiq/observiq-otel-collector/updater
 go 1.17
 
 require (
+	github.com/mholt/archiver/v3 v3.5.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.2
 )
 
 require (
+	github.com/andybalholm/brotli v1.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
+	github.com/golang/snappy v0.0.2 // indirect
+	github.com/klauspost/compress v1.11.4 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/nwaples/rardecode v1.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/ulikunitz/xz v0.5.9 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

updater/go.sum

@@ -1,13 +1,39 @@
+github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
+github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
+github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
+github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU=
+github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
+github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
+github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
+github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM=
+github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I=
+github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

updater/internal/download/download.go

@@ -0,0 +1,144 @@
+// Copyright  observIQ, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package download
+
+import (
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	archiver "github.com/mholt/archiver/v3"
+)
+
+const extractFolder = "latest"
+
+// Downloads the file into the outPath, truncating the file if it already exists
+func downloadFile(downloadURL string, outPath string) error {
+	//#nosec G107 HTTP request must be dynamic based on input
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		return fmt.Errorf("could not GET url: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("got non-200 status code (%d)", resp.StatusCode)
+	}
+
+	outPathClean := filepath.Clean(outPath)
+	f, err := os.OpenFile(outPathClean, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			log.Default().Printf("Failed to close file: %s", err.Error())
+		}
+	}()
+
+	if _, err = io.Copy(f, resp.Body); err != nil {
+		return fmt.Errorf("failed to copy request body to file: %w", err)
+	}
+
+	return nil
+}
+
+// getOutputFilePath gets the output path relative to the base dir for the archive from the given URL.
+func getOutputFilePath(basePath, downloadURL string) (string, error) {
+	url, err := url.Parse(downloadURL)
+	if err != nil {
+		return "", fmt.Errorf("cannot parse url: %w", err)
+	}
+
+	if url.Path == "" {
+		return "", errors.New("input url must have path")
+	}
+
+	return filepath.Join(basePath, filepath.Base(url.Path)), nil
+}
+
+func verifyContentHash(contentPath, hexExpectedContentHash string) error {
+	expectedContentHash, err := hex.DecodeString(hexExpectedContentHash)
+	if err != nil {
+		return fmt.Errorf("failed to decode content hash: %w", err)
+	}
+
+	// Hash file at contentPath using sha256
+	fileHash := sha256.New()
+	contentPathClean := filepath.Clean(contentPath)
+
+	f, err := os.Open(contentPathClean)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			log.Default().Printf("Failed to close file: %s", err.Error())
+		}
+	}()
+
+	if _, err = io.Copy(fileHash, f); err != nil {
+		return fmt.Errorf("failed to calculate file hash: %w", err)
+	}
+
+	actualContentHash := fileHash.Sum(nil)
+	if subtle.ConstantTimeCompare(expectedContentHash, actualContentHash) == 0 {
+		return errors.New("content hashes were not equal")
+	}
+
+	return nil
+}
+
+// FetchAndExtractArchive fetches the archive at the specified URL, placing it into dir.
+// It then checks to see if it matches the "expectedHash", a hex-encoded string representing the expected sha256 sum of the file.
+// If it matches, the archive is extracted into the $dir/latest directory.
+// If the archive cannot be extracted, downloaded, or verified, then an error is returned.
+func FetchAndExtractArchive(url, dir, expectedHash string) error {
+	archiveFilePath, err := getOutputFilePath(dir, url)
+	if err != nil {
+		return fmt.Errorf("failed to determine archive download path: %w", err)
+	}
+
+	if err := downloadFile(url, archiveFilePath); err != nil {
+		return fmt.Errorf("failed to download file: %w", err)
+	}
+
+	extractPath := filepath.Join(dir, extractFolder)
+
+	if err := verifyContentHash(archiveFilePath, expectedHash); err != nil {
+		return fmt.Errorf("content hash could not be verified: %w", err)
+	}
+
+	// Clean the "latest" dir before extraction
+	if err := os.RemoveAll(extractPath); err != nil {
+		return fmt.Errorf("error cleaning archive extraction target path: %w", err)
+	}
+
+	if err := archiver.Unarchive(archiveFilePath, extractPath); err != nil {
+		return fmt.Errorf("failed to extract file: %w", err)
+	}
+
+	return nil
+}