From c4de5b40681c9174bb7d14efcde92b4116e6114f Mon Sep 17 00:00:00 2001 From: 9547 Date: Thu, 3 Sep 2020 22:31:07 +0800 Subject: [PATCH 01/13] repository,cluster: download target in stream mode --- pkg/cluster/clusterutil/cluster.go | 15 +-------------- pkg/repository/mirror.go | 4 ++++ pkg/repository/v1_repository.go | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pkg/cluster/clusterutil/cluster.go b/pkg/cluster/clusterutil/cluster.go index 6e3f5d69c2..379a8130ad 100644 --- a/pkg/cluster/clusterutil/cluster.go +++ b/pkg/cluster/clusterutil/cluster.go @@ -14,7 +14,6 @@ package clusterutil import ( - "io" "os" "github.com/pingcap/tiup/pkg/environment" @@ -59,19 +58,7 @@ func (r *repositoryT) DownloadComponent(comp, version, target string) error { return err } - reader, err := r.repo.FetchComponent(versionItem) - if err != nil { - return err - } - - file, err := os.Create(target) - if err != nil { - return err - } - defer file.Close() - - _, err = io.Copy(file, reader) - return err + return r.repo.DownloadComponent(versionItem, target) } func (r *repositoryT) VerifyComponent(comp, version, target string) error { diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index 8c8ef50d59..75a63409aa 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -253,6 +253,10 @@ func (l *httpMirror) Download(resource, targetDir string) error { return errors.Trace(err) } defer r.Close() + + if err := os.MkdirAll(targetDir, 0755); err != nil { + return errors.Trace(err) + } return utils.Move(tmpFilePath, dstFilePath) } diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index f56a3166c4..7264ce6320 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -20,6 +20,7 @@ import ( stderrors "errors" "fmt" "io" + "os" "path/filepath" "runtime" "strconv" @@ -486,6 +487,24 @@ func (r *V1Repository) FetchComponent(item *v1manifest.VersionItem) (io.Reader, return checkHash(reader, item.Hashes[v1manifest.SHA256]) } +// DoDownloadComponent downloads the component specified by item into local file +func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target string) error { + targetDir := filepath.Dir(target) + err := r.mirror.Download(item.URL, targetDir) + if err != nil { + return errors.Trace(err) + } + + reader, err := os.Open(target) + if err != nil { + return err + } + defer reader.Close() + + _, err = checkHash(reader, item.Hashes[v1manifest.SHA256]) + return err +} + // FetchTimestamp downloads the timestamp file, validates it, and checks if the snapshot hash in it // has the same value of our local one. (not hashing the snapshot file itself) // Return weather the manifest is changed compared to the one in local ts and the FileHash of snapshot. From 651878abcb8cd40757d133adf08e6e3e34e5ef45 Mon Sep 17 00:00:00 2001 From: 9547 Date: Fri, 4 Sep 2020 00:50:49 +0800 Subject: [PATCH 02/13] repository: UpdateManifests use Download to repalce Fetch --- pkg/repository/v1_repository.go | 19 +++++++++++++++---- pkg/repository/v1manifest/local_manifests.go | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 7264ce6320..24c2571cec 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -30,6 +30,7 @@ import ( "github.com/fatih/color" cjson "github.com/gibson042/canonicaljson-go" "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/repository/v0manifest" "github.com/pingcap/tiup/pkg/repository/v1manifest" "github.com/pingcap/tiup/pkg/utils" @@ -154,16 +155,26 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error { } } - reader, err := r.FetchComponent(versionItem) + if spec.Version == "" { + spec.Version = version + } + + targetDir := filepath.Join(r.local.TargetRootDir(), localdata.ComponentParentDir, spec.ID, spec.Version) + target := filepath.Join(targetDir, versionItem.URL) + err = r.DownloadComponent(versionItem, target) if err != nil { errs = append(errs, err.Error()) continue } - if spec.Version == "" { - spec.Version = version + reader, err := os.Open(target) + if err != nil { + errs = append(errs, err.Error()) + continue } - err = r.local.InstallComponent(reader, spec.TargetDir, spec.ID, spec.Version, versionItem.URL, r.DisableDecompress) + defer reader.Close() + + err = r.local.InstallComponent(reader, targetDir, spec.ID, spec.Version, versionItem.URL, r.DisableDecompress) if err != nil { errs = append(errs, err.Error()) continue diff --git a/pkg/repository/v1manifest/local_manifests.go b/pkg/repository/v1manifest/local_manifests.go index 5d40937655..bcc6907ad8 100644 --- a/pkg/repository/v1manifest/local_manifests.go +++ b/pkg/repository/v1manifest/local_manifests.go @@ -49,6 +49,9 @@ type LocalManifests interface { // ManifestVersion opens filename, if it exists and is a manifest, returns its manifest version number. Otherwise // returns 0. ManifestVersion(filename string) uint + + // TargetRootDir returns the root directory of target + TargetRootDir() string } // FsManifests represents a collection of v1 manifests on disk. @@ -230,11 +233,6 @@ func (ms *FsManifests) ComponentInstalled(component, version string) (bool, erro // InstallComponent implements LocalManifests. func (ms *FsManifests) InstallComponent(reader io.Reader, targetDir, component, version, filename string, noExpand bool) error { - // TODO factor path construction to profile (also used by v0 repo). - if targetDir == "" { - targetDir = ms.profile.Path(localdata.ComponentParentDir, component, version) - } - if !noExpand { return utils.Untar(reader, targetDir) } @@ -281,6 +279,11 @@ func (ms *FsManifests) ManifestVersion(filename string) uint { return base.Version } +// TargetRootDir implements LocalManifests. +func (ms *FsManifests) TargetRootDir() string { + return ms.profile.Root() +} + // MockManifests is a LocalManifests implementation for testing. type MockManifests struct { Manifests map[string]*Manifest @@ -392,6 +395,11 @@ func (ms *MockManifests) KeyStore() *KeyStore { return ms.Ks } +// TargetRootDir implements LocalManifests. +func (ms *MockManifests) TargetRootDir() string { + return "" +} + // ManifestVersion implements LocalManifests. func (ms *MockManifests) ManifestVersion(filename string) uint { manifest, ok := ms.Manifests[filename] From 4dfb1eb3d4e1c9bf323a7ce1ee179e4c32a460be Mon Sep 17 00:00:00 2001 From: 9547 Date: Fri, 4 Sep 2020 01:04:32 +0800 Subject: [PATCH 03/13] pkg/repository: implement MockMirror.Download --- pkg/repository/mirror.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index 75a63409aa..d03ee59da3 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -291,7 +291,23 @@ func (l *MockMirror) Open() error { // Download implements Mirror. func (l *MockMirror) Download(resource, targetDir string) error { - return errors.New("MockMirror::Download not implemented") + content, ok := l.Resources[resource] + if !ok { + return errors.Annotatef(ErrNotFound, "resource %s", resource) + } + + if err := os.MkdirAll(targetDir, 0755); err != nil { + return err + } + target := filepath.Join(targetDir, resource) + + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer file.Close() + file.Write([]byte(content)) + return nil } // Fetch implements Mirror. From 1117b2c58bb577d8669cadbba4b2ce18627dba75 Mon Sep 17 00:00:00 2001 From: 9547 Date: Fri, 4 Sep 2020 01:21:43 +0800 Subject: [PATCH 04/13] gitignore: ignore failpoint files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 58fe173bb1..662b23b6b6 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ bin/ /tests/tiup_mirrors/ /logs docker/secret/ +*__failpoint_binding__.go +*__failpoint_stash__ From 1d94ec46afa8bdbe5b2614b68296a60fbde060c4 Mon Sep 17 00:00:00 2001 From: 9547 Date: Mon, 14 Sep 2020 20:28:47 +0800 Subject: [PATCH 05/13] pkg/repository: fix comment by `make check` --- pkg/repository/mirror.go | 4 ++-- pkg/repository/v1_repository.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index d03ee59da3..52aecba142 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -306,8 +306,8 @@ func (l *MockMirror) Download(resource, targetDir string) error { return err } defer file.Close() - file.Write([]byte(content)) - return nil + _, err = file.Write([]byte(content)) + return err } // Fetch implements Mirror. diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 24c2571cec..64ab31e473 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -498,7 +498,7 @@ func (r *V1Repository) FetchComponent(item *v1manifest.VersionItem) (io.Reader, return checkHash(reader, item.Hashes[v1manifest.SHA256]) } -// DoDownloadComponent downloads the component specified by item into local file +// DownloadComponent downloads the component specified by item into local file func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target string) error { targetDir := filepath.Dir(target) err := r.mirror.Download(item.URL, targetDir) From 2cb62d995ef93f28de4b1932bacb94ef14a5e4cc Mon Sep 17 00:00:00 2001 From: 9547 Date: Thu, 17 Sep 2020 07:09:50 +0800 Subject: [PATCH 06/13] repository: rm component if hash checksum no equal --- pkg/repository/v1_repository.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 64ab31e473..29a8359676 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -498,7 +498,8 @@ func (r *V1Repository) FetchComponent(item *v1manifest.VersionItem) (io.Reader, return checkHash(reader, item.Hashes[v1manifest.SHA256]) } -// DownloadComponent downloads the component specified by item into local file +// DownloadComponent downloads the component specified by item into local file, +// the component will be removed if hash is not correct func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target string) error { targetDir := filepath.Dir(target) err := r.mirror.Download(item.URL, targetDir) @@ -510,9 +511,14 @@ func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target st if err != nil { return err } - defer reader.Close() _, err = checkHash(reader, item.Hashes[v1manifest.SHA256]) + reader.Close() + + // remove the target compoonent to avoid attacking + if err != nil { + _ = os.Remove(target) + } return err } From 46e91bb9ca5a6c33251d83b1cc41c423c34faffd Mon Sep 17 00:00:00 2001 From: 9547 Date: Thu, 17 Sep 2020 07:12:17 +0800 Subject: [PATCH 07/13] cluster: ensure mirror.Open after NewMirror --- pkg/cluster/clusterutil/cluster.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cluster/clusterutil/cluster.go b/pkg/cluster/clusterutil/cluster.go index 379a8130ad..6fa7bc2076 100644 --- a/pkg/cluster/clusterutil/cluster.go +++ b/pkg/cluster/clusterutil/cluster.go @@ -40,6 +40,9 @@ func NewRepository(os, arch string) (Repository, error) { mirror := repository.NewMirror(environment.Mirror(), repository.MirrorOptions{ Progress: repository.DisableProgress{}, }) + if err := mirror.Open(); err != nil { + return nil, err + } local, err := v1manifest.NewManifests(profile) if err != nil { return nil, err From a4ac339d1bb5fe7191ba25dc07aafd5aca9586c6 Mon Sep 17 00:00:00 2001 From: 9547 Date: Thu, 17 Sep 2020 22:31:08 +0800 Subject: [PATCH 08/13] repository: move file if name not match --- pkg/repository/v1_repository.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 29a8359676..76d5cf2a6a 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "runtime" "strconv" @@ -507,6 +508,14 @@ func (r *V1Repository) DownloadComponent(item *v1manifest.VersionItem, target st return errors.Trace(err) } + // the downloaded file is named by item.URL, which maybe differ to target name + if downloaded := path.Join(targetDir, item.URL); downloaded != target { + err := os.Rename(downloaded, target) + if err != nil { + return errors.Trace(err) + } + } + reader, err := os.Open(target) if err != nil { return err From c16fa2d939f86af144162a6975df09d240fec76f Mon Sep 17 00:00:00 2001 From: 9547 Date: Thu, 17 Sep 2020 23:36:15 +0800 Subject: [PATCH 09/13] docker/control: killall is from package psmisc --- docker/control/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/control/Dockerfile b/docker/control/Dockerfile index 943bbbad39..682fa1492f 100644 --- a/docker/control/Dockerfile +++ b/docker/control/Dockerfile @@ -12,6 +12,7 @@ FROM golang:1.14 apt-get install -qqy \ dos2unix \ default-mysql-client \ + psmisc \ vim # not required by tiup-cluster itself, just for ease of use From 4e2138c6b8b701b708b3cb0023bf4e656afb86a5 Mon Sep 17 00:00:00 2001 From: 9547 Date: Fri, 18 Sep 2020 22:32:46 +0800 Subject: [PATCH 10/13] Fix: add mirror.Open after NewMirror --- cmd/cmd_test.go | 1 + pkg/environment/env.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 7f008c4ab1..6a3f93fdcf 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -43,6 +43,7 @@ type testCmdSuite struct { func (s *testCmdSuite) SetUpSuite(c *C) { s.testDir = filepath.Join(currentDir(), "testdata") s.mirror = repository.NewMirror(s.testDir, repository.MirrorOptions{}) + _ = s.mirror.Open() } /* diff --git a/pkg/environment/env.go b/pkg/environment/env.go index 1f822534f6..29631330a2 100644 --- a/pkg/environment/env.go +++ b/pkg/environment/env.go @@ -55,7 +55,7 @@ func Mirror() string { m := os.Getenv(repository.EnvMirrors) if m != "" { if cfg.Mirror != m { - fmt.Printf(`WARNING: both mirror config(%s) + fmt.Printf(`WARNING: both mirror config(%s) and TIUP_MIRRORS(%s) have been set. Setting mirror to TIUP_MIRRORS(%s) `, cfg.Mirror, m, m) @@ -94,6 +94,9 @@ func InitEnv(options repository.Options) (*Environment, error) { // Replace the mirror if some sub-commands use different mirror address mirrorAddr := Mirror() mirror := repository.NewMirror(mirrorAddr, repository.MirrorOptions{}) + if err := mirror.Open(); err != nil { + return nil, err + } var repo *repository.Repository var v1repo *repository.V1Repository From 0b6cf1f255da43967983ee5505924db89f5e56a7 Mon Sep 17 00:00:00 2001 From: 9547 Date: Fri, 18 Sep 2020 22:54:58 +0800 Subject: [PATCH 11/13] Chore(make): indent --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 831a9081b8..08d2feb4f6 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ tools/bin/failpoint-ctl: go.mod $(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl pkger: - $(GO) run tools/pkger/main.go -s templates -d pkg/cluster/embed + $(GO) run tools/pkger/main.go -s templates -d pkg/cluster/embed fmt: @echo "gofmt (simplify)" @@ -121,4 +121,3 @@ tools/bin/revive: tools/check/go.mod tools/bin/golangci-lint: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.27.0 - From b23f865a7982f6da307f53127918fce17aaa084e Mon Sep 17 00:00:00 2001 From: 9547 Date: Mon, 21 Sep 2020 22:46:54 +0800 Subject: [PATCH 12/13] localdata: add EnvNameKeepSourceTarget to keep source target or not --- pkg/localdata/constant.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/localdata/constant.go b/pkg/localdata/constant.go index 3825841604..f42d4cec15 100644 --- a/pkg/localdata/constant.go +++ b/pkg/localdata/constant.go @@ -78,6 +78,9 @@ const ( // EnvNameSCPPath is the variable name by which user can specific the executable scp binary path EnvNameSCPPath = "TIUP_SCP_PATH" + // EnvNameKeepSourceTarget is the variable name by which user can keep the source target or not + EnvNameKeepSourceTarget = "TIUP_KEEP_SOURCE_TARGET" + // MetaFilename represents the process meta file name MetaFilename = "tiup_process_meta" ) From ac0bcff7732675e6fd564453c8a65c5e000533a9 Mon Sep 17 00:00:00 2001 From: 9547 Date: Mon, 21 Sep 2020 22:47:56 +0800 Subject: [PATCH 13/13] repository: rm source target if expand --- pkg/repository/v1_repository.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 76d5cf2a6a..d538b79de0 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -104,6 +104,10 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error { return errors.Trace(err) } + keepSource := false + if v := os.Getenv(localdata.EnvNameKeepSourceTarget); v == "enable" || v == "true" { + keepSource = true + } var errs []string for _, spec := range specs { manifest, err := r.updateComponentManifest(spec.ID, false) @@ -173,12 +177,17 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error { errs = append(errs, err.Error()) continue } - defer reader.Close() err = r.local.InstallComponent(reader, targetDir, spec.ID, spec.Version, versionItem.URL, r.DisableDecompress) + reader.Close() + if err != nil { errs = append(errs, err.Error()) - continue + } + + // remove the source gzip target if expand is on + if !r.DisableDecompress && !keepSource { + _ = os.Remove(target) } }