From d980079db5d4b121d258e1d5e5e823e49c62cbd4 Mon Sep 17 00:00:00 2001 From: amecea Date: Wed, 20 Mar 2019 15:59:50 +0200 Subject: [PATCH] Use http trailer to check if the backup was done successfully --- pkg/sidecar/appclone.go | 8 ++++-- pkg/sidecar/apptakebackup.go | 31 ++++++++++++++++++--- pkg/sidecar/server.go | 53 ++++++++++++++++++++++++++++++++++-- pkg/sidecar/util.go | 29 +------------------- 4 files changed, 85 insertions(+), 36 deletions(-) diff --git a/pkg/sidecar/appclone.go b/pkg/sidecar/appclone.go index 3921f325c..9aa2cd9c8 100644 --- a/pkg/sidecar/appclone.go +++ b/pkg/sidecar/appclone.go @@ -153,7 +153,7 @@ func cloneFromBucket(initBucket string) error { func cloneFromSource(cfg *Config, host string) error { log.Info("cloning from node", "host", host) - backupBody, err := requestABackup(cfg, host, serverBackupEndpoint) + response, err := requestABackup(cfg, host, serverBackupEndpoint) if err != nil { return fmt.Errorf("fail to get backup: %s", err) } @@ -164,7 +164,7 @@ func cloneFromSource(cfg *Config, host string) error { // nolint: gosec xbstream := exec.Command("xbstream", "-x", "-C", dataDir) - xbstream.Stdin = backupBody + xbstream.Stdin = response.Body xbstream.Stderr = os.Stderr if err := xbstream.Start(); err != nil { @@ -175,6 +175,10 @@ func cloneFromSource(cfg *Config, host string) error { return fmt.Errorf("xbstream wait error: %s", err) } + if err := checkBackupTrailers(response); err != nil { + return err + } + return nil } diff --git a/pkg/sidecar/apptakebackup.go b/pkg/sidecar/apptakebackup.go index 79dd72b9c..776ebf209 100644 --- a/pkg/sidecar/apptakebackup.go +++ b/pkg/sidecar/apptakebackup.go @@ -31,8 +31,9 @@ func RunTakeBackupCommand(cfg *Config, srcHost, destBucket string) error { } func pushBackupFromTo(cfg *Config, srcHost, destBucket string) error { + tmpDestBucket := fmt.Sprintf("%s.tmp", destBucket) - backupBody, err := requestABackup(cfg, srcHost, serverBackupEndpoint) + response, err := requestABackup(cfg, srcHost, serverBackupEndpoint) if err != nil { return fmt.Errorf("getting backup: %s", err) } @@ -42,9 +43,9 @@ func pushBackupFromTo(cfg *Config, srcHost, destBucket string) error { // nolint: gosec rclone := exec.Command("rclone", - fmt.Sprintf("--config=%s", rcloneConfigFile), "rcat", destBucket) + fmt.Sprintf("--config=%s", rcloneConfigFile), "rcat", tmpDestBucket) - gzip.Stdin = backupBody + gzip.Stdin = response.Body gzip.Stderr = os.Stderr rclone.Stderr = os.Stderr @@ -66,11 +67,33 @@ func pushBackupFromTo(cfg *Config, srcHost, destBucket string) error { // wait for both commands to finish successful for i := 1; i <= 2; i++ { - if err := <-errChan; err != nil { + if err = <-errChan; err != nil { return err } } + if err = checkBackupTrailers(response); err != nil { + // backup failed so delete it from remote + log.Info("backup was partially taken", "trailers", response.Trailer) + return err + } + + log.Info("backup was taken successfully now move, now move it to permanent URL") + + // the backup was a success + // remove .tmp extension + // nolint: gosec + rcMove := exec.Command("rclone", + fmt.Sprintf("--config=%s", rcloneConfigFile), "moveto", tmpDestBucket, destBucket) + + if err = rcMove.Start(); err != nil { + return fmt.Errorf("final move failed: %s", err) + } + + if err = rcMove.Wait(); err != nil { + return fmt.Errorf("final move failed: %s", err) + } + return nil } diff --git a/pkg/sidecar/server.go b/pkg/sidecar/server.go index 4e7bad43d..ce4281e2f 100644 --- a/pkg/sidecar/server.go +++ b/pkg/sidecar/server.go @@ -79,6 +79,7 @@ func (s *server) backupHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Connection", "keep-alive") + w.Header().Set("Trailer", "Success") // nolint: gosec xtrabackup := exec.Command("xtrabackup", "--backup", "--slave-info", "--stream=xbstream", @@ -112,13 +113,15 @@ func (s *server) backupHandler(w http.ResponseWriter, r *http.Request) { return } - flusher.Flush() - if err := xtrabackup.Wait(); err != nil { log.Error(err, "failed waiting for xtrabackup to finish") http.Error(w, "xtrabackup failed", http.StatusInternalServerError) return } + + // success + w.Header().Set("Success", "true") + flusher.Flush() } func (s *server) isAuthenticated(r *http.Request) bool { @@ -137,3 +140,49 @@ func maxClients(h http.Handler, n int) http.Handler { h.ServeHTTP(w, r) }) } + +// requestABackup connects to specified host and endpoint and gets the backup +func requestABackup(cfg *Config, host, endpoint string) (*http.Response, error) { + log.Info("initialize a backup", "host", host, "endpoint", endpoint) + + req, err := http.NewRequest("GET", fmt.Sprintf( + "http://%s:%d%s", host, serverPort, endpoint), nil) + + if err != nil { + return nil, fmt.Errorf("fail to create request: %s", err) + } + + // set authentification user and password + req.SetBasicAuth(cfg.BackupUser, cfg.BackupPassword) + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil || resp.StatusCode != 200 { + status := "unknown" + if resp != nil { + status = resp.Status + } + return nil, fmt.Errorf("fail to get backup: %s, code: %s", err, status) + } + + return resp, nil +} + +func checkBackupTrailers(resp *http.Response) error { + if values, ok := resp.Trailer["Success"]; !ok || !stringInSlice("true", values) { + // backup is failed, remove from remote + return fmt.Errorf("backup failed to be taken: no 'Success' trailer found") + } + + return nil +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/pkg/sidecar/util.go b/pkg/sidecar/util.go index 49393d39d..9ac33bb12 100644 --- a/pkg/sidecar/util.go +++ b/pkg/sidecar/util.go @@ -20,7 +20,6 @@ import ( "database/sql" "fmt" "io" - "net/http" "os" // add mysql driver @@ -47,7 +46,7 @@ func runQuery(cfg *Config, q string, args ...interface{}) error { } }() - log.V(1).Info("running query", "query", q, "args", args) + log.V(1).Info("running query", "query", q) if _, err := db.Exec(q, args...); err != nil { return err } @@ -86,32 +85,6 @@ func copyFile(src, dst string) error { return nil } -// requestABackup connects to specified host and endpoint and gets the backup -func requestABackup(cfg *Config, host, endpoint string) (io.Reader, error) { - log.Info("initialize a backup", "host", host, "endpoint", endpoint) - - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d%s", host, serverPort, endpoint), nil) - if err != nil { - return nil, fmt.Errorf("fail to create request: %s", err) - } - - // set authentification user and password - req.SetBasicAuth(cfg.BackupUser, cfg.BackupPassword) - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil || resp.StatusCode != 200 { - status := "unknown" - if resp != nil { - status = resp.Status - } - return nil, fmt.Errorf("fail to get backup: %s, code: %s", err, status) - } - - return resp.Body, nil -} - // shouldBootstrapNode checks if the mysql data is at the first initialization func shouldBootstrapNode() bool { _, err := os.Open(fmt.Sprintf("%s/%s/%s.CSV", dataDir,