From 016de8c700d993157e379c96492612f4263af387 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 18 May 2022 15:27:12 +0200 Subject: [PATCH 1/2] Add new features to alb-replay * Allow oppening gzip log files directly * Allow filtering requests in the log file by path and by status code * Add quiet mode (which only logs failing requests) --- tools/alb-replay/main.go | 92 ++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 22 deletions(-) diff --git a/tools/alb-replay/main.go b/tools/alb-replay/main.go index 692c83429a..4936bba124 100644 --- a/tools/alb-replay/main.go +++ b/tools/alb-replay/main.go @@ -3,6 +3,7 @@ package main import ( + "compress/gzip" "context" "encoding/csv" "errors" @@ -14,7 +15,8 @@ import ( "net/url" "os" "os/signal" - "strconv" + "path/filepath" + "regexp" "strings" "sync" "syscall" @@ -32,23 +34,29 @@ type NumberedURL struct { URL string } -func isSuccesfulStatusCode(statusCode int) bool { - // consider all 2XX HTTP errors a success - return statusCode/100 == 2 +type ALBLogEntryReader struct { + csvReader *csv.Reader + config ALBLogEntryReaderConfig } -type ALBLogEntryReader csv.Reader +type ALBLogEntryReaderConfig struct { + pathRegexp *regexp.Regexp + statusCodeRegexp *regexp.Regexp +} -func newALBLogEntryReader(input io.Reader) *ALBLogEntryReader { +func newALBLogEntryReader(input io.Reader, config ALBLogEntryReaderConfig) *ALBLogEntryReader { reader := csv.NewReader(input) reader.Comma = ' ' reader.FieldsPerRecord = albLogEntryCount reader.ReuseRecord = true - return (*ALBLogEntryReader)(reader) + return &ALBLogEntryReader{ + csvReader: reader, + config: config, + } } func (r *ALBLogEntryReader) GetRequestURI() (string, error) { - records, err := (*csv.Reader)(r).Read() + records, err := r.csvReader.Read() if err != nil { return "", err } @@ -58,13 +66,9 @@ func (r *ALBLogEntryReader) GetRequestURI() (string, error) { if statusCodeStr == "-" { return "", nil } - statusCode, err := strconv.Atoi(statusCodeStr) - if err != nil { - return "", fmt.Errorf("error parsing target status code %q: %v", statusCodeStr, err) - } - // discard unsuccesful requests - if !isSuccesfulStatusCode(statusCode) { + if !r.config.statusCodeRegexp.MatchString(statusCodeStr) { + // discard url return "", nil } @@ -86,6 +90,13 @@ func (r *ALBLogEntryReader) GetRequestURI() (string, error) { return "", fmt.Errorf("error parsing url %q: %v", urlStr, err) } + if r.config.pathRegexp != nil { + if !r.config.pathRegexp.MatchString(urlStr) { + // discard url + return "", nil + } + } + return parsed.RequestURI(), nil } @@ -121,7 +132,7 @@ func parseURLs(ctx context.Context, startFrom int, baseURL string, logReader *AL } } -func queryURLs(ctx context.Context, timeout time.Duration, urlChan chan NumberedURL) { +func queryURLs(ctx context.Context, timeout time.Duration, urlChan chan NumberedURL, quiet bool) { client := http.Client{ Timeout: timeout, } @@ -146,18 +157,23 @@ func queryURLs(ctx context.Context, timeout time.Duration, urlChan chan Numbered continue } resp.Body.Close() - if !isSuccesfulStatusCode(resp.StatusCode) { + if resp.StatusCode/100 != 2 { log.Printf("(%d) unexpected status code: %d %q", numURL.Number, resp.StatusCode, numURL.URL) continue } - log.Printf("(%d) %s %s", numURL.Number, time.Since(start), numURL.URL) + if !quiet { + log.Printf("(%d) %s %s", numURL.Number, time.Since(start), numURL.URL) + } } } func main() { workers := flag.Int("workers", 1, "How many parallel workers to use") startFromURLNum := flag.Int("start-from", 1, "What URL number to start from") + pathRegexp := flag.String("path-filter", "", "Regular expression with which to filter in requests based on their paths") + statusCodeRegexp := flag.String("status-code-filter", "^2[0-9][0-9]$", "Regular expression with which to filter in request based on their status codes") timeout := flag.Duration("timeout", time.Second*5, "HTTP request timeout") + quiet := flag.Bool("quiet", false, "Only log failed requests") flag.Parse() if *workers < 1 { log.Fatal("--workers parameter must be > 0") @@ -165,16 +181,48 @@ func main() { if *startFromURLNum < 1 { log.Fatal("--start-from must be > 0") } + var pathRE *regexp.Regexp + if *pathRegexp != "" { + var err error + pathRE, err = regexp.Compile(*pathRegexp) + if err != nil { + log.Fatalf("error parsing --path-filter %q: %v", *pathRegexp, err) + } + } + var statusCodeRE *regexp.Regexp + if *statusCodeRegexp != "" { + var err error + statusCodeRE, err = regexp.Compile(*statusCodeRegexp) + if err != nil { + log.Fatalf("error parsing --status-code-filter parameter %q: %v", *statusCodeRegexp, err) + } + } if flag.NArg() != 2 { - log.Fatalf("usage: %s ", os.Args[0]) + log.Fatalf("usage: %s ", os.Args[0]) } - file, err := os.Open(flag.Args()[0]) + var reader io.ReadCloser + filePath := flag.Args()[0] + reader, err := os.Open(filePath) if err != nil { - log.Fatalf("error opening file %q: %v", os.Args[1], err) + log.Fatalf("error opening file %q: %v", filePath, err) } + defer reader.Close() + if filepath.Ext(filePath) == ".gz" { + var err error + reader, err = gzip.NewReader(reader) + if err != nil { + log.Fatalf("error opening file %q: %v", filePath, err) + } + defer reader.Close() + } + baseURL := flag.Args()[1] - logReader := newALBLogEntryReader(file) + logReaderConfig := ALBLogEntryReaderConfig{ + pathRegexp: pathRE, + statusCodeRegexp: statusCodeRE, + } + logReader := newALBLogEntryReader(reader, logReaderConfig) urlChan := make(chan NumberedURL, *workers) var wg sync.WaitGroup @@ -186,7 +234,7 @@ func main() { for i := 0; i < *workers; i++ { wg.Add(1) go func() { - queryURLs(ctx, *timeout, urlChan) + queryURLs(ctx, *timeout, urlChan, *quiet) wg.Done() }() } From 7156d468e705d4f8b1a690b0bec591520117e663 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 19 May 2022 13:30:32 +0200 Subject: [PATCH 2/2] Update README --- tools/alb-replay/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/alb-replay/README.md b/tools/alb-replay/README.md index e767ee7319..6ccca2acc1 100644 --- a/tools/alb-replay/README.md +++ b/tools/alb-replay/README.md @@ -14,9 +14,14 @@ go install ./tools/alb-replay ``` Usage of ./alb-replay: - alb-replay + -path-filter string + Regular expression with which to filter in requests based on their paths + -quiet + Only log failed requests -start-from int What URL number to start from (default 1) + -status-code-filter string + Regular expression with which to filter in request based on their status codes (default "^2[0-9][0-9]$") -timeout duration HTTP request timeout (default 5s) -workers int