Convert http-server output into a mock server

CHANGELOG.md

@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 - Added option to set up custom buffer size for the log reader. [#22](https://github.com/elastic/stream/pull/22)
 - Added support for glob patterns. [#22](https://github.com/elastic/stream/pull/22)
+- Convert `http-server` output into a mock HTTP server.[#23](https://github.com/elastic/stream/pull/23)
 
 ## [0.4.0]
 

README.md

@@ -9,4 +9,72 @@ stream is a test utility for streaming data via:
 - TLS
 - Webhook
 - GCP Pub-Sub
-- HTTP Server
+- HTTP Mock Server
+
+## HTTP Server mock reference
+
+`stream` can also serve logs setting up a complete HTTP mock server.
+
+Usage:
+
+```bash
+stream http-server --addr=":8080" --config="./config.yml"
+```
+
+The server can be configured to serve specific log files
+on certain routes. The config should be defined in a yaml file of the following format:
+
+```yaml
+---
+    rules:
+    - path: "/path1/test"
+      methods: ["GET"]
+
+      user: username
+      password: passwd
+      query_params:
+        p1: ["v1"]
+      request_headers:
+        accept: ["application/json"]
+
+      responses:
+      - headers:
+          x-foo: ["test"]
+        status_code: 200
+        body: |-
+          {"next": "http://{{ hostname }}/page/{{ sum (.req_num) 1 }}"}
+    - path: "/page/{pagenum:[0-9]}" params.
+      methods: ["POST"]
+
+      responses:
+      - status_code: 200
+        body: "{{ .request.vars.pagenum }}"
+        headers:
+          content-type: ["text/plain"]
+```
+
+The rules will be defined in order, and will only match if all criteria is true for a request. This means that you need to define the more restrictive rules on top.
+
+### Options
+
+- `rules`: a list of rules. More restrictive rules need to go on top.
+- `path`: the path to match. It can use [gorilla/mux](https://pkg.go.dev/github.com/gorilla/mux#pkg-overview) parameters patterns.
+- `methods`: a list of methods to match with the rule.
+- `user` and `password`: username and password for basic auth matching.
+- `query_params`: Key-Value definitions of the query parameters to match. It can use [gorilla/mux](https://pkg.go.dev/github.com/gorilla/mux#Route.Queries) parameters patterns for the values. Web form params will also be added and compared against this for simplicity.
+- `request_headers`: Key-Value definitions of the headers to match. Any headers outside of this list will be ignored. The matches can be defined [as regular expressions](https://pkg.go.dev/github.com/gorilla/mux#Route.HeadersRegexp).
+- `request_body`: a string defining the expected body to match for the request.
+- `responses`: a list of zero or more responses to return on matches. If more than one are set, they will be returned in rolling sequence.
+- `status_code`: the status code to return.
+- `headers`: Key-Value list of the headers to return with the response. The values will be evaluated as [Go templates](https://golang.org/pkg/text/template/).
+- `body`: a string defining the body that will be returned as a response. It will be evaluated as a [Go template](https://golang.org/pkg/text/template/).
+
+When using [Go templates](https://golang.org/pkg/text/template/) as part of the `response.headers` or `response.body`, some functions and data will be available:
+
+- `hostname`: function that returns the hostname.
+- `env KEY`: function that returns the KEY from environment.
+- `sum A B`: function that returns the sum of numbers A and B (only for integers).
+- `.req_num`: variable containing the current request number, auto incremented after every request for the rule.
+- `.request.vars`: map containing the variables received in the request (both query and form).
+- `.request.url`: the url object. Can be used as per [the Go URL documentation.](https://golang.org/pkg/net/url/#URL)
+- `.request.headers` the headers object. Can be used as per [the Go http.Header documentation.](https://golang.org/pkg/net/http/#Header)

command/httpserver.go

@@ -0,0 +1,51 @@
+// Licensed to Elasticsearch B.V. under one or more agreements.
+// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+package command
+
+import (
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/elastic/stream/pkg/httpserver"
+)
+
+type httpServerRunner struct {
+	logger *zap.SugaredLogger
+	cmd    *cobra.Command
+	opts   *httpserver.Options
+}
+
+func newHTTPServerRunner(options *httpserver.Options, logger *zap.Logger) *cobra.Command {
+	r := &httpServerRunner{
+		opts: options,
+		cmd: &cobra.Command{
+			Use:   "http-server [options]",
+			Short: "Set up a mock http server",
+		},
+	}
+
+	r.cmd.RunE = func(_ *cobra.Command, args []string) error {
+		r.logger = logger.Sugar().With("address", options.Addr)
+		return r.Run()
+	}
+
+	return r.cmd
+}
+
+func (r *httpServerRunner) Run() error {
+	r.logger.Debug("mock server running...")
+	server, err := httpserver.New(r.opts, r.logger)
+	if err != nil {
+		return err
+	}
+
+	if err := server.Start(r.cmd.Context()); err != nil {
+		return err
+	}
+
+	<-r.cmd.Context().Done()
+
+	return server.Close()
+}

command/root.go

@@ -21,12 +21,12 @@ import (
 	"github.com/elastic/go-concert/ctxtool/osctx"
 	"github.com/elastic/go-concert/timed"
 
+	"github.com/elastic/stream/pkg/httpserver"
 	"github.com/elastic/stream/pkg/log"
 	"github.com/elastic/stream/pkg/output"
 
 	// Register outputs.
 	_ "github.com/elastic/stream/pkg/output/gcppubsub"
-	_ "github.com/elastic/stream/pkg/output/httpserver"
 	_ "github.com/elastic/stream/pkg/output/tcp"
 	_ "github.com/elastic/stream/pkg/output/tls"
 	_ "github.com/elastic/stream/pkg/output/udp"
@@ -76,16 +76,19 @@ func ExecuteContext(ctx context.Context) error {
 	rootCmd.PersistentFlags().StringVar(&opts.GCPPubsubOptions.Subscription, "gcppubsub-subscription", "subscription", "GCP Pubsub subscription name")
 	rootCmd.PersistentFlags().BoolVar(&opts.GCPPubsubOptions.Clear, "gcppubsub-clear", true, "GCP Pubsub clear flag")
 
-	// HTTP output flags.
-	rootCmd.PersistentFlags().DurationVar(&opts.HTTPServerOptions.ReadTimeout, "http-server-read-timeout", 5*time.Second, "HTTP Server read timeout")
-	rootCmd.PersistentFlags().DurationVar(&opts.HTTPServerOptions.WriteTimeout, "http-server-write-timeout", 5*time.Second, "HTTP Server write timeout")
-	rootCmd.PersistentFlags().StringVar(&opts.HTTPServerOptions.TLSCertificate, "http-server-tls-cert", "", "Path to the TLS certificate")
-	rootCmd.PersistentFlags().StringVar(&opts.HTTPServerOptions.TLSKey, "http-server-tls-key", "", "Path to the TLS key file")
-	rootCmd.PersistentFlags().StringArrayVar(&opts.HTTPServerOptions.ResponseHeaders, "http-server-response-headers", []string{"content-type", "application/json"}, "List of headers key-values")
-
 	// Sub-commands.
 	rootCmd.AddCommand(newLogRunner(&opts, logger))
 	rootCmd.AddCommand(newPCAPRunner(&opts, logger))
+
+	httpOpts := httpserver.Options{Options: &opts}
+	httpCommand := newHTTPServerRunner(&httpOpts, logger)
+	httpCommand.PersistentFlags().DurationVar(&httpOpts.ReadTimeout, "read-timeout", 5*time.Second, "HTTP Server read timeout")
+	httpCommand.PersistentFlags().DurationVar(&httpOpts.WriteTimeout, "write-timeout", 5*time.Second, "HTTP Server write timeout")
+	httpCommand.PersistentFlags().StringVar(&httpOpts.TLSCertificate, "tls-cert", "", "Path to the TLS certificate")
+	httpCommand.PersistentFlags().StringVar(&httpOpts.TLSKey, "tls-key", "", "Path to the TLS key file")
+	httpCommand.PersistentFlags().StringVar(&httpOpts.ConfigPath, "config", "", "Path to the config file")
+	rootCmd.AddCommand(httpCommand)
+
 	rootCmd.AddCommand(versionCmd)
 
 	// Add common start-up delay logic.

go.mod

@@ -3,16 +3,27 @@ module github.com/elastic/stream
 go 1.15
 
 require (
-	cloud.google.com/go/pubsub v1.0.1
-	github.com/elastic/go-concert v0.0.4
+	cloud.google.com/go v0.83.0 // indirect
+	cloud.google.com/go/pubsub v1.11.0
+	github.com/Microsoft/go-winio v0.5.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/containerd/continuity v0.1.0 // indirect
+	github.com/elastic/go-concert v0.2.0
+	github.com/elastic/go-ucfg v0.8.3
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gopacket v1.1.19
-	github.com/ory/dockertest/v3 v3.6.3
-	github.com/spf13/cobra v1.1.1
+	github.com/gorilla/mux v1.8.0
+	github.com/opencontainers/runc v1.0.0-rc95 // indirect
+	github.com/ory/dockertest/v3 v3.6.5
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.4.0
-	go.uber.org/multierr v1.1.0
-	go.uber.org/zap v1.10.0
-	golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	google.golang.org/api v0.13.0
+	github.com/stretchr/testify v1.7.0
+	go.uber.org/multierr v1.7.0
+	go.uber.org/zap v1.17.0
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
+	golang.org/x/sys v0.0.0-20210603125802-9665404d3644
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	google.golang.org/api v0.47.0
+	google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 // indirect
 )