From fe178202fa5f30898ac9e57a11813903475ad922 Mon Sep 17 00:00:00 2001 From: DustinKLo Date: Fri, 16 Oct 2020 12:10:43 -0700 Subject: [PATCH 1/3] added region flag to avoid having to parse the endpoint to get the region hard-coded proxy.service value to "es" added insecure flag and logic to avoid SSL verification --- README.md | 34 +++++++++--- aws-es-proxy.go | 144 +++++++++++++++++++++++++----------------------- 2 files changed, 100 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 2cd4ed03..a29c334d 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ brew install aws-es-proxy ### Build from Source #### Dependencies: -* go1.14+ + +- go1.14+ ```sh #requires go1.14 @@ -82,8 +83,6 @@ export AWS_SECRET_ACCESS_KEY=MY-SECRET-KEY } ``` - - ## Usage example: You can use either argument `-endpoint` OR environment variable `ENDPOINT` to specify AWS ElasticSearch endpoint. @@ -100,14 +99,14 @@ export ENDPOINT=https://test-es-somerandomvalue.eu-west-1.es.amazonaws.com Listening on 10.0.0.1:9200 ``` -*aws-es-proxy* listens on 127.0.0.1:9200 if no additional argument is provided. You can change the IP and Port passing the argument `-listen` +_aws-es-proxy_ listens on 127.0.0.1:9200 if no additional argument is provided. You can change the IP and Port passing the argument `-listen` ```sh ./aws-es-proxy -listen :8080 -endpoint ... ./aws-es-proxy -listen 10.0.0.1:9200 -endpoint ... ``` -By default, *aws-es-proxy* will not display any message in the console. However, it has the ability to print requests being sent to Amazon Elasticsearch, and the duration it takes to receive the request back. This can be enabled using the option `-verbose` +By default, _aws-es-proxy_ will not display any message in the console. However, it has the ability to print requests being sent to Amazon Elasticsearch, and the duration it takes to receive the request back. This can be enabled using the option `-verbose` ```sh ./aws-es-proxy -verbose ... @@ -124,6 +123,10 @@ For a full list of available options, use `-h`: ```sh ./aws-es-proxy -h Usage of ./aws-es-proxy: + -auth + Require HTTP Basic Auth + -debug + Print debug messages -endpoint string Amazon ElasticSearch Endpoint (e.g: https://dummy-host.eu-west-1.es.amazonaws.com) -listen string @@ -132,15 +135,30 @@ Usage of ./aws-es-proxy: Log user requests and ElasticSearch responses to files -no-sign-reqs Disable AWS Signature v4 + -password string + HTTP Basic Auth Password -pretty Prettify verbose and file output + -realm string + Authentication Required + -remote-terminate + Allow HTTP remote termination + -timeout int + Set a request timeout to ES. Specify in seconds, defaults to 15 (default 15) + -username string + HTTP Basic Auth Username -verbose Print user requests + -version + Print aws-es-proxy version + -region + AWS region (ex. us-west-2) (Required) + -insecure + Will not verify SSL (default false) ``` - ## Using HTTP Clients -After you run *aws-es-proxy*, you can now open your Web browser on [http://localhost:9200](http://localhost:9200). Everything should be working as you have your own instance of ElasticSearch running on port 9200. +After you run _aws-es-proxy_, you can now open your Web browser on [http://localhost:9200](http://localhost:9200). Everything should be working as you have your own instance of ElasticSearch running on port 9200. -To access Kibana, use [http://localhost:9200/_plugin/kibana/app/kibana](http://localhost:9200/_plugin/kibana/app/kibana) +To access Kibana, use [http://localhost:9200/\_plugin/kibana/app/kibana](http://localhost:9200/_plugin/kibana/app/kibana) diff --git a/aws-es-proxy.go b/aws-es-proxy.go index 54bdd575..435f2c64 100644 --- a/aws-es-proxy.go +++ b/aws-es-proxy.go @@ -3,6 +3,7 @@ package main import ( "bytes" "crypto/subtle" + "crypto/tls" "encoding/json" "flag" "fmt" @@ -22,7 +23,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/sirupsen/logrus" @@ -69,23 +69,25 @@ type responseStruct struct { } type proxy struct { - scheme string - host string - region string - service string - endpoint string - verbose bool - prettify bool - logtofile bool - nosignreq bool - fileRequest *os.File - fileResponse *os.File - credentials *credentials.Credentials - httpClient *http.Client - auth bool - username string - password string - realm string + scheme string + host string + region string + service string + endpoint string + verbose bool + prettify bool + logtofile bool + nosignreq bool + fileRequest *os.File + fileResponse *os.File + credentials *credentials.Credentials + httpClient *http.Client + auth bool + username string + password string + realm string + remoteTerminate bool + insecure bool } func newProxy(args ...interface{}) *proxy { @@ -99,25 +101,33 @@ func newProxy(args ...interface{}) *proxy { CheckRedirect: noRedirect, } + if args[12].(bool) == true { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + return &proxy{ - endpoint: args[0].(string), - verbose: args[1].(bool), - prettify: args[2].(bool), - logtofile: args[3].(bool), - nosignreq: args[4].(bool), - httpClient: &client, - auth: args[6].(bool), - username: args[7].(string), - password: args[8].(string), - realm: args[9].(string), + endpoint: args[0].(string), + verbose: args[1].(bool), + prettify: args[2].(bool), + logtofile: args[3].(bool), + nosignreq: args[4].(bool), + httpClient: &client, + auth: args[6].(bool), + username: args[7].(string), + password: args[8].(string), + realm: args[9].(string), + remoteTerminate: args[10].(bool), + region: args[11].(string), + insecure: args[12].(bool), } } func (p *proxy) parseEndpoint() error { var ( - link *url.URL - err error - isAWSEndpoint bool + link *url.URL + err error ) if link, err = url.Parse(p.endpoint); err != nil { @@ -146,6 +156,9 @@ func (p *proxy) parseEndpoint() error { p.scheme = link.Scheme p.host = link.Host + p.service = "es" + logrus.Debugln("AWS Region", p.region) + // AWS SignV4 enabled, extract required parts for signing process if !p.nosignreq { @@ -154,29 +167,6 @@ func (p *proxy) parseEndpoint() error { if len(split) < 2 { logrus.Debugln("Endpoint split is less than 2") } - - awsEndpoints := []string{} - for _, partition := range endpoints.DefaultPartitions() { - for region := range partition.Regions() { - awsEndpoints = append(awsEndpoints, fmt.Sprintf("%s.es.%s", region, partition.DNSSuffix())) - } - } - - isAWSEndpoint = false - for _, v := range awsEndpoints { - if split[1] == v { - logrus.Debugln("Provided endpoint is a valid AWS Elasticsearch endpoint") - isAWSEndpoint = true - break - } - } - - if isAWSEndpoint { - // Extract region and service from link. This should be save now - parts := strings.Split(link.Host, ".") - p.region, p.service = parts[1], "es" - logrus.Debugln("AWS Region", p.region) - } } return nil @@ -210,6 +200,10 @@ func (p *proxy) getSigner() *v4.Signer { } func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if p.remoteTerminate && r.URL.Path == "/terminate-proxy" && r.Method == http.MethodPost { + logrus.Infoln("Terminate Signal") + os.Exit(0) + } if p.auth { user, pass, ok := r.BasicAuth() @@ -330,6 +324,7 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { fmt.Println() fmt.Println("========================") + fmt.Println("Region: ", p.region) fmt.Println(t.Format("2006/01/02 15:04:05")) fmt.Println("Remote Address: ", r.RemoteAddr) fmt.Println("Request URI: ", proxied.RequestURI()) @@ -420,22 +415,25 @@ func copyHeaders(dst, src http.Header) { func main() { var ( - debug bool - auth bool - username string - password string - realm string - verbose bool - prettify bool - logtofile bool - nosignreq bool - ver bool - endpoint string - listenAddress string - fileRequest *os.File - fileResponse *os.File - err error - timeout int + debug bool + auth bool + username string + password string + realm string + verbose bool + prettify bool + logtofile bool + nosignreq bool + ver bool + endpoint string + listenAddress string + fileRequest *os.File + fileResponse *os.File + err error + timeout int + remoteTerminate bool + region string + insecure bool ) flag.StringVar(&endpoint, "endpoint", "", "Amazon ElasticSearch Endpoint (e.g: https://dummy-host.eu-west-1.es.amazonaws.com)") @@ -451,6 +449,9 @@ func main() { flag.StringVar(&username, "username", "", "HTTP Basic Auth Username") flag.StringVar(&password, "password", "", "HTTP Basic Auth Password") flag.StringVar(&realm, "realm", "", "Authentication Required") + flag.BoolVar(&remoteTerminate, "remote-terminate", false, "Allow HTTP remote termination") + flag.StringVar(®ion, "region", "", "AWS Region (ex. us-west-2)") + flag.BoolVar(&insecure, "insecure", false, "Verify SSL") flag.Parse() if endpoint == "" { @@ -496,6 +497,9 @@ func main() { username, password, realm, + remoteTerminate, + region, + insecure, ) if err = p.parseEndpoint(); err != nil { From 88a70862dd53143ba415900d505830bbcf243d9e Mon Sep 17 00:00:00 2001 From: DustinKLo Date: Sat, 17 Oct 2020 13:47:05 -0700 Subject: [PATCH 2/3] cleaning up code, moving insecure logic to main function (TODO: still need to test) --- .gitignore | 3 ++- aws-es-proxy.go | 27 +++++++-------------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 5a847aff..11df5a61 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ vendor glide.lock dist -.idea \ No newline at end of file +.idea +aws-es-proxy diff --git a/aws-es-proxy.go b/aws-es-proxy.go index 435f2c64..438348e9 100644 --- a/aws-es-proxy.go +++ b/aws-es-proxy.go @@ -30,7 +30,6 @@ import ( ) func logger(debug bool) { - formatFilePath := func(path string) string { arr := strings.Split(path, "/") return arr[len(arr)-1] @@ -87,7 +86,6 @@ type proxy struct { password string realm string remoteTerminate bool - insecure bool } func newProxy(args ...interface{}) *proxy { @@ -101,12 +99,6 @@ func newProxy(args ...interface{}) *proxy { CheckRedirect: noRedirect, } - if args[12].(bool) == true { - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - return &proxy{ endpoint: args[0].(string), verbose: args[1].(bool), @@ -120,7 +112,7 @@ func newProxy(args ...interface{}) *proxy { realm: args[9].(string), remoteTerminate: args[10].(bool), region: args[11].(string), - insecure: args[12].(bool), + service: "es", } } @@ -156,14 +148,9 @@ func (p *proxy) parseEndpoint() error { p.scheme = link.Scheme p.host = link.Host - p.service = "es" - logrus.Debugln("AWS Region", p.region) - // AWS SignV4 enabled, extract required parts for signing process if !p.nosignreq { - split := strings.SplitAfterN(link.Hostname(), ".", 2) - if len(split) < 2 { logrus.Debugln("Endpoint split is less than 2") } @@ -175,7 +162,6 @@ func (p *proxy) parseEndpoint() error { func (p *proxy) getSigner() *v4.Signer { // Refresh credentials after expiration. Required for STS if p.credentials == nil { - sess, err := session.NewSession( &aws.Config{ Region: aws.String(p.region), @@ -342,7 +328,6 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if p.logtofile { - requestID := primitive.NewObjectID().Hex() reqStruct := &requestStruct{ @@ -408,7 +393,6 @@ func copyHeaders(dst, src http.Header) { dst.Add(k, v) } } - } } @@ -499,16 +483,20 @@ func main() { realm, remoteTerminate, region, - insecure, ) + if insecure == true { + p.httpClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + if err = p.parseEndpoint(); err != nil { logrus.Fatalln(err) os.Exit(1) } if p.logtofile { - requestFname := fmt.Sprintf("request-%s.log", primitive.NewObjectID().Hex()) if fileRequest, err = os.Create(requestFname); err != nil { log.Fatalln(err.Error()) @@ -523,7 +511,6 @@ func main() { p.fileRequest = fileRequest p.fileResponse = fileResponse - } logrus.Infof("Listening on %s...\n", listenAddress) From 61ff35d64920cad55abdd0b145d588340fddb8c7 Mon Sep 17 00:00:00 2001 From: DustinKLo Date: Sat, 17 Oct 2020 20:15:53 -0700 Subject: [PATCH 3/3] removed redundant use of insecure flag --- aws-es-proxy.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/aws-es-proxy.go b/aws-es-proxy.go index 8f91820d..32a2f3ad 100644 --- a/aws-es-proxy.go +++ b/aws-es-proxy.go @@ -99,12 +99,6 @@ func newProxy(args ...interface{}) *proxy { CheckRedirect: noRedirect, } - if args[12].(bool) == true { - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - return &proxy{ endpoint: args[0].(string), verbose: args[1].(bool),