diff --git a/Gopkg.lock b/Gopkg.lock index 3848c057ea..1702fe641e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -17,21 +17,6 @@ revision = "74b12019e2aa53ec27882158f59192d7cd6d1998" version = "v0.33.1" -[[projects]] - digest = "1:94eff9ba92a0bf4938a2ec911c44b58b5c366a94f5f72b2b3c454629781c506f" - name = "fortio.org/fortio" - packages = [ - "fhttp", - "fnet", - "log", - "periodic", - "stats", - "version", - ] - pruneopts = "NUT" - revision = "bf3f2d9ff07ed03ef16be56af20d58dc0300e60f" - version = "v1.3.0" - [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" @@ -605,6 +590,20 @@ revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" version = "v2.2.1" +[[projects]] + digest = "1:ebecaaec2e2f0ae0bdac3ce5697ae8a8e5bb2107ae00fd6acbd8c70b9d8698c2" + name = "istio.io/fortio" + packages = [ + "fhttp", + "fnet", + "log", + "periodic", + "stats", + ] + pruneopts = "NUT" + revision = "abaacfd56859479cd712c45e70c8f1362be15e5b" + version = "v0.6.8" + [[projects]] digest = "1:6dd53537a34d9b612e66acd39859953cf7ad70e5b5953937bed5ea998355e7b8" name = "k8s.io/api" @@ -832,10 +831,6 @@ analyzer-version = 1 input-imports = [ "cloud.google.com/go/storage", - "fortio.org/fortio/fhttp", - "fortio.org/fortio/periodic", - "fortio.org/fortio/stats", - "github.com/golang/glog", "github.com/google/go-github/github", "github.com/google/licenseclassifier", "github.com/knative/pkg/configmap", @@ -849,10 +844,13 @@ "github.com/prometheus/client_golang/api/prometheus/v1", "github.com/prometheus/common/model", "go.uber.org/zap", + "golang.org/x/net/context", "golang.org/x/oauth2", "google.golang.org/api/iterator", "google.golang.org/api/option", "gopkg.in/yaml.v2", + "istio.io/fortio/fhttp", + "istio.io/fortio/periodic", "k8s.io/api/admission/v1beta1", "k8s.io/api/admissionregistration/v1beta1", "k8s.io/api/extensions/v1beta1", diff --git a/shared/loadgenerator/loadgenerator.go b/shared/loadgenerator/loadgenerator.go index 3c39e13e0c..5e7e29c505 100644 --- a/shared/loadgenerator/loadgenerator.go +++ b/shared/loadgenerator/loadgenerator.go @@ -26,10 +26,10 @@ import ( "path" "time" - "fortio.org/fortio/fhttp" - "fortio.org/fortio/periodic" "github.com/knative/test-infra/shared/common" "github.com/knative/test-infra/shared/prow" + "istio.io/fortio/fhttp" + "istio.io/fortio/periodic" ) const ( diff --git a/vendor/fortio.org/fortio/debian/copyright b/vendor/fortio.org/fortio/debian/copyright deleted file mode 100644 index ee65daad6f..0000000000 --- a/vendor/fortio.org/fortio/debian/copyright +++ /dev/null @@ -1,22 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: fortio -Source: https://github.com/fortio/fortio/ - -Files: * -Copyright: 2017 Istio Authors. -License: Apache-2.0 - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - . - On Debian systems, the full text of the Apache License, Version 2.0 - can be found in the file - `/usr/share/common-licenses/Apache-2.0'. diff --git a/vendor/fortio.org/fortio/fhttp/http_client.go b/vendor/fortio.org/fortio/fhttp/http_client.go deleted file mode 100644 index 943eb92d7b..0000000000 --- a/vendor/fortio.org/fortio/fhttp/http_client.go +++ /dev/null @@ -1,814 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fhttp // import "fortio.org/fortio/fhttp" - -import ( - "bufio" - "bytes" - "crypto/tls" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "net/http/httputil" - "net/url" - "strconv" - "strings" - "time" - - "fortio.org/fortio/fnet" - "fortio.org/fortio/log" - "fortio.org/fortio/version" -) - -// Fetcher is the Url content fetcher that the different client implements. -type Fetcher interface { - // Fetch returns http code, data, offset of body (for client which returns - // headers) - Fetch() (int, []byte, int) - // Close() cleans up connections and state - must be paired with NewClient calls. - // returns how many sockets have been used (Fastclient only) - Close() int -} - -var ( - // BufferSizeKb size of the buffer (max data) for optimized client in kilobytes defaults to 128k. - BufferSizeKb = 128 - // CheckConnectionClosedHeader indicates whether to check for server side connection closed headers. - CheckConnectionClosedHeader = false - // 'constants', case doesn't matter for those 3 - contentLengthHeader = []byte("\r\ncontent-length:") - connectionCloseHeader = []byte("\r\nconnection: close") - chunkedHeader = []byte("\r\nTransfer-Encoding: chunked") -) - -// NewHTTPOptions creates and initialize a HTTPOptions object. -// It replaces plain % to %25 in the url. If you already have properly -// escaped URLs use o.URL = to set it. -func NewHTTPOptions(url string) *HTTPOptions { - h := HTTPOptions{} - return h.Init(url) -} - -// Init initializes the headers in an HTTPOptions (User-Agent). -func (h *HTTPOptions) Init(url string) *HTTPOptions { - if h.initDone { - return h - } - h.initDone = true - h.URL = url - h.NumConnections = 1 - if h.HTTPReqTimeOut == 0 { - log.Debugf("Request timeout not set, using default %v", HTTPReqTimeOutDefaultValue) - h.HTTPReqTimeOut = HTTPReqTimeOutDefaultValue - } - if h.HTTPReqTimeOut < 0 { - log.Warnf("Invalid timeout %v, setting to %v", h.HTTPReqTimeOut, HTTPReqTimeOutDefaultValue) - h.HTTPReqTimeOut = HTTPReqTimeOutDefaultValue - } - h.URLSchemeCheck() - return h -} - -const ( - contentType = "Content-Type" - contentLength = "Content-Length" -) - -// GenerateHeaders completes the header generation, including Content-Type/Length -// and user credential coming from the http options in addition to extra headers -// coming from flags and AddAndValidateExtraHeader(). -// Warning this gets called more than once, do not generate duplicate headers. -func (h *HTTPOptions) GenerateHeaders() http.Header { - if h.extraHeaders == nil { // not already initialized from flags. - h.InitHeaders() - } - allHeaders := h.extraHeaders - payloadLen := len(h.Payload) - // If content-type isn't already specified and we have a payload, let's use the - // standard for binary content: - if payloadLen > 0 && len(h.ContentType) == 0 && len(allHeaders.Get(contentType)) == 0 { - h.ContentType = "application/octet-stream" - } - if len(h.ContentType) > 0 { - allHeaders.Set(contentType, h.ContentType) - } - // Add content-length unless already set in custom headers (or we're not doing a POST) - if (payloadLen > 0 || len(h.ContentType) > 0) && len(allHeaders.Get(contentLength)) == 0 { - allHeaders.Set(contentLength, strconv.Itoa(payloadLen)) - } - err := h.ValidateAndAddBasicAuthentication(allHeaders) - if err != nil { - log.Errf("User credential is not valid: %v", err) - } - return allHeaders -} - -// URLSchemeCheck makes sure the client will work with the scheme requested. -// it also adds missing http:// to emulate curl's behavior. -func (h *HTTPOptions) URLSchemeCheck() { - log.LogVf("URLSchemeCheck %+v", h) - if len(h.URL) == 0 { - log.Errf("Unexpected init with empty url") - return - } - hs := "https://" // longer of the 2 prefixes - lcURL := h.URL - if len(lcURL) > len(hs) { - lcURL = strings.ToLower(h.URL[:len(hs)]) // no need to tolower more than we check - } - if strings.HasPrefix(lcURL, hs) { - h.https = true - if !h.DisableFastClient { - log.Warnf("https requested, switching to standard go client") - h.DisableFastClient = true - } - return // url is good - } - if !strings.HasPrefix(lcURL, "http://") { - log.Warnf("Assuming http:// on missing scheme for '%s'", h.URL) - h.URL = "http://" + h.URL - } -} - -var userAgent = "fortio.org/fortio-" + version.Short() - -const ( - retcodeOffset = len("HTTP/1.X ") - // HTTPReqTimeOutDefaultValue is the default timeout value. 15s. - HTTPReqTimeOutDefaultValue = 15 * time.Second -) - -// HTTPOptions holds the common options of both http clients and the headers. -type HTTPOptions struct { - URL string - NumConnections int // num connections (for std client) - Compression bool // defaults to no compression, only used by std client - DisableFastClient bool // defaults to fast client - HTTP10 bool // defaults to http1.1 - DisableKeepAlive bool // so default is keep alive - AllowHalfClose bool // if not keepalive, whether to half close after request - Insecure bool // do not verify certs for https - FollowRedirects bool // For the Std Client only: follow redirects. - initDone bool - https bool // whether URLSchemeCheck determined this was an https:// call or not - // ExtraHeaders to be added to each request (UserAgent and headers set through AddAndValidateExtraHeader()). - extraHeaders http.Header - // Host is treated specially, remember that virtual header separately. - hostOverride string - HTTPReqTimeOut time.Duration // timeout value for http request - - UserCredentials string // user credentials for authorization - ContentType string // indicates request body type, implies POST instead of GET - Payload []byte // body for http request, implies POST if not empty. - - UnixDomainSocket string // Path of unix domain socket to use instead of host:port from URL -} - -// ResetHeaders resets all the headers, including the User-Agent: one (and the Host: logical special header). -// This is used from the UI as the user agent is settable from the form UI. -func (h *HTTPOptions) ResetHeaders() { - h.extraHeaders = make(http.Header) - h.hostOverride = "" -} - -// InitHeaders initialize and/or resets the default headers (ie just User-Agent). -func (h *HTTPOptions) InitHeaders() { - h.ResetHeaders() - h.extraHeaders.Add("User-Agent", userAgent) - // No other headers should be added here based on options content as this is called only once - // before command line option -H are parsed/set. -} - -// PayloadString returns the payload as a string. If payload is null return empty string -// This is only needed due to grpc ping proto. It takes string instead of byte array. -func (h *HTTPOptions) PayloadString() string { - if len(h.Payload) == 0 { - return "" - } - return string(h.Payload) -} - -// ValidateAndAddBasicAuthentication validates user credentials and adds basic authentication to http header, -// if user credentials are valid. -func (h *HTTPOptions) ValidateAndAddBasicAuthentication(headers http.Header) error { - if len(h.UserCredentials) <= 0 { - return nil // user credential is not entered - } - s := strings.SplitN(h.UserCredentials, ":", 2) - if len(s) != 2 { - return fmt.Errorf("invalid user credentials \"%s\", expecting \"user:password\"", h.UserCredentials) - } - headers.Set("Authorization", generateBase64UserCredentials(h.UserCredentials)) - return nil -} - -// AllHeaders returns the current set of headers including virtual/special Host header. -func (h *HTTPOptions) AllHeaders() http.Header { - headers := h.GenerateHeaders() - if h.hostOverride != "" { - headers.Add("Host", h.hostOverride) - } - return headers -} - -// Method returns the method of the http req. -func (h *HTTPOptions) Method() string { - if len(h.Payload) > 0 || h.ContentType != "" { - return fnet.POST - } - return fnet.GET -} - -// AddAndValidateExtraHeader collects extra headers (see commonflags.go for example). -func (h *HTTPOptions) AddAndValidateExtraHeader(hdr string) error { - // This function can be called from the flag settings, before we have a URL - // so we can't just call h.Init(h.URL) - if h.extraHeaders == nil { - h.InitHeaders() - } - s := strings.SplitN(hdr, ":", 2) - if len(s) != 2 { - return fmt.Errorf("invalid extra header '%s', expecting Key: Value", hdr) - } - key := strings.TrimSpace(s[0]) - value := strings.TrimSpace(s[1]) - if strings.EqualFold(key, "host") { - log.LogVf("Will be setting special Host header to %s", value) - h.hostOverride = value - } else { - log.LogVf("Setting regular extra header %s: %s", key, value) - h.extraHeaders.Add(key, value) - log.Debugf("headers now %+v", h.extraHeaders) - } - return nil -} - -// newHttpRequest makes a new http GET request for url with User-Agent. -func newHTTPRequest(o *HTTPOptions) *http.Request { - method := o.Method() - var body io.Reader - if method == fnet.POST { - body = bytes.NewReader(o.Payload) - } - req, err := http.NewRequest(method, o.URL, body) - if err != nil { - log.Errf("Unable to make %s request for %s : %v", method, o.URL, err) - return nil - } - req.Header = o.GenerateHeaders() - if o.hostOverride != "" { - req.Host = o.hostOverride - } - if !log.LogDebug() { - return req - } - bytes, err := httputil.DumpRequestOut(req, false) - if err != nil { - log.Errf("Unable to dump request: %v", err) - } else { - log.Debugf("For URL %s, sending:\n%s", o.URL, bytes) - } - return req -} - -// Client object for making repeated requests of the same URL using the same -// http client (net/http) -type Client struct { - url string - req *http.Request - client *http.Client - transport *http.Transport -} - -// Close cleans up any resources used by NewStdClient -func (c *Client) Close() int { - log.Debugf("Close() on %+v", c) - if c.req != nil { - if c.req.Body != nil { - if err := c.req.Body.Close(); err != nil { - log.Warnf("Error closing std client body: %v", err) - } - } - c.req = nil - } - if c.transport != nil { - c.transport.CloseIdleConnections() - } - return 0 // TODO: find a way to track std client socket usage. -} - -// ChangeURL only for standard client, allows fetching a different URL -func (c *Client) ChangeURL(urlStr string) (err error) { - c.url = urlStr - c.req.URL, err = url.Parse(urlStr) - return err -} - -// Fetch fetches the byte and code for pre created client -func (c *Client) Fetch() (int, []byte, int) { - // req can't be null (client itself would be null in that case) - resp, err := c.client.Do(c.req) - if err != nil { - log.Errf("Unable to send %s request for %s : %v", c.req.Method, c.url, err) - return http.StatusBadRequest, []byte(err.Error()), 0 - } - var data []byte - if log.LogDebug() { - if data, err = httputil.DumpResponse(resp, false); err != nil { - log.Errf("Unable to dump response %v", err) - } else { - log.Debugf("For URL %s, received:\n%s", c.url, data) - } - } - data, err = ioutil.ReadAll(resp.Body) - resp.Body.Close() //nolint(errcheck) - if err != nil { - log.Errf("Unable to read response for %s : %v", c.url, err) - code := resp.StatusCode - if code == http.StatusOK { - code = http.StatusNoContent - log.Warnf("Ok code despite read error, switching code to %d", code) - } - return code, data, 0 - } - code := resp.StatusCode - log.Debugf("Got %d : %s for %s %s - response is %d bytes", code, resp.Status, c.req.Method, c.url, len(data)) - return code, data, 0 -} - -// NewClient creates either a standard or fast client (depending on -// the DisableFastClient flag) -func NewClient(o *HTTPOptions) Fetcher { - o.Init(o.URL) // For completely new options - // For changes to options after init - o.URLSchemeCheck() - if o.DisableFastClient { - return NewStdClient(o) - } - return NewFastClient(o) -} - -// NewStdClient creates a client object that wraps the net/http standard client. -func NewStdClient(o *HTTPOptions) *Client { - o.Init(o.URL) // also normalizes NumConnections etc to be valid. - req := newHTTPRequest(o) - if req == nil { - return nil - } - tr := http.Transport{ - MaxIdleConns: o.NumConnections, - MaxIdleConnsPerHost: o.NumConnections, - DisableCompression: !o.Compression, - DisableKeepAlives: o.DisableKeepAlive, - Dial: (&net.Dialer{ - Timeout: o.HTTPReqTimeOut, - }).Dial, - TLSHandshakeTimeout: o.HTTPReqTimeOut, - } - if o.Insecure && o.https { - log.LogVf("using insecure https") - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // nolint: gas - } - client := Client{ - url: o.URL, - req: req, - client: &http.Client{ - Timeout: o.HTTPReqTimeOut, - Transport: &tr, - }, - transport: &tr, - } - if !o.FollowRedirects { - // Lets us see the raw response instead of auto following redirects. - client.client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - } - return &client -} - -// FetchURL fetches the data at the given url using the standard client and default options. -// Returns the http status code (http.StatusOK == 200 for success) and the data. -// To be used only for single fetches or when performance doesn't matter as the client is closed at the end. -func FetchURL(url string) (int, []byte) { - o := NewHTTPOptions(url) - // Maximize chances of getting the data back, vs the raw payload like the fast client - o.DisableFastClient = true - o.FollowRedirects = true - return Fetch(o) -} - -// Fetch creates a client an performs a fetch according to the http options passed in. -// To be used only for single fetches or when performance doesn't matter as the client is closed at the end. -func Fetch(httpOptions *HTTPOptions) (int, []byte) { - cli := NewClient(httpOptions) - code, data, _ := cli.Fetch() - cli.Close() - return code, data -} - -// FastClient is a fast, lockfree single purpose http 1.0/1.1 client. -type FastClient struct { - buffer []byte - req []byte - dest net.Addr - socket net.Conn - socketCount int - size int - code int - errorCount int - headerLen int - url string - host string - hostname string - port string - http10 bool // http 1.0, simplest: no Host, forced no keepAlive, no parsing - keepAlive bool - parseHeaders bool // don't bother in http/1.0 - halfClose bool // allow/do half close when keepAlive is false - reqTimeout time.Duration -} - -// Close cleans up any resources used by FastClient -func (c *FastClient) Close() int { - log.Debugf("Closing %p %s socket count %d", c, c.url, c.socketCount) - if c.socket != nil { - if err := c.socket.Close(); err != nil { - log.Warnf("Error closing fast client's socket: %v", err) - } - c.socket = nil - } - return c.socketCount -} - -// NewFastClient makes a basic, efficient http 1.0/1.1 client. -// This function itself doesn't need to be super efficient as it is created at -// the beginning and then reused many times. -func NewFastClient(o *HTTPOptions) Fetcher { - method := o.Method() - payloadLen := len(o.Payload) - o.Init(o.URL) - proto := "1.1" - if o.HTTP10 { - proto = "1.0" - } - // Parse the url, extract components. - url, err := url.Parse(o.URL) - if err != nil { - log.Errf("Bad url '%s' : %v", o.URL, err) - return nil - } - if url.Scheme != "http" { - log.Errf("Only http is supported with the optimized client, use -stdclient for url %s", o.URL) - return nil - } - // note: Host includes the port - bc := FastClient{url: o.URL, host: url.Host, hostname: url.Hostname(), port: url.Port(), - http10: o.HTTP10, halfClose: o.AllowHalfClose} - bc.buffer = make([]byte, BufferSizeKb*1024) - if bc.port == "" { - bc.port = url.Scheme // ie http which turns into 80 later - log.LogVf("No port specified, using %s", bc.port) - } - var addr net.Addr - if o.UnixDomainSocket != "" { - log.Infof("Using unix domain socket %v instead of %v %v", o.UnixDomainSocket, bc.hostname, bc.port) - uds := &net.UnixAddr{Name: o.UnixDomainSocket, Net: fnet.UnixDomainSocket} - addr = uds - } else { - addr = fnet.Resolve(bc.hostname, bc.port) - } - if addr == nil { - // Error already logged - return nil - } - bc.dest = addr - // Create the bytes for the request: - host := bc.host - if o.hostOverride != "" { - host = o.hostOverride - } - var buf bytes.Buffer - buf.WriteString(method + " " + url.RequestURI() + " HTTP/" + proto + "\r\n") - if !bc.http10 { - buf.WriteString("Host: " + host + "\r\n") - bc.parseHeaders = true - if !o.DisableKeepAlive { - bc.keepAlive = true - } else { - buf.WriteString("Connection: close\r\n") - } - } - bc.reqTimeout = o.HTTPReqTimeOut - w := bufio.NewWriter(&buf) - // This writes multiple valued headers properly (unlike calling Get() to do it ourselves) - o.GenerateHeaders().Write(w) // nolint: errcheck,gas - w.Flush() // nolint: errcheck,gas - buf.WriteString("\r\n") - //Add the payload to http body - if payloadLen > 0 { - buf.Write(o.Payload) - } - bc.req = buf.Bytes() - log.Debugf("Created client:\n%+v\n%s", bc.dest, bc.req) - return &bc -} - -// return the result from the state. -func (c *FastClient) returnRes() (int, []byte, int) { - return c.code, c.buffer[:c.size], c.headerLen -} - -// connect to destination. -func (c *FastClient) connect() net.Conn { - c.socketCount++ - socket, err := net.Dial(c.dest.Network(), c.dest.String()) - if err != nil { - log.Errf("Unable to connect to %v : %v", c.dest, err) - return nil - } - tcpSock, ok := socket.(*net.TCPConn) - if !ok { - log.LogVf("Not setting socket options on non tcp socket %v", socket.RemoteAddr()) - return socket - } - // For now those errors are not critical/breaking - if err = tcpSock.SetNoDelay(true); err != nil { - log.Warnf("Unable to connect to set tcp no delay %v %v : %v", socket, c.dest, err) - } - if err = tcpSock.SetWriteBuffer(len(c.req)); err != nil { - log.Warnf("Unable to connect to set write buffer %d %v %v : %v", len(c.req), socket, c.dest, err) - } - if err = tcpSock.SetReadBuffer(len(c.buffer)); err != nil { - log.Warnf("Unable to connect to read buffer %d %v %v : %v", len(c.buffer), socket, c.dest, err) - } - return socket -} - -// Extra error codes outside of the HTTP Status code ranges. ie negative. -const ( - // SocketError is return when a transport error occurred: unexpected EOF, connection error, etc... - SocketError = -1 - // RetryOnce is used internally as an error code to allow 1 retry for bad socket reuse. - RetryOnce = -2 -) - -// Fetch fetches the url content. Returns http code, data, offset of body. -func (c *FastClient) Fetch() (int, []byte, int) { - c.code = SocketError - c.size = 0 - c.headerLen = 0 - // Connect or reuse existing socket: - conn := c.socket - reuse := (conn != nil) - if !reuse { - conn = c.connect() - if conn == nil { - return c.returnRes() - } - } else { - log.Debugf("Reusing socket %v", conn) - } - c.socket = nil // because of error returns and single retry - conErr := conn.SetReadDeadline(time.Now().Add(c.reqTimeout)) - // Send the request: - n, err := conn.Write(c.req) - if err != nil || conErr != nil { - if reuse { - // it's ok for the (idle) socket to die once, auto reconnect: - log.Infof("Closing dead socket %v (%v)", conn, err) - conn.Close() // nolint: errcheck,gas - c.errorCount++ - return c.Fetch() // recurse once - } - log.Errf("Unable to write to %v %v : %v", conn, c.dest, err) - return c.returnRes() - } - if n != len(c.req) { - log.Errf("Short write to %v %v : %d instead of %d", conn, c.dest, n, len(c.req)) - return c.returnRes() - } - if !c.keepAlive && c.halfClose { - tcpConn, ok := conn.(*net.TCPConn) - if ok { - if err = tcpConn.CloseWrite(); err != nil { - log.Errf("Unable to close write to %v %v : %v", conn, c.dest, err) - return c.returnRes() - } // else: - log.Debugf("Half closed ok after sending request %v %v", conn, c.dest) - } else { - log.Warnf("Unable to close write non tcp connection %v", conn) - } - } - // Read the response: - c.readResponse(conn, reuse) - if c.code == RetryOnce { - // Special "eof on reused socket" code - return c.Fetch() // recurse once - } - // Return the result: - return c.returnRes() -} - -// Response reading: -// TODO: refactor - unwiedly/ugly atm -func (c *FastClient) readResponse(conn net.Conn, reusedSocket bool) { - max := len(c.buffer) - parsedHeaders := false - // TODO: safer to start with -1 / SocketError and fix ok for http 1.0 - c.code = http.StatusOK // In http 1.0 mode we don't bother parsing anything - endofHeadersStart := retcodeOffset + 3 - keepAlive := c.keepAlive - chunkedMode := false - checkConnectionClosedHeader := CheckConnectionClosedHeader - skipRead := false - for { - // Ugly way to cover the case where we get more than 1 chunk at the end - // TODO: need automated tests - if !skipRead { - n, err := conn.Read(c.buffer[c.size:]) - if err != nil { - if reusedSocket && c.size == 0 { - // Ok for reused socket to be dead once (close by server) - log.Infof("Closing dead socket %v (err %v at first read)", conn, err) - c.errorCount++ - err = conn.Close() // close the previous one - if err != nil { - log.Warnf("Error closing dead socket %v: %v", conn, err) - } - c.code = RetryOnce // special "retry once" code - return - } - if err == io.EOF && c.size != 0 { - // handled below as possibly normal end of stream after we read something - break - } - log.Errf("Read error %v %v %d : %v", conn, c.dest, c.size, err) - c.code = SocketError - break - } - c.size += n - if log.LogDebug() { - log.Debugf("Read ok %d total %d so far (-%d headers = %d data) %s", - n, c.size, c.headerLen, c.size-c.headerLen, DebugSummary(c.buffer[c.size-n:c.size], 256)) - } - } - skipRead = false - // Have not yet parsed the headers, need to parse the headers, and have enough data to - // at least parse the http retcode: - if !parsedHeaders && c.parseHeaders && c.size >= retcodeOffset+3 { - // even if the bytes are garbage we'll get a non 200 code (bytes are unsigned) - c.code = ParseDecimal(c.buffer[retcodeOffset : retcodeOffset+3]) //TODO do that only once... - // TODO handle 100 Continue - if c.code != http.StatusOK { - log.Warnf("Parsed non ok code %d (%v)", c.code, string(c.buffer[:retcodeOffset+3])) - break - } - if log.LogDebug() { - log.Debugf("Code %d, looking for end of headers at %d / %d, last CRLF %d", - c.code, endofHeadersStart, c.size, c.headerLen) - } - // TODO: keep track of list of newlines to efficiently search headers only there - idx := endofHeadersStart - for idx < c.size-1 { - if c.buffer[idx] == '\r' && c.buffer[idx+1] == '\n' { - if c.headerLen == idx-2 { // found end of headers - parsedHeaders = true - break - } - c.headerLen = idx - idx++ - } - idx++ - } - endofHeadersStart = c.size // start there next read - if parsedHeaders { - // We have headers ! - c.headerLen += 4 // we use this and not endofHeadersStart so http/1.0 does return 0 and not the optimization for search start - if log.LogDebug() { - log.Debugf("headers are %d: %s", c.headerLen, c.buffer[:idx]) - } - // Find the content length or chunked mode - if keepAlive { - var contentLength int - found, offset := FoldFind(c.buffer[:c.headerLen], contentLengthHeader) - if found { - // Content-Length mode: - contentLength = ParseDecimal(c.buffer[offset+len(contentLengthHeader) : c.headerLen]) - if contentLength < 0 { - log.Warnf("Warning: content-length unparsable %s", string(c.buffer[offset+2:offset+len(contentLengthHeader)+4])) - keepAlive = false - break - } - max = c.headerLen + contentLength - if log.LogDebug() { // somehow without the if we spend 400ms/10s in LogV (!) - log.Debugf("found content length %d", contentLength) - } - } else { - // Chunked mode (or err/missing): - if found, _ := FoldFind(c.buffer[:c.headerLen], chunkedHeader); found { - chunkedMode = true - var dataStart int - dataStart, contentLength = ParseChunkSize(c.buffer[c.headerLen:c.size]) - if contentLength == -1 { - // chunk length not available yet - log.LogVf("chunk mode but no first chunk length yet, reading more") - max = c.headerLen - continue - } - max = c.headerLen + dataStart + contentLength + 2 // extra CR LF - log.Debugf("chunk-length is %d (%s) setting max to %d", - contentLength, c.buffer[c.headerLen:c.headerLen+dataStart-2], - max) - } else { - if log.LogVerbose() { - log.LogVf("Warning: content-length missing in %s", string(c.buffer[:c.headerLen])) - } else { - log.Warnf("Warning: content-length missing (%d bytes headers)", c.headerLen) - } - keepAlive = false // can't keep keepAlive - break - } - } // end of content-length section - if max > len(c.buffer) { - log.Warnf("Buffer is too small for headers %d + data %d - change -httpbufferkb flag to at least %d", - c.headerLen, contentLength, (c.headerLen+contentLength)/1024+1) - // TODO: just consume the extra instead - max = len(c.buffer) - } - if checkConnectionClosedHeader { - if found, _ := FoldFind(c.buffer[:c.headerLen], connectionCloseHeader); found { - log.Infof("Server wants to close connection, no keep-alive!") - keepAlive = false - max = len(c.buffer) // reset to read as much as available - } - } - } - } - } // end of big if parse header - if c.size >= max { - if !keepAlive { - log.Errf("More data is available but stopping after %d, increase -httpbufferkb", max) - } - if !parsedHeaders && c.parseHeaders { - log.Errf("Buffer too small (%d) to even finish reading headers, increase -httpbufferkb to get all the data", max) - keepAlive = false - } - if chunkedMode { - // Next chunk: - dataStart, nextChunkLen := ParseChunkSize(c.buffer[max:c.size]) - if nextChunkLen == -1 { - if c.size == max { - log.Debugf("Couldn't find next chunk size, reading more %d %d", max, c.size) - } else { - log.Infof("Partial chunk size (%s), reading more %d %d", DebugSummary(c.buffer[max:c.size], 20), max, c.size) - } - continue - } else if nextChunkLen == 0 { - log.Debugf("Found last chunk %d %d", max+dataStart, c.size) - if c.size != max+dataStart+2 || string(c.buffer[c.size-2:c.size]) != "\r\n" { - log.Errf("Unexpected mismatch at the end sz=%d expected %d; end of buffer %q", c.size, max+dataStart+2, c.buffer[max:c.size]) - } - } else { - max += dataStart + nextChunkLen + 2 // extra CR LF - log.Debugf("One more chunk %d -> new max %d", nextChunkLen, max) - if max > len(c.buffer) { - log.Errf("Buffer too small for %d data", max) - } else { - if max <= c.size { - log.Debugf("Enough data to reach next chunk, skipping a read") - skipRead = true - } - continue - } - } - } - break // we're done! - } - } // end of big for loop - // Figure out whether to keep or close the socket: - if keepAlive && c.code == http.StatusOK { - c.socket = conn // keep the open socket - } else { - if err := conn.Close(); err != nil { - log.Errf("Close error %v %v %d : %v", conn, c.dest, c.size, err) - } else { - log.Debugf("Closed ok %v from %v after reading %d bytes", conn, c.dest, c.size) - } - // we cleared c.socket in caller already - } -} diff --git a/vendor/fortio.org/fortio/fhttp/http_server.go b/vendor/fortio.org/fortio/fhttp/http_server.go deleted file mode 100644 index a6caa66f5f..0000000000 --- a/vendor/fortio.org/fortio/fhttp/http_server.go +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fhttp // import "fortio.org/fortio/fhttp" - -import ( - "bytes" - "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "sort" - "strconv" - "strings" - "sync/atomic" - "time" - // get /debug/pprof endpoints on a mux through SetupPPROF - "net/http/pprof" - - "fortio.org/fortio/fnet" - "fortio.org/fortio/log" - "fortio.org/fortio/version" -) - -// -- Echo Server -- - -var ( - // Start time of the server (used in debug handler for uptime). - startTime time.Time - // EchoRequests is the number of request received. Only updated in Debug mode. - EchoRequests int64 -) - -// EchoHandler is an http server handler echoing back the input. -func EchoHandler(w http.ResponseWriter, r *http.Request) { - if log.LogVerbose() { - LogRequest(r, "Echo") // will also print headers - } - data, err := ioutil.ReadAll(r.Body) // must be done before calling FormValue - if err != nil { - log.Errf("Error reading %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - log.Debugf("Read %d", len(data)) - dur := generateDelay(r.FormValue("delay")) - if dur > 0 { - log.LogVf("Sleeping for %v", dur) - time.Sleep(dur) - } - statusStr := r.FormValue("status") - var status int - if statusStr != "" { - status = generateStatus(statusStr) - } else { - status = http.StatusOK - } - if log.LogDebug() { - // TODO: this easily lead to contention - use 'thread local' - rqNum := atomic.AddInt64(&EchoRequests, 1) - log.Debugf("Request # %v", rqNum) - } - if r.FormValue("close") != "" { - log.Debugf("Adding Connection:close / will close socket") - w.Header().Set("Connection", "close") - } - // process header(s) args, must be before size to compose properly - for _, hdr := range r.Form["header"] { - log.LogVf("Adding requested header %s", hdr) - if len(hdr) == 0 { - continue - } - s := strings.SplitN(hdr, ":", 2) - if len(s) != 2 { - log.Errf("invalid extra header '%s', expecting Key: Value", hdr) - continue - } - w.Header().Add(s[0], s[1]) - } - size := generateSize(r.FormValue("size")) - if size >= 0 { - log.LogVf("Writing %d size with %d status", size, status) - writePayload(w, status, size) - return - } - // echo back the Content-Type and Content-Length in the response - for _, k := range []string{"Content-Type", "Content-Length"} { - if v := r.Header.Get(k); v != "" { - w.Header().Set(k, v) - } - } - w.WriteHeader(status) - if _, err = w.Write(data); err != nil { - log.Errf("Error writing response %v to %v", err, r.RemoteAddr) - } -} - -func writePayload(w http.ResponseWriter, status int, size int) { - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(size)) - w.WriteHeader(status) - n, err := w.Write(fnet.Payload[:size]) - if err != nil || n != size { - log.Errf("Error writing payload of size %d: %d %v", size, n, err) - } -} - -func closingServer(listener net.Listener) error { - var err error - for { - var c net.Conn - c, err = listener.Accept() - if err != nil { - log.Errf("Accept error in dummy server %v", err) - break - } - log.LogVf("Got connection from %v, closing", c.RemoteAddr()) - err = c.Close() - if err != nil { - log.Errf("Close error in dummy server %v", err) - break - } - } - return err -} - -// HTTPServer creates an http server named name on address/port port. -// Port can include binding address and/or be port 0. -func HTTPServer(name string, port string) (*http.ServeMux, net.Addr) { - m := http.NewServeMux() - s := &http.Server{ - Handler: m, - } - listener, addr := fnet.Listen(name, port) - if listener == nil { - return nil, nil // error already logged - } - go func() { - err := s.Serve(listener) - if err != nil { - log.Fatalf("Unable to serve %s on %s: %v", name, addr.String(), err) - } - }() - return m, addr -} - -// DynamicHTTPServer listens on an available port, sets up an http or a closing -// server simulating an https server (when closing is true) server on it and -// returns the listening port and mux to which one can attach handlers to. -// Note: in a future version of istio, the closing will be actually be secure -// on/off and create an https server instead of a closing server. -// As this is a dynamic tcp socket server, the address is TCP. -func DynamicHTTPServer(closing bool) (*http.ServeMux, *net.TCPAddr) { - if !closing { - mux, addr := HTTPServer("dynamic", "0") - return mux, addr.(*net.TCPAddr) - } - // Note: we actually use the fact it's not supported as an error server for tests - need to change that - log.Errf("Secure setup not yet supported. Will just close incoming connections for now") - listener, addr := fnet.Listen("closing server", "0") - //err = http.ServeTLS(listener, nil, "", "") // go 1.9 - go func() { - err := closingServer(listener) - if err != nil { - log.Fatalf("Unable to serve closing server on %s: %v", addr.String(), err) - } - }() - return nil, addr.(*net.TCPAddr) -} - -/* -// DebugHandlerTemplate returns debug/useful info on the http requet. -// slower heavier but nicer source code version of DebugHandler -func DebugHandlerTemplate(w http.ResponseWriter, r *http.Request) { - log.LogVf("%v %v %v %v", r.Method, r.URL, r.Proto, r.RemoteAddr) - hostname, _ := os.Hostname() - data, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Errf("Error reading %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // Note: this looks nicer but is about 2x slower / less qps / more cpu and 25% bigger executable than doing the writes oneself: - const templ = `Φορτίο version {{.Version}} echo debug server on {{.Hostname}} - request from {{.R.RemoteAddr}} - -{{.R.Method}} {{.R.URL}} {{.R.Proto}} - -headers: - -{{ range $name, $vals := .R.Header }}{{range $val := $vals}}{{$name}}: {{ $val }} -{{end}}{{end}} -body: - -{{.Body}} -{{if .DumpEnv}} -environment: -{{ range $idx, $e := .Env }} -{{$e}}{{end}} -{{end}}` - t := template.Must(template.New("debugOutput").Parse(templ)) - err = t.Execute(w, &struct { - R *http.Request - Hostname string - Version string - Body string - DumpEnv bool - Env []string - }{r, hostname, Version, DebugSummary(data, 512), r.FormValue("env") == "dump", os.Environ()}) - if err != nil { - Critf("Template execution failed: %v", err) - } - w.Header().Set("Content-Type", "text/plain; charset=UTF-8") -} -*/ - -// DebugHandler returns debug/useful info to http client. -func DebugHandler(w http.ResponseWriter, r *http.Request) { - if log.LogVerbose() { - LogRequest(r, "Debug") - } - var buf bytes.Buffer - buf.WriteString("Φορτίο version ") - buf.WriteString(version.Long()) - buf.WriteString(" echo debug server up for ") - buf.WriteString(fmt.Sprint(RoundDuration(time.Since(startTime)))) - buf.WriteString(" on ") - hostname, _ := os.Hostname() // nolint: gas - buf.WriteString(hostname) - buf.WriteString(" - request from ") - buf.WriteString(r.RemoteAddr) - buf.WriteString("\n\n") - buf.WriteString(r.Method) - buf.WriteByte(' ') - buf.WriteString(r.URL.String()) - buf.WriteByte(' ') - buf.WriteString(r.Proto) - buf.WriteString("\n\nheaders:\n\n") - // Host is removed from headers map and put here (!) - buf.WriteString("Host: ") - buf.WriteString(r.Host) - - var keys []string - for k := range r.Header { - keys = append(keys, k) - } - sort.Strings(keys) - for _, name := range keys { - buf.WriteByte('\n') - buf.WriteString(name) - buf.WriteString(": ") - first := true - headers := r.Header[name] - for _, h := range headers { - if !first { - buf.WriteByte(',') - } - buf.WriteString(h) - first = false - } - } - data, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Errf("Error reading %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - buf.WriteString("\n\nbody:\n\n") - buf.WriteString(DebugSummary(data, 512)) - buf.WriteByte('\n') - if r.FormValue("env") == "dump" { - buf.WriteString("\nenvironment:\n\n") - for _, v := range os.Environ() { - buf.WriteString(v) - buf.WriteByte('\n') - } - } - w.Header().Set("Content-Type", "text/plain; charset=UTF-8") - if _, err = w.Write(buf.Bytes()); err != nil { - log.Errf("Error writing response %v to %v", err, r.RemoteAddr) - } -} - -// CacheOn sets the header for indefinite caching. -func CacheOn(w http.ResponseWriter) { - w.Header().Set("Cache-Control", "max-age=365000000, immutable") -} - -// Serve starts a debug / echo http server on the given port. -// Returns the mux and addr where the listening socket is bound. -// The .Port can be retrieved from it when requesting the 0 port as -// input for dynamic http server. -func Serve(port, debugPath string) (*http.ServeMux, net.Addr) { - startTime = time.Now() - mux, addr := HTTPServer("echo", port) - if addr == nil { - return nil, nil // error already logged - } - if debugPath != "" { - mux.HandleFunc(debugPath, DebugHandler) - } - mux.HandleFunc("/", EchoHandler) - return mux, addr -} - -// ServeTCP is Serve() but restricted to TCP (return address is assumed -// to be TCP - will panic for unix domain) -func ServeTCP(port, debugPath string) (*http.ServeMux, *net.TCPAddr) { - mux, addr := Serve(port, debugPath) - if addr == nil { - return nil, nil // error already logged - } - return mux, addr.(*net.TCPAddr) -} - -// -- formerly in ui handler - -// SetupPPROF add pprof to the mux (mirror the init() of http pprof). -func SetupPPROF(mux *http.ServeMux) { - mux.HandleFunc("/debug/pprof/", LogAndCall("pprof:index", pprof.Index)) - mux.HandleFunc("/debug/pprof/cmdline", LogAndCall("pprof:cmdline", pprof.Cmdline)) - mux.HandleFunc("/debug/pprof/profile", LogAndCall("pprof:profile", pprof.Profile)) - mux.HandleFunc("/debug/pprof/symbol", LogAndCall("pprof:symbol", pprof.Symbol)) - mux.HandleFunc("/debug/pprof/trace", LogAndCall("pprof:trace", pprof.Trace)) -} - -// -- Fetch er (simple http proxy) -- - -// FetcherHandler is the handler for the fetcher/proxy. -func FetcherHandler(w http.ResponseWriter, r *http.Request) { - LogRequest(r, "Fetch (prefix stripped)") - hj, ok := w.(http.Hijacker) - if !ok { - log.Critf("hijacking not supported") - return - } - conn, _, err := hj.Hijack() - if err != nil { - log.Errf("hijacking error %v", err) - return - } - // Don't forget to close the connection: - defer conn.Close() // nolint: errcheck - // Stripped prefix gets replaced by ./ - sometimes... - url := strings.TrimPrefix(r.URL.String(), "./") - opts := NewHTTPOptions("http://" + url) - opts.HTTPReqTimeOut = 5 * time.Minute - OnBehalfOf(opts, r) - client := NewClient(opts) - if client == nil { - return // error logged already - } - _, data, _ := client.Fetch() - _, err = conn.Write(data) - if err != nil { - log.Errf("Error writing fetched data to %v: %v", r.RemoteAddr, err) - } - client.Close() -} - -// -- Redirection to https feature -- - -// RedirectToHTTPSHandler handler sends a redirect to same URL with https. -func RedirectToHTTPSHandler(w http.ResponseWriter, r *http.Request) { - dest := "https://" + r.Host + r.URL.String() - LogRequest(r, "Redirecting to "+dest) - http.Redirect(w, r, dest, http.StatusSeeOther) -} - -// RedirectToHTTPS Sets up a redirector to https on the given port. -// (Do not create a loop, make sure this is addressed from an ingress) -func RedirectToHTTPS(port string) net.Addr { - m, a := HTTPServer("https redirector", port) - if m == nil { - return nil // error already logged - } - m.HandleFunc("/", RedirectToHTTPSHandler) - return a -} - -// LogRequest logs the incoming request, including headers when loglevel is verbose -func LogRequest(r *http.Request, msg string) { - log.Infof("%s: %v %v %v %v (%s)", msg, r.Method, r.URL, r.Proto, r.RemoteAddr, - r.Header.Get("X-Forwarded-Proto")) - if log.LogVerbose() { - for name, headers := range r.Header { - for _, h := range headers { - log.LogVf("Header %v: %v\n", name, h) - } - } - } -} - -// LogAndCall wrapps an HTTP handler to log the request first. -func LogAndCall(msg string, hf http.HandlerFunc) http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - LogRequest(r, msg) - hf(w, r) - }) -} diff --git a/vendor/fortio.org/fortio/fhttp/http_utils.go b/vendor/fortio.org/fortio/fhttp/http_utils.go deleted file mode 100644 index 195461e8af..0000000000 --- a/vendor/fortio.org/fortio/fhttp/http_utils.go +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fhttp // import "fortio.org/fortio/fhttp" - -import ( - "encoding/base64" - "fmt" - "html/template" - "io" - "math/rand" - "net/http" - "strconv" - "strings" - "time" - "unicode/utf8" - - "fortio.org/fortio/fnet" - "fortio.org/fortio/log" - "fortio.org/fortio/stats" -) - -// Used for the fast case insensitive search -const toUpperMask = ^byte('a' - 'A') - -// Slow but correct version -func toUpper(b byte) byte { - if b >= 'a' && b <= 'z' { - b -= ('a' - 'A') - } - return b -} - -// ASCIIToUpper returns a byte array equal to the input string but in lowercase. -// Only works for ASCII, not meant for unicode. -func ASCIIToUpper(str string) []byte { - numChars := utf8.RuneCountInString(str) - if numChars != len(str) && log.LogVerbose() { - log.Errf("ASCIIFold(\"%s\") contains %d characters, some non ascii (byte length %d): will mangle", str, numChars, len(str)) - } - res := make([]byte, numChars) - // less surprising if we only mangle the extended characters - i := 0 - for _, c := range str { // Attention: _ here != i for unicode characters - res[i] = toUpper(byte(c)) - i++ - } - return res -} - -// FoldFind searches the bytes assuming ascii, ignoring the lowercase bit -// for testing. Not intended to work with unicode, meant for http headers -// and to be fast (see benchmark in test file). -func FoldFind(haystack []byte, needle []byte) (bool, int) { - idx := 0 - found := false - hackstackLen := len(haystack) - needleLen := len(needle) - if needleLen == 0 { - return true, 0 - } - if needleLen > hackstackLen { // those 2 ifs also handles haystackLen == 0 - return false, -1 - } - needleOffset := 0 - for { - h := haystack[idx] - n := needle[needleOffset] - // This line is quite performance sensitive. calling toUpper() for instance - // is a 30% hit, even if called only on the haystack. The XOR lets us be - // true for equality and the & with mask also true if the only difference - // between the 2 is the case bit. - xor := h ^ n // == 0 if strictly equal - if (xor&toUpperMask) != 0 || (((h < 32) || (n < 32)) && (xor != 0)) { - idx -= (needleOffset - 1) // does ++ most of the time - needleOffset = 0 - if idx >= hackstackLen { - break - } - continue - } - if needleOffset == needleLen-1 { - found = true - break - } - needleOffset++ - idx++ - if idx >= hackstackLen { - break - } - } - if !found { - return false, -1 - } - return true, idx - needleOffset -} - -// ParseDecimal extracts the first positive integer number from the input. -// spaces are ignored. -// any character that isn't a digit cause the parsing to stop -func ParseDecimal(inp []byte) int { - res := -1 - for _, b := range inp { - if b == ' ' && res == -1 { - continue - } - if b < '0' || b > '9' { - break - } - digit := int(b - '0') - if res == -1 { - res = digit - } else { - res = 10*res + digit - } - } - return res -} - -// ParseChunkSize extracts the chunk size and consumes the line. -// Returns the offset of the data and the size of the chunk, -// 0, -1 when not found. -func ParseChunkSize(inp []byte) (int, int) { - if log.LogDebug() { - log.Debugf("ParseChunkSize(%s)", DebugSummary(inp, 128)) - } - res := -1 - off := 0 - end := len(inp) - inDigits := true - for { - if off >= end { - return off, -1 - } - if inDigits { - b := toUpper(inp[off]) - var digit int - if b >= 'A' && b <= 'F' { - digit = 10 + int(b-'A') - } else if b >= '0' && b <= '9' { - digit = int(b - '0') - } else { - inDigits = false - if res == -1 { - log.Errf("Didn't find hex number %q", inp) - return off, res - } - continue - } - if res == -1 { - res = digit - } else { - res = 16*res + digit - } - } else { - // After digits, skipping ahead to find \r\n - if inp[off] == '\r' { - off++ - if off >= end { - return off, -1 - } - if inp[off] == '\n' { - // good case - return off + 1, res - } - } - } - off++ - } -} - -// EscapeBytes returns printable string. Same as %q format without the -// surrounding/extra "". -func EscapeBytes(buf []byte) string { - e := fmt.Sprintf("%q", buf) - return e[1 : len(e)-1] -} - -// DebugSummary returns a string with the size and escaped first max/2 and -// last max/2 bytes of a buffer (or the whole escaped buffer if small enough). -func DebugSummary(buf []byte, max int) string { - l := len(buf) - if l <= max+3 { //no point in shortening to add ... if we could return those 3 - return EscapeBytes(buf) - } - max /= 2 - return fmt.Sprintf("%d: %s...%s", l, EscapeBytes(buf[:max]), EscapeBytes(buf[l-max:])) -} - -// -- server utils - -func removeTrailingPercent(s string) string { - if strings.HasSuffix(s, "%") { - return s[:len(s)-1] - } - return s -} - -// generateStatus from string, format: status="503" for 100% 503s -// status="503:20,404:10,403:0.5" for 20% 503s, 10% 404s, 0.5% 403s 69.5% 200s -func generateStatus(status string) int { - lst := strings.Split(status, ",") - log.Debugf("Parsing status %s -> %v", status, lst) - // Simple non probabilistic status case: - if len(lst) == 1 && !strings.ContainsRune(status, ':') { - s, err := strconv.Atoi(status) - if err != nil { - log.Warnf("Bad input status %v, not a number nor comma and colon separated %% list", status) - return http.StatusBadRequest - } - log.Debugf("Parsed status %s -> %d", status, s) - return s - } - weights := make([]float32, len(lst)) - codes := make([]int, len(lst)) - lastPercent := float64(0) - i := 0 - for _, entry := range lst { - l2 := strings.Split(entry, ":") - if len(l2) != 2 { - log.Warnf("Should have exactly 1 : in status list %s -> %v", status, entry) - return http.StatusBadRequest - } - s, err := strconv.Atoi(l2[0]) - if err != nil { - log.Warnf("Bad input status %v -> %v, not a number before colon", status, l2[0]) - return http.StatusBadRequest - } - percStr := removeTrailingPercent(l2[1]) - p, err := strconv.ParseFloat(percStr, 32) - if err != nil || p < 0 || p > 100 { - log.Warnf("Percentage is not a [0. - 100.] number in %v -> %v : %v %f", status, percStr, err, p) - return http.StatusBadRequest - } - lastPercent += p - // Round() needed to cover 'exactly' 100% and not more or less because of rounding errors - p32 := float32(stats.Round(lastPercent)) - if p32 > 100. { - log.Warnf("Sum of percentage is greater than 100 in %v %f %f %f", status, lastPercent, p, p32) - return http.StatusBadRequest - } - weights[i] = p32 - codes[i] = s - i++ - } - res := 100. * rand.Float32() - for i, v := range weights { - if res <= v { - log.Debugf("[0.-100.[ for %s roll %f got #%d -> %d", status, res, i, codes[i]) - return codes[i] - } - } - log.Debugf("[0.-100.[ for %s roll %f no hit, defaulting to OK", status, res) - return http.StatusOK // default/reminder of probability table -} - -// generateSize from string, format: "size=512" for 100% 512 bytes body replies, -// size="512:20,16384:10" for 20% 512 bytes, 10% 16k, 70% default echo back. -// returns -1 for the default case, so one can specify 0 and force no payload -// even if it's a post request with a payload (to test asymmetric large inbound -// small outbound). -// TODO: refactor similarities with status and delay -func generateSize(sizeInput string) (size int) { - size = -1 // default value/behavior - if len(sizeInput) == 0 { - return size - } - lst := strings.Split(sizeInput, ",") - log.Debugf("Parsing size %s -> %v", sizeInput, lst) - // Simple non probabilistic status case: - if len(lst) == 1 && !strings.ContainsRune(sizeInput, ':') { - s, err := strconv.Atoi(sizeInput) - if err != nil { - log.Warnf("Bad input size %v, not a number nor comma and colon separated %% list", sizeInput) - return size - } - size = s - log.Debugf("Parsed size %s -> %d", sizeInput, size) - fnet.ValidatePayloadSize(&size) - return size - } - weights := make([]float32, len(lst)) - sizes := make([]int, len(lst)) - lastPercent := float64(0) - i := 0 - for _, entry := range lst { - l2 := strings.Split(entry, ":") - if len(l2) != 2 { - log.Warnf("Should have exactly 1 : in size list %s -> %v", sizeInput, entry) - return size - } - s, err := strconv.Atoi(l2[0]) - if err != nil { - log.Warnf("Bad input size %v -> %v, not a number before colon", sizeInput, l2[0]) - return size - } - fnet.ValidatePayloadSize(&s) - percStr := removeTrailingPercent(l2[1]) - p, err := strconv.ParseFloat(percStr, 32) - if err != nil || p < 0 || p > 100 { - log.Warnf("Percentage is not a [0. - 100.] number in %v -> %v : %v %f", sizeInput, percStr, err, p) - return size - } - lastPercent += p - // Round() needed to cover 'exactly' 100% and not more or less because of rounding errors - p32 := float32(stats.Round(lastPercent)) - if p32 > 100. { - log.Warnf("Sum of percentage is greater than 100 in %v %f %f %f", sizeInput, lastPercent, p, p32) - return size - } - weights[i] = p32 - sizes[i] = s - i++ - } - res := 100. * rand.Float32() - for i, v := range weights { - if res <= v { - log.Debugf("[0.-100.[ for %s roll %f got #%d -> %d", sizeInput, res, i, sizes[i]) - return sizes[i] - } - } - log.Debugf("[0.-100.[ for %s roll %f no hit, defaulting to -1", sizeInput, res) - return size // default/reminder of probability table -} - -// MaxDelay is the maximum delay allowed for the echoserver responses. -// 1.5s so we can test the default 1s timeout in envoy. -const MaxDelay = 1500 * time.Millisecond - -// generateDelay from string, format: delay="100ms" for 100% 100ms delay -// delay="10ms:20,20ms:10,1s:0.5" for 20% 10ms, 10% 20ms, 0.5% 1s and 69.5% 0 -// TODO: very similar with generateStatus - refactor? -func generateDelay(delay string) time.Duration { - lst := strings.Split(delay, ",") - log.Debugf("Parsing delay %s -> %v", delay, lst) - if len(delay) == 0 { - return -1 - } - // Simple non probabilistic status case: - if len(lst) == 1 && !strings.ContainsRune(delay, ':') { - d, err := time.ParseDuration(delay) - if err != nil { - log.Warnf("Bad input delay %v, not a duration nor comma and colon separated %% list", delay) - return -1 - } - log.Debugf("Parsed delay %s -> %d", delay, d) - if d > MaxDelay { - d = MaxDelay - } - return d - } - weights := make([]float32, len(lst)) - delays := make([]time.Duration, len(lst)) - lastPercent := float64(0) - i := 0 - for _, entry := range lst { - l2 := strings.Split(entry, ":") - if len(l2) != 2 { - log.Warnf("Should have exactly 1 : in delay list %s -> %v", delay, entry) - return -1 - } - d, err := time.ParseDuration(l2[0]) - if err != nil { - log.Warnf("Bad input delay %v -> %v, not a number before colon", delay, l2[0]) - return -1 - } - if d > MaxDelay { - d = MaxDelay - } - percStr := removeTrailingPercent(l2[1]) - p, err := strconv.ParseFloat(percStr, 32) - if err != nil || p < 0 || p > 100 { - log.Warnf("Percentage is not a [0. - 100.] number in %v -> %v : %v %f", delay, percStr, err, p) - return -1 - } - lastPercent += p - // Round() needed to cover 'exactly' 100% and not more or less because of rounding errors - p32 := float32(stats.Round(lastPercent)) - if p32 > 100. { - log.Warnf("Sum of percentage is greater than 100 in %v %f %f %f", delay, lastPercent, p, p32) - return -1 - } - weights[i] = p32 - delays[i] = d - i++ - } - res := 100. * rand.Float32() - for i, v := range weights { - if res <= v { - log.Debugf("[0.-100.[ for %s roll %f got #%d -> %d", delay, res, i, delays[i]) - return delays[i] - } - } - log.Debugf("[0.-100.[ for %s roll %f no hit, defaulting to 0", delay, res) - return 0 -} - -// RoundDuration rounds to 10th of second. Only for positive durations. -// TODO: switch to Duration.Round once switched to go 1.9 -func RoundDuration(d time.Duration) time.Duration { - tenthSec := int64(100 * time.Millisecond) - r := int64(d+50*time.Millisecond) / tenthSec - return time.Duration(tenthSec * r) -} - -// -- formerly in uihandler: - -// HTMLEscapeWriter is an io.Writer escaping the output for safe html inclusion. -type HTMLEscapeWriter struct { - NextWriter io.Writer - Flusher http.Flusher -} - -func (w *HTMLEscapeWriter) Write(p []byte) (int, error) { - template.HTMLEscape(w.NextWriter, p) - if w.Flusher != nil { - w.Flusher.Flush() - } - return len(p), nil -} - -// NewHTMLEscapeWriter creates a io.Writer that can safely output -// to an http.ResponseWrite with HTMLEscape-ing. -func NewHTMLEscapeWriter(w io.Writer) io.Writer { - flusher, ok := w.(http.Flusher) - if !ok { - log.Errf("expected writer %+v to be an http.ResponseWriter and thus a http.Flusher", w) - flusher = nil - } - return &HTMLEscapeWriter{NextWriter: w, Flusher: flusher} -} - -// OnBehalfOf adds a header with the remote addr to an http options object. -func OnBehalfOf(o *HTTPOptions, r *http.Request) { - _ = o.AddAndValidateExtraHeader("X-On-Behalf-Of: " + r.RemoteAddr) -} - -// AddHTTPS replaces "http://" in url with "https://" or prepends "https://" -// if url does not contain prefix "http://". -func AddHTTPS(url string) string { - if len(url) > len(fnet.PrefixHTTP) { - if strings.EqualFold(url[:len(fnet.PrefixHTTP)], fnet.PrefixHTTP) { - log.Infof("Replacing http scheme with https for url: %s", url) - return fnet.PrefixHTTPS + url[len(fnet.PrefixHTTP):] - } - // returns url with normalized lowercase https prefix - if strings.EqualFold(url[:len(fnet.PrefixHTTPS)], fnet.PrefixHTTPS) { - return fnet.PrefixHTTPS + url[len(fnet.PrefixHTTPS):] - } - } - // url must not contain any prefix, so add https prefix - log.Infof("Prepending https:// to url: %s", url) - return fnet.PrefixHTTPS + url -} - -// generateBase64UserCredentials encodes the user credential to base64 and adds a Basic as prefix. -func generateBase64UserCredentials(userCredentials string) string { - return "Basic " + base64.StdEncoding.EncodeToString([]byte(userCredentials)) -} diff --git a/vendor/fortio.org/fortio/fnet/network.go b/vendor/fortio.org/fortio/fnet/network.go deleted file mode 100644 index 66aa8477bd..0000000000 --- a/vendor/fortio.org/fortio/fnet/network.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fnet // import "fortio.org/fortio/fnet" - -import ( - "fmt" - "io" - "io/ioutil" - "math/rand" - "net" - "os" - "strconv" - "strings" - "sync" - - "fortio.org/fortio/log" - "fortio.org/fortio/version" -) - -const ( - // DefaultGRPCPort is the Fortio gRPC server default port number. - DefaultGRPCPort = "8079" - // StandardHTTPPort is the Standard http port number. - StandardHTTPPort = "80" - // StandardHTTPSPort is the Standard https port number. - StandardHTTPSPort = "443" - // PrefixHTTP is a constant value for representing http protocol that can be added prefix of url - PrefixHTTP = "http://" - // PrefixHTTPS is a constant value for representing secure http protocol that can be added prefix of url - PrefixHTTPS = "https://" - - // POST is a constant value that indicates http method as post - POST = "POST" - // GET is a constant value that indicates http method as get - GET = "GET" - // UnixDomainSocket type for network addresses. - UnixDomainSocket = "unix" -) - -var ( - // MaxPayloadSize is the maximum size of payload to be generated by the - // EchoHandler size= argument. In bytes. - MaxPayloadSize = 256 * 1024 - // Payload that is returned during echo call - Payload []byte -) - -func init() { - ChangeMaxPayloadSize(MaxPayloadSize) -} - -// ChangeMaxPayloadSize is used to change max payload size and fill it with pseudorandom content -func ChangeMaxPayloadSize(newMaxPayloadSize int) { - if newMaxPayloadSize >= 0 { - MaxPayloadSize = newMaxPayloadSize - } else { - MaxPayloadSize = 0 - } - Payload = make([]byte, MaxPayloadSize) - // One shared and 'constant' (over time) but pseudo random content for payload - // (to defeat compression). We don't need crypto strength here, just low cpu - // and speed: - _, err := rand.Read(Payload) - if err != nil { - log.Errf("Error changing payload size, read for %d random payload failed: %v", newMaxPayloadSize, err) - } -} - -// NormalizePort parses port and returns host:port if port is in the form -// of host:port already or :port if port is only a port (doesn't contain :). -func NormalizePort(port string) string { - if strings.ContainsAny(port, ":") { - return port - } - return ":" + port -} - -// Listen returns a listener for the port. Port can be a port or a -// bind address and a port (e.g. "8080" or "[::1]:8080"...). If the -// port component is 0 a free port will be returned by the system. -// If the port is a pathname (contains a /) a unix domain socket listener -// will be used instead of regular tcp socket. -// This logs critical on error and returns nil (is meant for servers -// that must start). -func Listen(name string, port string) (net.Listener, net.Addr) { - sockType := "tcp" - nPort := port - if strings.Contains(port, "/") { - sockType = UnixDomainSocket - } else { - nPort = NormalizePort(port) - } - listener, err := net.Listen(sockType, nPort) - if err != nil { - log.Critf("Can't listen to %s socket %v (%v) for %s: %v", sockType, port, nPort, name, err) - return nil, nil - } - lAddr := listener.Addr() - if len(name) > 0 { - fmt.Printf("Fortio %s %s server listening on %s\n", version.Short(), name, lAddr) - } - return listener, lAddr -} - -// GetPort extracts the port for TCP sockets and the path for unix domain sockets. -func GetPort(lAddr net.Addr) string { - var lPort string - // Note: might panic if called with something else than unix or tcp socket addr, it's ok. - if lAddr.Network() == UnixDomainSocket { - lPort = lAddr.(*net.UnixAddr).Name - } else { - lPort = strconv.Itoa(lAddr.(*net.TCPAddr).Port) - } - return lPort -} - -// ResolveDestination returns the TCP address of the "host:port" suitable for net.Dial. -// nil in case of errors. -func ResolveDestination(dest string) net.Addr { - i := strings.LastIndex(dest, ":") // important so [::1]:port works - if i < 0 { - log.Errf("Destination '%s' is not host:port format", dest) - return nil - } - host := dest[0:i] - port := dest[i+1:] - return Resolve(host, port) -} - -// Resolve returns the TCP address of the host,port suitable for net.Dial. -// nil in case of errors. -func Resolve(host string, port string) net.Addr { - log.Debugf("Resolve() called with host=%s port=%s", host, port) - dest := &net.TCPAddr{} - if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { - log.Debugf("host %s looks like an IPv6, stripping []", host) - host = host[1 : len(host)-1] - } - isAddr := net.ParseIP(host) - var err error - if isAddr != nil { - log.Debugf("Host already an IP, will go to %s", isAddr) - dest.IP = isAddr - } else { - var addrs []net.IP - addrs, err = net.LookupIP(host) - if err != nil { - log.Errf("Unable to lookup '%s' : %v", host, err) - return nil - } - if len(addrs) > 1 && log.LogDebug() { - log.Debugf("Using only the first of the addresses for %s : %v", host, addrs) - } - log.Debugf("Will go to %s", addrs[0]) - dest.IP = addrs[0] - } - dest.Port, err = net.LookupPort("tcp", port) - if err != nil { - log.Errf("Unable to resolve port '%s' : %v", port, err) - return nil - } - return dest -} - -func transfer(wg *sync.WaitGroup, dst net.Conn, src net.Conn) { - n, oErr := io.Copy(dst, src) // keep original error for logs below - log.LogVf("Proxy: transferred %d bytes from %v to %v (err=%v)", n, src.RemoteAddr(), dst.RemoteAddr(), oErr) - sTCP, ok := src.(*net.TCPConn) - if ok { - err := sTCP.CloseRead() - if err != nil { // We got an eof so it's already half closed. - log.LogVf("Proxy: semi expected error CloseRead on src %v: %v,%v", src.RemoteAddr(), err, oErr) - } - } - dTCP, ok := dst.(*net.TCPConn) - if ok { - err := dTCP.CloseWrite() - if err != nil { - log.Errf("Proxy: error CloseWrite on dst %v: %v,%v", dst.RemoteAddr(), err, oErr) - } - } - wg.Done() -} - -func handleProxyRequest(conn net.Conn, dest net.Addr) { - err := fmt.Errorf("nil destination") - var d net.Conn - if dest != nil { - d, err = net.Dial(dest.Network(), dest.String()) - } - if err != nil { - log.Errf("Proxy: unable to connect to %v for %v : %v", dest, conn.RemoteAddr(), err) - _ = conn.Close() - return - } - var wg sync.WaitGroup - wg.Add(2) - go transfer(&wg, d, conn) - transfer(&wg, conn, d) - wg.Wait() - log.LogVf("Proxy: both sides of transfer to %v for %v done", dest, conn.RemoteAddr()) - // Not checking as we are closing/ending anyway - note: bad side effect of coverage... - _ = d.Close() - _ = conn.Close() -} - -// Proxy starts a tcp proxy. -func Proxy(port string, dest net.Addr) net.Addr { - listener, lAddr := Listen(fmt.Sprintf("proxy for %v", dest), port) - if listener == nil { - return nil // error already logged - } - go func() { - for { - conn, err := listener.Accept() - if err != nil { - log.Critf("Proxy: error accepting: %v", err) // will this loop with error? - } else { - log.LogVf("Proxy: Accepted proxy connection from %v -> %v (for listener %v)", - conn.RemoteAddr(), conn.LocalAddr(), dest) - // TODO limit number of go request, use worker pool, etc... - go handleProxyRequest(conn, dest) - } - } - }() - return lAddr -} - -// ProxyToDestination opens a proxy from the listenPort (or addr:port or unix domain socket path) and forwards -// all traffic to destination (host:port) -func ProxyToDestination(listenPort string, destination string) net.Addr { - return Proxy(listenPort, ResolveDestination(destination)) -} - -// NormalizeHostPort generates host:port string for the address or uses localhost instead of [::] -// when the original port binding input didn't specify an address -func NormalizeHostPort(inputPort string, addr net.Addr) string { - urlHostPort := addr.String() - if addr.Network() == UnixDomainSocket { - urlHostPort = fmt.Sprintf("-unix-socket=%s", urlHostPort) - } else { - if strings.HasPrefix(inputPort, ":") || !strings.Contains(inputPort, ":") { - urlHostPort = fmt.Sprintf("localhost:%d", addr.(*net.TCPAddr).Port) - } - } - return urlHostPort -} - -// ValidatePayloadSize compares input size with MaxPayLoadSize. If size exceeds the MaxPayloadSize -// size will set to MaxPayLoadSize -func ValidatePayloadSize(size *int) { - if *size > MaxPayloadSize && *size > 0 { - log.Warnf("Requested size %d greater than max size %d, using max instead (change max using -maxpayloadsizekb)", - *size, MaxPayloadSize) - *size = MaxPayloadSize - } else if *size < 0 { - log.Warnf("Requested size %d is negative, using 0 (no additional payload) instead.", *size) - *size = 0 - } -} - -// GenerateRandomPayload generates a random payload with given input size -func GenerateRandomPayload(payloadSize int) []byte { - ValidatePayloadSize(&payloadSize) - return Payload[:payloadSize] -} - -// ReadFileForPayload reads the file from given input path -func ReadFileForPayload(payloadFilePath string) ([]byte, error) { - data, err := ioutil.ReadFile(payloadFilePath) - if err != nil { - return nil, err - } - return data, nil -} - -// GeneratePayload generates a payload with given inputs. -// First tries filePath, then random payload, at last payload -func GeneratePayload(payloadFilePath string, payloadSize int, payload string) []byte { - if len(payloadFilePath) > 0 { - p, err := ReadFileForPayload(payloadFilePath) - if err != nil { - log.Warnf("File read operation is failed %v", err) - return nil - } - return p - } else if payloadSize > 0 { - return GenerateRandomPayload(payloadSize) - } else { - return []byte(payload) - } -} - -// GetUniqueUnixDomainPath returns a path to be used for unix domain socket. -func GetUniqueUnixDomainPath(prefix string) string { - if prefix == "" { - prefix = "fortio-uds" - } - f, err := ioutil.TempFile(os.TempDir(), prefix) - if err != nil { - log.Errf("Unable to generate temp file with prefix %s: %v", prefix, err) - return "/tmp/fortio-default-uds" - } - fname := f.Name() - _ = f.Close() - // for the bind to succeed we need the file to not pre exist: - _ = os.Remove(fname) - return fname -} diff --git a/vendor/fortio.org/fortio/version/version.go b/vendor/fortio.org/fortio/version/version.go deleted file mode 100644 index 4b2dbb63fa..0000000000 --- a/vendor/fortio.org/fortio/version/version.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package version for fortio holds version information and build information. -package version // import "fortio.org/fortio/version" -import ( - "fmt" - "runtime" - - "fortio.org/fortio/log" -) - -const ( - major = 1 - minor = 3 - patch = 0 - - debug = false // turn on to debug init() -) - -var ( - // The following are set by Dockerfile during link time: - tag = "n/a" - buildInfo = "unknown" - // Number of lines in git status --porcelain; 0 means clean - gitstatus = "0" // buildInfo default is unknown so no need to add -dirty - // computed in init() - version = "" - longVersion = "" -) - -// Major returns the numerical major version number (first digit of version.Short()). -func Major() int { - return major -} - -// Minor returns the numerical minor version number (second digit of version.Short()). -func Minor() int { - return minor -} - -// Patch returns the numerical patch level (third digit of version.Short()). -func Patch() int { - return patch -} - -// Short returns the 3 digit short version string Major.Minor.Patch[-pre] -// version.Short() is the overall project version (used to version json -// output too). "-pre" is added when the version doesn't match exactly -// a git tag or the build isn't from a clean source tree. (only standard -// dockerfile based build of a clean, tagged source tree should print "X.Y.Z" -// as short version). -func Short() string { - return version -} - -// Long returns the full version and build information. -// Format is "X.Y.X[-pre] YYYY-MM-DD HH:MM SHA[-dirty]" date and time is -// the build date (UTC), sha is the git sha of the source tree. -func Long() string { - return longVersion -} - -// Carefully manually tested all the combinations in pair with Dockerfile -func init() { - if debug { - log.SetLogLevel(log.Debug) - } - version = fmt.Sprintf("%d.%d.%d", major, minor, patch) - clean := (gitstatus == "0") - // The docker build will pass the git tag to the build, if it is clean - // from a tag it will look like v0.7.0 - if tag != "v"+version || !clean { - log.Debugf("tag is %v, clean is %v marking as pre release", tag, clean) - version += "-pre" - } - if !clean { - buildInfo += "-dirty" - log.Debugf("gitstatus is %q, marking buildinfo as dirty: %v", gitstatus, buildInfo) - } - longVersion = version + " " + buildInfo + " " + runtime.Version() -} diff --git a/vendor/fortio.org/fortio/LICENSE b/vendor/istio.io/fortio/LICENSE similarity index 100% rename from vendor/fortio.org/fortio/LICENSE rename to vendor/istio.io/fortio/LICENSE diff --git a/vendor/istio.io/fortio/fhttp/http.go b/vendor/istio.io/fortio/fhttp/http.go new file mode 100644 index 0000000000..180638297c --- /dev/null +++ b/vendor/istio.io/fortio/fhttp/http.go @@ -0,0 +1,1162 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fhttp // import "istio.io/fortio/fhttp" + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "istio.io/fortio/fnet" + "istio.io/fortio/log" + "istio.io/fortio/periodic" + "istio.io/fortio/stats" +) + +// Fetcher is the Url content fetcher that the different client implements. +type Fetcher interface { + // Fetch returns http code, data, offset of body (for client which returns + // headers) + Fetch() (int, []byte, int) +} + +var ( + // BufferSizeKb size of the buffer (max data) for optimized client in kilobytes defaults to 32k. + BufferSizeKb = 128 + // CheckConnectionClosedHeader indicates whether to check for server side connection closed headers. + CheckConnectionClosedHeader = false + // 'constants', case doesn't matter for those 3 + contentLengthHeader = []byte("\r\ncontent-length:") + connectionCloseHeader = []byte("\r\nconnection: close") + chunkedHeader = []byte("\r\nTransfer-Encoding: chunked") + // Start time of the server (used in debug handler for uptime). + startTime time.Time +) + +// NewHTTPOptions creates and initialize a HTTPOptions object. +// It replaces plain % to %25 in the url. If you already have properly +// escaped URLs use o.URL = to set it. +func NewHTTPOptions(url string) *HTTPOptions { + h := HTTPOptions{} + return h.Init(url) +} + +// Init initializes the headers in an HTTPOptions (User-Agent). +// It replaces plain % to %25 in the url. If you already have properly +// escaped URLs use o.URL = to set it. +func (h *HTTPOptions) Init(url string) *HTTPOptions { + if h.initDone { + return h + } + h.initDone = true + // unescape then rescape % to %25 (so if it was already %25 it stays) + h.URL = strings.Replace(strings.Replace(url, "%25", "%", -1), "%", "%25", -1) + h.NumConnections = 1 + if h.HTTPReqTimeOut <= 0 { + h.HTTPReqTimeOut = HTTPReqTimeOutDefaultValue + } + h.ResetHeaders() + h.extraHeaders.Add("User-Agent", userAgent) + h.URLSchemeCheck() + return h +} + +// URLSchemeCheck makes sure the client will work with the scheme requested. +// it also adds missing http:// to emulate curl's behavior. +func (h *HTTPOptions) URLSchemeCheck() { + log.LogVf("URLSchemeCheck %+v", h) + if len(h.URL) == 0 { + log.Errf("unexpected init with empty url") + return + } + hs := "https://" // longer of the 2 prefixes + lcURL := h.URL + if len(lcURL) > len(hs) { + lcURL = strings.ToLower(h.URL[:len(hs)]) // no need to tolower more than we check + } + if strings.HasPrefix(lcURL, hs) { + if !h.DisableFastClient { + log.Warnf("https requested, switching to standard go client") + h.DisableFastClient = true + } + return // url is good + } + if !strings.HasPrefix(lcURL, "http://") { + log.Warnf("assuming http:// on missing scheme for '%s'", h.URL) + h.URL = "http://" + h.URL + } +} + +// Version is the fortio package version (TODO:auto gen/extract). +const ( + userAgent = "istio/fortio-" + periodic.Version + retcodeOffset = len("HTTP/1.X ") + HTTPReqTimeOutDefaultValue = 15 * time.Second +) + +// HTTPOptions holds the common options of both http clients and the headers. +type HTTPOptions struct { + URL string + NumConnections int // num connections (for std client) + Compression bool // defaults to no compression, only used by std client + DisableFastClient bool // defaults to fast client + HTTP10 bool // defaults to http1.1 + DisableKeepAlive bool // so default is keep alive + AllowHalfClose bool // if not keepalive, whether to half close after request + initDone bool + // ExtraHeaders to be added to each request. + extraHeaders http.Header + // Host is treated specially, remember that one separately. + hostOverride string + HTTPReqTimeOut time.Duration // timeout value for http request +} + +// ResetHeaders resets all the headers, including the User-Agent one. +func (h *HTTPOptions) ResetHeaders() { + h.extraHeaders = make(http.Header) + h.hostOverride = "" +} + +// GetHeaders returns the current set of headers. +func (h *HTTPOptions) GetHeaders() http.Header { + if h.hostOverride == "" { + return h.extraHeaders + } + cp := h.extraHeaders + cp.Add("Host", h.hostOverride) + return cp +} + +// AddAndValidateExtraHeader collects extra headers (see main.go for example). +func (h *HTTPOptions) AddAndValidateExtraHeader(hdr string) error { + s := strings.SplitN(hdr, ":", 2) + if len(s) != 2 { + return fmt.Errorf("invalid extra header '%s', expecting Key: Value", hdr) + } + key := strings.TrimSpace(s[0]) + value := strings.TrimSpace(s[1]) + if strings.EqualFold(key, "host") { + log.LogVf("Will be setting special Host header to %s", value) + h.hostOverride = value + } else { + log.LogVf("Setting regular extra header %s: %s", key, value) + h.extraHeaders.Add(key, value) + log.Debugf("headers now %+v", h.extraHeaders) + } + return nil +} + +// newHttpRequest makes a new http GET request for url with User-Agent. +func newHTTPRequest(o *HTTPOptions) *http.Request { + req, err := http.NewRequest("GET", o.URL, nil) + if err != nil { + log.Errf("Unable to make request for %s : %v", o.URL, err) + return nil + } + req.Header = o.extraHeaders + if o.hostOverride != "" { + req.Host = o.hostOverride + } + if !log.LogDebug() { + return req + } + bytes, err := httputil.DumpRequestOut(req, false) + if err != nil { + log.Errf("Unable to dump request %v", err) + } else { + log.Debugf("For URL %s, sending:\n%s", o.URL, bytes) + } + return req +} + +// Client object for making repeated requests of the same URL using the same +// http client (net/http) +type Client struct { + url string + req *http.Request + client *http.Client +} + +// ChangeURL only for standard client, allows fetching a different URL +func (c *Client) ChangeURL(urlStr string) (err error) { + c.url = urlStr + c.req.URL, err = url.Parse(urlStr) + return err +} + +// Fetch fetches the byte and code for pre created client +func (c *Client) Fetch() (int, []byte, int) { + // req can't be null (client itself would be null in that case) + resp, err := c.client.Do(c.req) + if err != nil { + log.Errf("Unable to send request for %s : %v", c.url, err) + return http.StatusBadRequest, []byte(err.Error()), 0 + } + var data []byte + if log.LogDebug() { + if data, err = httputil.DumpResponse(resp, false); err != nil { + log.Errf("Unable to dump response %v", err) + } else { + log.Debugf("For URL %s, received:\n%s", c.url, data) + } + } + data, err = ioutil.ReadAll(resp.Body) + resp.Body.Close() //nolint(errcheck) + if err != nil { + log.Errf("Unable to read response for %s : %v", c.url, err) + code := resp.StatusCode + if code == http.StatusOK { + code = http.StatusNoContent + log.Warnf("Ok code despite read error, switching code to %d", code) + } + return code, data, 0 + } + code := resp.StatusCode + log.Debugf("Got %d : %s for %s - response is %d bytes", code, resp.Status, c.url, len(data)) + return code, data, 0 +} + +// NewClient creates either a standard or fast client (depending on +// the DisableFastClient flag) +func NewClient(o *HTTPOptions) Fetcher { + o.URLSchemeCheck() + if o.DisableFastClient { + return NewStdClient(o) + } + return NewBasicClient(o) +} + +// NewStdClient creates a client object that wraps the net/http standard client. +func NewStdClient(o *HTTPOptions) *Client { + req := newHTTPRequest(o) + if req == nil { + return nil + } + if o.NumConnections < 1 { + o.NumConnections = 1 + } + // 0 timeout for stdclient doesn't mean 0 timeout... so just warn and leave it + if o.HTTPReqTimeOut <= 0 { + log.Warnf("Std call with client timeout %v", o.HTTPReqTimeOut) + } + client := Client{ + o.URL, + req, + &http.Client{ + Timeout: o.HTTPReqTimeOut, + Transport: &http.Transport{ + MaxIdleConns: o.NumConnections, + MaxIdleConnsPerHost: o.NumConnections, + DisableCompression: !o.Compression, + DisableKeepAlives: o.DisableKeepAlive, + Dial: (&net.Dialer{ + Timeout: o.HTTPReqTimeOut, + }).Dial, + TLSHandshakeTimeout: o.HTTPReqTimeOut, + }, + // Lets us see the raw response instead of auto following redirects. + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + } + return &client +} + +// BasicClient is a fast, lockfree single purpose http 1.0/1.1 client. +type BasicClient struct { + buffer []byte + req []byte + dest net.TCPAddr + socket *net.TCPConn + size int + code int + errorCount int + headerLen int + url string + host string + hostname string + port string + http10 bool // http 1.0, simplest: no Host, forced no keepAlive, no parsing + keepAlive bool + parseHeaders bool // don't bother in http/1.0 + halfClose bool // allow/do half close when keepAlive is false + reqTimeout time.Duration +} + +// NewBasicClient makes a basic, efficient http 1.0/1.1 client. +// This function itself doesn't need to be super efficient as it is created at +// the beginning and then reused many times. +func NewBasicClient(o *HTTPOptions) Fetcher { + proto := "1.1" + if o.HTTP10 { + proto = "1.0" + } + // Parse the url, extract components. + url, err := url.Parse(o.URL) + if err != nil { + log.Errf("Bad url '%s' : %v", o.URL, err) + return nil + } + if url.Scheme != "http" { + log.Errf("Only http is supported with the optimized client, use -stdclient for url %s", o.URL) + return nil + } + // note: Host includes the port + bc := BasicClient{url: o.URL, host: url.Host, hostname: url.Hostname(), port: url.Port(), + http10: o.HTTP10, halfClose: o.AllowHalfClose} + bc.buffer = make([]byte, BufferSizeKb*1024) + if bc.port == "" { + bc.port = url.Scheme // ie http which turns into 80 later + log.LogVf("No port specified, using %s", bc.port) + } + addrs, err := net.LookupIP(bc.hostname) + if err != nil { + log.Errf("Unable to lookup '%s' : %v", bc.host, err) + return nil + } + if len(addrs) > 1 && log.LogDebug() { + log.Debugf("Using only the first of the addresses for %s : %v", bc.host, addrs) + } + log.Debugf("Will go to %s", addrs[0]) + bc.dest.IP = addrs[0] + bc.dest.Port, err = net.LookupPort("tcp", bc.port) + if err != nil { + log.Errf("Unable to resolve port '%s' : %v", bc.port, err) + return nil + } + // Create the bytes for the request: + host := bc.host + if o.hostOverride != "" { + host = o.hostOverride + } + var buf bytes.Buffer + buf.WriteString("GET " + url.RequestURI() + " HTTP/" + proto + "\r\n") + if !bc.http10 { + buf.WriteString("Host: " + host + "\r\n") + bc.parseHeaders = true + if !o.DisableKeepAlive { + bc.keepAlive = true + } else { + buf.WriteString("Connection: close\r\n") + } + } + if o.HTTPReqTimeOut <= 0 { + log.Warnf("Invalid timeout %v, setting to %v", o.HTTPReqTimeOut, HTTPReqTimeOutDefaultValue) + o.HTTPReqTimeOut = HTTPReqTimeOutDefaultValue + } + bc.reqTimeout = o.HTTPReqTimeOut + w := bufio.NewWriter(&buf) + // This writes multiple valued headers properly (unlike calling Get() to do it ourselves) + o.extraHeaders.Write(w) // nolint: errcheck,gas + w.Flush() // nolint: errcheck,gas + buf.WriteString("\r\n") + bc.req = buf.Bytes() + log.Debugf("Created client:\n%+v\n%s", bc.dest, bc.req) + return &bc +} + +// Used for the fast case insensitive search +const toUpperMask = ^byte('a' - 'A') + +// Slow but correct version +func toUpper(b byte) byte { + if b >= 'a' && b <= 'z' { + b -= ('a' - 'A') + } + return b +} + +// ASCIIToUpper returns a byte array equal to the input string but in lowercase. +// Only wotks for ASCII, not meant for unicode. +func ASCIIToUpper(str string) []byte { + numChars := utf8.RuneCountInString(str) + if numChars != len(str) && log.LogVerbose() { + log.Errf("ASCIIFold(\"%s\") contains %d characters, some non ascii (byte length %d): will mangle", str, numChars, len(str)) + } + res := make([]byte, numChars) + // less surprising if we only mangle the extended characters + i := 0 + for _, c := range str { // Attention: _ here != i for unicode characters + res[i] = toUpper(byte(c)) + i++ + } + return res +} + +// FoldFind searches the bytes assuming ascii, ignoring the lowercase bit +// for testing. Not intended to work with unicode, meant for http headers +// and to be fast (see benchmark in test file). +func FoldFind(haystack []byte, needle []byte) (bool, int) { + idx := 0 + found := false + hackstackLen := len(haystack) + needleLen := len(needle) + if needleLen == 0 { + return true, 0 + } + if needleLen > hackstackLen { // those 2 ifs also handles haystackLen == 0 + return false, -1 + } + needleOffset := 0 + for { + h := haystack[idx] + n := needle[needleOffset] + // This line is quite performance sensitive. calling toUpper() for instance + // is a 30% hit, even if called only on the haystack. The XOR lets us be + // true for equality and the & with mask also true if the only difference + // between the 2 is the case bit. + xor := h ^ n // == 0 if strictly equal + if (xor&toUpperMask) != 0 || (((h < 32) || (n < 32)) && (xor != 0)) { + idx -= (needleOffset - 1) // does ++ most of the time + needleOffset = 0 + if idx >= hackstackLen { + break + } + continue + } + if needleOffset == needleLen-1 { + found = true + break + } + needleOffset++ + idx++ + if idx >= hackstackLen { + break + } + } + if !found { + return false, -1 + } + return true, idx - needleOffset +} + +// ParseDecimal extracts the first positive integer number from the input. +// spaces are ignored. +// any character that isn't a digit cause the parsing to stop +func ParseDecimal(inp []byte) int { + res := -1 + for _, b := range inp { + if b == ' ' && res == -1 { + continue + } + if b < '0' || b > '9' { + break + } + digit := int(b - '0') + if res == -1 { + res = digit + } else { + res = 10*res + digit + } + } + return res +} + +// ParseChunkSize extracts the chunk size and consumes the line. +// Returns the offset of the data and the size of the chunk, +// 0, -1 when not found. +func ParseChunkSize(inp []byte) (int, int) { + if log.LogDebug() { + log.Debugf("ParseChunkSize(%s)", DebugSummary(inp, 128)) + } + res := -1 + off := 0 + end := len(inp) + inDigits := true + for { + if off >= end { + return off, -1 + } + if inDigits { + b := toUpper(inp[off]) + var digit int + if b >= 'A' && b <= 'F' { + digit = 10 + int(b-'A') + } else if b >= '0' && b <= '9' { + digit = int(b - '0') + } else { + inDigits = false + if res == -1 { + log.Errf("Didn't find hex number %q", inp) + return off, res + } + continue + } + if res == -1 { + res = digit + } else { + res = 16*res + digit + } + } else { + // After digits, skipping ahead to find \r\n + if inp[off] == '\r' { + off++ + if off >= end { + return off, -1 + } + if inp[off] == '\n' { + // good case + return off + 1, res + } + } + } + off++ + } +} + +// return the result from the state. +func (c *BasicClient) returnRes() (int, []byte, int) { + return c.code, c.buffer[:c.size], c.headerLen +} + +// connect to destination. +func (c *BasicClient) connect() *net.TCPConn { + socket, err := net.DialTCP("tcp", nil, &c.dest) + if err != nil { + log.Errf("Unable to connect to %v : %v", c.dest, err) + return nil + } + // For now those errors are not critical/breaking + if err = socket.SetNoDelay(true); err != nil { + log.Warnf("Unable to connect to set tcp no delay %v %v : %v", socket, c.dest, err) + } + if err = socket.SetWriteBuffer(len(c.req)); err != nil { + log.Warnf("Unable to connect to set write buffer %d %v %v : %v", len(c.req), socket, c.dest, err) + } + if err = socket.SetReadBuffer(len(c.buffer)); err != nil { + log.Warnf("Unable to connect to read buffer %d %v %v : %v", len(c.buffer), socket, c.dest, err) + } + return socket +} + +// Fetch fetches the url content. Returns http code, data, offset of body. +func (c *BasicClient) Fetch() (int, []byte, int) { + c.code = -1 + c.size = 0 + c.headerLen = 0 + // Connect or reuse existing socket: + conn := c.socket + reuse := (conn != nil) + if !reuse { + conn = c.connect() + if conn == nil { + return c.returnRes() + } + } else { + log.Debugf("Reusing socket %v", *conn) + } + c.socket = nil // because of error returns + conErr := conn.SetReadDeadline(time.Now().Add(c.reqTimeout)) + // Send the request: + n, err := conn.Write(c.req) + if err != nil || conErr != nil { + if reuse { + // it's ok for the (idle) socket to die once, auto reconnect: + log.Infof("Closing dead socket %v (%v)", *conn, err) + conn.Close() // nolint: errcheck,gas + c.errorCount++ + return c.Fetch() // recurse once + } + log.Errf("Unable to write to %v %v : %v", conn, c.dest, err) + return c.returnRes() + } + if n != len(c.req) { + log.Errf("Short write to %v %v : %d instead of %d", conn, c.dest, n, len(c.req)) + return c.returnRes() + } + if !c.keepAlive && c.halfClose { + if err = conn.CloseWrite(); err != nil { + log.Errf("Unable to close write to %v %v : %v", conn, c.dest, err) + return c.returnRes() + } // else: + log.Debugf("Half closed ok after sending request %v %v", conn, c.dest) + } + // Read the response: + c.readResponse(conn) + // Return the result: + return c.returnRes() +} + +// EscapeBytes returns printable string. Same as %q format without the +// surrounding/extra "". +func EscapeBytes(buf []byte) string { + e := fmt.Sprintf("%q", buf) + return e[1 : len(e)-1] +} + +// DebugSummary returns a string with the size and escaped first max/2 and +// last max/2 bytes of a buffer (or the whole escaped buffer if small enough). +func DebugSummary(buf []byte, max int) string { + l := len(buf) + if l <= max+3 { //no point in shortening to add ... if we could return those 3 + return EscapeBytes(buf) + } + max /= 2 + return fmt.Sprintf("%d: %s...%s", l, EscapeBytes(buf[:max]), EscapeBytes(buf[l-max:])) +} + +// Response reading: +// TODO: refactor - unwiedly/ugly atm +func (c *BasicClient) readResponse(conn *net.TCPConn) { + max := len(c.buffer) + parsedHeaders := false + // TODO: safer to start with -1 and fix ok for http 1.0 + c.code = http.StatusOK // In http 1.0 mode we don't bother parsing anything + endofHeadersStart := retcodeOffset + 3 + keepAlive := c.keepAlive + chunkedMode := false + checkConnectionClosedHeader := CheckConnectionClosedHeader + skipRead := false + for { + // Ugly way to cover the case where we get more than 1 chunk at the end + // TODO: need automated tests + if !skipRead { + n, err := conn.Read(c.buffer[c.size:]) + if err == io.EOF { + if c.size == 0 { + log.Errf("EOF before reading anything on %v %v", conn, c.dest) + c.code = -1 + } + break + } + if err != nil { + log.Errf("Read error %v %v %d : %v", conn, c.dest, c.size, err) + c.code = -1 + break + } + c.size += n + if log.LogDebug() { + log.Debugf("Read ok %d total %d so far (-%d headers = %d data) %s", + n, c.size, c.headerLen, c.size-c.headerLen, DebugSummary(c.buffer[c.size-n:c.size], 256)) + } + } + skipRead = false + // Have not yet parsed the headers, need to parse the headers, and have enough data to + // at least parse the http retcode: + if !parsedHeaders && c.parseHeaders && c.size >= retcodeOffset+3 { + // even if the bytes are garbage we'll get a non 200 code (bytes are unsigned) + c.code = ParseDecimal(c.buffer[retcodeOffset : retcodeOffset+3]) //TODO do that only once... + // TODO handle 100 Continue + if c.code != http.StatusOK { + log.Warnf("Parsed non ok code %d (%v)", c.code, string(c.buffer[:retcodeOffset+3])) + break + } + if log.LogDebug() { + log.Debugf("Code %d, looking for end of headers at %d / %d, last CRLF %d", + c.code, endofHeadersStart, c.size, c.headerLen) + } + // TODO: keep track of list of newlines to efficiently search headers only there + idx := endofHeadersStart + for idx < c.size-1 { + if c.buffer[idx] == '\r' && c.buffer[idx+1] == '\n' { + if c.headerLen == idx-2 { // found end of headers + parsedHeaders = true + break + } + c.headerLen = idx + idx++ + } + idx++ + } + endofHeadersStart = c.size // start there next read + if parsedHeaders { + // We have headers ! + c.headerLen += 4 // we use this and not endofHeadersStart so http/1.0 does return 0 and not the optimization for search start + if log.LogDebug() { + log.Debugf("headers are %d: %s", c.headerLen, c.buffer[:idx]) + } + // Find the content length or chunked mode + if keepAlive { + var contentLength int + found, offset := FoldFind(c.buffer[:c.headerLen], contentLengthHeader) + if found { + // Content-Length mode: + contentLength = ParseDecimal(c.buffer[offset+len(contentLengthHeader) : c.headerLen]) + if contentLength < 0 { + log.Warnf("Warning: content-length unparsable %s", string(c.buffer[offset+2:offset+len(contentLengthHeader)+4])) + keepAlive = false + break + } + max = c.headerLen + contentLength + if log.LogDebug() { // somehow without the if we spend 400ms/10s in LogV (!) + log.Debugf("found content length %d", contentLength) + } + } else { + // Chunked mode (or err/missing): + if found, _ := FoldFind(c.buffer[:c.headerLen], chunkedHeader); found { + chunkedMode = true + var dataStart int + dataStart, contentLength = ParseChunkSize(c.buffer[c.headerLen:c.size]) + max = c.headerLen + dataStart + contentLength + 2 // extra CR LF + log.Debugf("chunk-length is %d (%s) setting max to %d", + contentLength, c.buffer[c.headerLen:c.headerLen+dataStart-2], + max) + } else { + if log.LogVerbose() { + log.LogVf("Warning: content-length missing in %s", string(c.buffer[:c.headerLen])) + } else { + log.Warnf("Warning: content-length missing (%d bytes headers)", c.headerLen) + } + keepAlive = false // can't keep keepAlive + break + } + } // end of content-length section + if max > len(c.buffer) { + log.Warnf("Buffer is too small for headers %d + data %d - change -httpbufferkb flag to at least %d", + c.headerLen, contentLength, (c.headerLen+contentLength)/1024+1) + // TODO: just consume the extra instead + max = len(c.buffer) + } + if checkConnectionClosedHeader { + if found, _ := FoldFind(c.buffer[:c.headerLen], connectionCloseHeader); found { + log.Infof("Server wants to close connection, no keep-alive!") + keepAlive = false + max = len(c.buffer) // reset to read as much as available + } + } + } + } + } // end of big if parse header + if c.size >= max { + if !keepAlive { + log.Errf("More data is available but stopping after %d, increase -httpbufferkb", max) + } + if !parsedHeaders && c.parseHeaders { + log.Errf("Buffer too small (%d) to even finish reading headers, increase -httpbufferkb to get all the data", max) + keepAlive = false + } + if chunkedMode { + // Next chunk: + dataStart, nextChunkLen := ParseChunkSize(c.buffer[max:c.size]) + if nextChunkLen == -1 { + if c.size == max { + log.Debugf("Couldn't find next chunk size, reading more %d %d", max, c.size) + } else { + log.Infof("Partial chunk size (%s), reading more %d %d", DebugSummary(c.buffer[max:c.size], 20), max, c.size) + } + continue + } else if nextChunkLen == 0 { + log.Debugf("Found last chunk %d %d", max+dataStart, c.size) + if c.size != max+dataStart+2 || string(c.buffer[c.size-2:c.size]) != "\r\n" { + log.Errf("Unexpected mismatch at the end sz=%d expected %d; end of buffer %q", c.size, max+dataStart+2, c.buffer[max:c.size]) + } + } else { + max += dataStart + nextChunkLen + 2 // extra CR LF + log.Debugf("One more chunk %d -> new max %d", nextChunkLen, max) + if max > len(c.buffer) { + log.Errf("Buffer too small for %d data", max) + } else { + if max <= c.size { + log.Debugf("Enough data to reach next chunk, skipping a read") + skipRead = true + } + continue + } + } + } + break // we're done! + } + } // end of big for loop + // Figure out whether to keep or close the socket: + if keepAlive && c.code == http.StatusOK { + c.socket = conn // keep the open socket + } else { + if err := conn.Close(); err != nil { + log.Errf("Close error %v %v %d : %v", conn, c.dest, c.size, err) + } else { + log.Debugf("Closed ok %v from %v after reading %d bytes", conn, c.dest, c.size) + } + // we cleared c.socket in caller already + } +} + +// -- Echo Server -- + +var ( + // EchoRequests is the number of request received. Only updated in Debug mode. + EchoRequests int64 +) + +func removeTrailingPercent(s string) string { + if strings.HasSuffix(s, "%") { + return s[:len(s)-1] + } + return s +} + +// generateStatus from string, format: status="503" for 100% 503s +// status="503:20,404:10,403:0.5" for 20% 503s, 10% 404s, 0.5% 403s 69.5% 200s +func generateStatus(status string) int { + lst := strings.Split(status, ",") + log.Debugf("Parsing status %s -> %v", status, lst) + // Simple non probabilistic status case: + if len(lst) == 1 && !strings.ContainsRune(status, ':') { + s, err := strconv.Atoi(status) + if err != nil { + log.Warnf("Bad input status %v, not a number nor comma and colon separated %% list", status) + return http.StatusBadRequest + } + log.Debugf("Parsed status %s -> %d", status, s) + return s + } + weights := make([]float32, len(lst)) + codes := make([]int, len(lst)) + lastPercent := float64(0) + i := 0 + for _, entry := range lst { + l2 := strings.Split(entry, ":") + if len(l2) != 2 { + log.Warnf("Should have exactly 1 : in status list %s -> %v", status, entry) + return http.StatusBadRequest + } + s, err := strconv.Atoi(l2[0]) + if err != nil { + log.Warnf("Bad input status %v -> %v, not a number before colon", status, l2[0]) + return http.StatusBadRequest + } + percStr := removeTrailingPercent(l2[1]) + p, err := strconv.ParseFloat(percStr, 32) + if err != nil || p < 0 || p > 100 { + log.Warnf("Percentage is not a [0. - 100.] number in %v -> %v : %v %f", status, percStr, err, p) + return http.StatusBadRequest + } + lastPercent += p + // Round() needed to cover 'exactly' 100% and not more or less because of rounding errors + p32 := float32(stats.Round(lastPercent)) + if p32 > 100. { + log.Warnf("Sum of percentage is greater than 100 in %v %f %f %f", status, lastPercent, p, p32) + return http.StatusBadRequest + } + weights[i] = p32 + codes[i] = s + i++ + } + res := 100. * rand.Float32() + for i, v := range weights { + if res <= v { + log.Debugf("[0.-100.[ for %s roll %f got #%d -> %d", status, res, i, codes[i]) + return codes[i] + } + } + log.Debugf("[0.-100.[ for %s roll %f no hit, defaulting to OK", status, res) + return http.StatusOK // default/reminder of probability table +} + +// MaxDelay is the maximum delay allowed for the echoserver responses. +const MaxDelay = 1 * time.Second + +// generateDelay from string, format: delay="100ms" for 100% 100ms delay +// delay="10ms:20,20ms:10,1s:0.5" for 20% 10ms, 10% 20ms, 0.5% 1s and 69.5% 0 +// TODO: very similar with generateStatus - refactor? +func generateDelay(delay string) time.Duration { + lst := strings.Split(delay, ",") + log.Debugf("Parsing delay %s -> %v", delay, lst) + if len(delay) == 0 { + return -1 + } + // Simple non probabilistic status case: + if len(lst) == 1 && !strings.ContainsRune(delay, ':') { + d, err := time.ParseDuration(delay) + if err != nil { + log.Warnf("Bad input delay %v, not a duration nor comma and colon separated %% list", delay) + return -1 + } + log.Debugf("Parsed delay %s -> %d", delay, d) + if d > MaxDelay { + d = MaxDelay + } + return d + } + weights := make([]float32, len(lst)) + delays := make([]time.Duration, len(lst)) + lastPercent := float64(0) + i := 0 + for _, entry := range lst { + l2 := strings.Split(entry, ":") + if len(l2) != 2 { + log.Warnf("Should have exactly 1 : in delay list %s -> %v", delay, entry) + return -1 + } + d, err := time.ParseDuration(l2[0]) + if err != nil { + log.Warnf("Bad input delay %v -> %v, not a number before colon", delay, l2[0]) + return -1 + } + if d > MaxDelay { + d = MaxDelay + } + percStr := removeTrailingPercent(l2[1]) + p, err := strconv.ParseFloat(percStr, 32) + if err != nil || p < 0 || p > 100 { + log.Warnf("Percentage is not a [0. - 100.] number in %v -> %v : %v %f", delay, percStr, err, p) + return -1 + } + lastPercent += p + // Round() needed to cover 'exactly' 100% and not more or less because of rounding errors + p32 := float32(stats.Round(lastPercent)) + if p32 > 100. { + log.Warnf("Sum of percentage is greater than 100 in %v %f %f %f", delay, lastPercent, p, p32) + return -1 + } + weights[i] = p32 + delays[i] = d + i++ + } + res := 100. * rand.Float32() + for i, v := range weights { + if res <= v { + log.Debugf("[0.-100.[ for %s roll %f got #%d -> %d", delay, res, i, delays[i]) + return delays[i] + } + } + log.Debugf("[0.-100.[ for %s roll %f no hit, defaulting to 0", delay, res) + return 0 +} + +// EchoHandler is an http server handler echoing back the input. +func EchoHandler(w http.ResponseWriter, r *http.Request) { + log.LogVf("%v %v %v %v", r.Method, r.URL, r.Proto, r.RemoteAddr) + data, err := ioutil.ReadAll(r.Body) // must be done before calling FormValue + if err != nil { + log.Errf("Error reading %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + log.Debugf("Read %d", len(data)) + dur := generateDelay(r.FormValue("delay")) + if dur > 0 { + log.LogVf("Sleeping for %v", dur) + time.Sleep(dur) + } + statusStr := r.FormValue("status") + var status int + if statusStr != "" { + status = generateStatus(statusStr) + } else { + status = http.StatusOK + } + if log.LogDebug() { + for name, headers := range r.Header { + for _, h := range headers { + log.Debugf("Header %v: %v\n", name, h) + } + } + } + // echo back the Content-Type and Content-Length in the response + for _, k := range []string{"Content-Type", "Content-Length"} { + if v := r.Header.Get(k); v != "" { + w.Header().Set(k, v) + } + } + w.WriteHeader(status) + if _, err = w.Write(data); err != nil { + log.Errf("Error writing response %v to %v", err, r.RemoteAddr) + } + if log.LogDebug() { + // TODO: this easily lead to contention - use 'thread local' + rqNum := atomic.AddInt64(&EchoRequests, 1) + log.Debugf("Requests: %v", rqNum) + } +} + +func closingServer(listener net.Listener) error { + var err error + for { + var c net.Conn + c, err = listener.Accept() + if err != nil { + log.Errf("Accept error in dummy server %v", err) + break + } + log.LogVf("Got connection from %v, closing", c.RemoteAddr()) + err = c.Close() + if err != nil { + log.Errf("Close error in dummy server %v", err) + break + } + } + return err +} + +// DynamicHTTPServer listens on an available port, sets up an http or https +// (when secure is true) server on it and returns the listening port and +// mux to which one can attach handlers to. +func DynamicHTTPServer(secure bool) (int, *http.ServeMux) { + m := http.NewServeMux() + s := &http.Server{ + Handler: m, + } + listener, err := net.Listen("tcp", ":0") // nolint: gas + if err != nil { + log.Fatalf("Unable to listen to dynamic port: %v", err) + } + port := listener.Addr().(*net.TCPAddr).Port + log.Infof("Using port: %d", port) + go func() { + var err error + if secure { + log.Errf("Secure setup not yet supported. Will just close incoming connections for now") + //err = http.ServeTLS(listener, nil, "", "") // go 1.9 + err = closingServer(listener) + } else { + err = s.Serve(listener) + } + if err != nil { + log.Fatalf("Unable to serve with secure=%v on %d: %v", secure, port, err) + } + }() + return port, m +} + +/* +// DebugHandlerTemplate returns debug/useful info on the http requet. +// slower heavier but nicer source code version of DebugHandler +func DebugHandlerTemplate(w http.ResponseWriter, r *http.Request) { + log.LogVf("%v %v %v %v", r.Method, r.URL, r.Proto, r.RemoteAddr) + hostname, _ := os.Hostname() + data, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errf("Error reading %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Note: this looks nicer but is about 2x slower / less qps / more cpu and 25% bigger executable than doing the writes oneself: + const templ = `Φορτίο version {{.Version}} echo debug server on {{.Hostname}} - request from {{.R.RemoteAddr}} + +{{.R.Method}} {{.R.URL}} {{.R.Proto}} + +headers: + +{{ range $name, $vals := .R.Header }}{{range $val := $vals}}{{$name}}: {{ $val }} +{{end}}{{end}} +body: + +{{.Body}} +{{if .DumpEnv}} +environment: +{{ range $idx, $e := .Env }} +{{$e}}{{end}} +{{end}}` + t := template.Must(template.New("debugOutput").Parse(templ)) + err = t.Execute(w, &struct { + R *http.Request + Hostname string + Version string + Body string + DumpEnv bool + Env []string + }{r, hostname, Version, DebugSummary(data, 512), r.FormValue("env") == "dump", os.Environ()}) + if err != nil { + Critf("Template execution failed: %v", err) + } + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") +} +*/ + +// RoundDuration rounds to 10th of second. Only for positive durations. +// TODO: switch to Duration.Round once switched to go 1.9 +func RoundDuration(d time.Duration) time.Duration { + tenthSec := int64(100 * time.Millisecond) + r := int64(d+50*time.Millisecond) / tenthSec + return time.Duration(tenthSec * r) +} + +// DebugHandler returns debug/useful info to http client. +func DebugHandler(w http.ResponseWriter, r *http.Request) { + log.LogVf("%v %v %v %v", r.Method, r.URL, r.Proto, r.RemoteAddr) + var buf bytes.Buffer + buf.WriteString("Φορτίο version ") + buf.WriteString(periodic.Version) + buf.WriteString(" echo debug server up for ") + buf.WriteString(fmt.Sprint(RoundDuration(time.Since(startTime)))) + buf.WriteString(" on ") + hostname, _ := os.Hostname() // nolint: gas + buf.WriteString(hostname) + buf.WriteString(" - request from ") + buf.WriteString(r.RemoteAddr) + buf.WriteString("\n\n") + buf.WriteString(r.Method) + buf.WriteByte(' ') + buf.WriteString(r.URL.String()) + buf.WriteByte(' ') + buf.WriteString(r.Proto) + buf.WriteString("\n\nheaders:\n\n") + // Host is removed from headers map and put here (!) + buf.WriteString("Host: ") + buf.WriteString(r.Host) + for name, headers := range r.Header { + buf.WriteByte('\n') + buf.WriteString(name) + buf.WriteString(": ") + first := true + for _, h := range headers { + if !first { + buf.WriteByte(',') + } + buf.WriteString(h) + first = false + } + } + data, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errf("Error reading %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + buf.WriteString("\n\nbody:\n\n") + buf.WriteString(DebugSummary(data, 512)) + buf.WriteByte('\n') + if r.FormValue("env") == "dump" { + buf.WriteString("\nenvironment:\n\n") + for _, v := range os.Environ() { + buf.WriteString(v) + buf.WriteByte('\n') + } + } + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") + if _, err = w.Write(buf.Bytes()); err != nil { + log.Errf("Error writing response %v to %v", err, r.RemoteAddr) + } +} + +// Serve starts a debug / echo http server on the given port. +// TODO: make it work for port 0 and return the port found and also +// add a non blocking mode that makes sure the socket exists before returning +func Serve(port, debugPath string) { + startTime = time.Now() + nPort := fnet.NormalizePort(port) + fmt.Printf("Fortio %s echo server listening on port %s\n", periodic.Version, nPort) + if debugPath != "" { + http.HandleFunc(debugPath, DebugHandler) + } + http.HandleFunc("/", EchoHandler) + if err := http.ListenAndServe(nPort, nil); err != nil { + fmt.Println("Error starting server", err) + } +} diff --git a/vendor/fortio.org/fortio/fhttp/httprunner.go b/vendor/istio.io/fortio/fhttp/httprunner.go similarity index 79% rename from vendor/fortio.org/fortio/fhttp/httprunner.go rename to vendor/istio.io/fortio/fhttp/httprunner.go index 5e15671128..fc85db7e5f 100644 --- a/vendor/fortio.org/fortio/fhttp/httprunner.go +++ b/vendor/istio.io/fortio/fhttp/httprunner.go @@ -22,9 +22,9 @@ import ( "runtime/pprof" "sort" - "fortio.org/fortio/log" - "fortio.org/fortio/periodic" - "fortio.org/fortio/stats" + "istio.io/fortio/log" + "istio.io/fortio/periodic" + "istio.io/fortio/stats" ) // Most of the code in this file is the library-fication of code originally @@ -43,10 +43,6 @@ type HTTPRunnerResults struct { Sizes *stats.HistogramData HeaderSizes *stats.HistogramData URL string - SocketCount int - // http code to abort the run on (-1 for connection or other socket error) - AbortOn int - aborter *periodic.Aborter } // Run tests http request fetching. Main call being run at the target QPS. @@ -55,14 +51,10 @@ func (httpstate *HTTPRunnerResults) Run(t int) { log.Debugf("Calling in %d", t) code, body, headerSize := httpstate.client.Fetch() size := len(body) - log.Debugf("Got in %3d hsz %d sz %d - will abort on %d", code, headerSize, size, httpstate.AbortOn) + log.Debugf("Got in %3d hsz %d sz %d", code, headerSize, size) httpstate.RetCodes[code]++ httpstate.sizes.Record(float64(size)) httpstate.headerSizes.Record(float64(headerSize)) - if httpstate.AbortOn == code { - httpstate.aborter.Abort() - log.Infof("Aborted run because of code %d - data %s", code, DebugSummary(body, 1024)) - } } // HTTPRunnerOptions includes the base RunnerOptions plus http specific @@ -72,13 +64,11 @@ type HTTPRunnerOptions struct { HTTPOptions // Need to call Init() to initialize Profiler string // file to save profiles to. defaults to no profiling AllowInitialErrors bool // whether initial errors don't cause an abort - // Which status code cause an abort of the run (default 0 = don't abort; reminder -1 is returned for socket errors) - AbortOn int } // RunHTTPTest runs an http test and returns the aggregated stats. func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) { - o.RunType = "HTTP" + // TODO 1. use std client automatically when https url log.Infof("Starting http test for %s with %d threads at %.1f qps", o.URL, o.NumThreads, o.QPS) r := periodic.NewPeriodicRunner(&o.RunnerOptions) defer r.Options().Abort() @@ -90,8 +80,6 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) { sizes: stats.NewHistogram(0, 100), headerSizes: stats.NewHistogram(0, 5), URL: o.URL, - AbortOn: o.AbortOn, - aborter: r.Options().Stop, } httpstate := make([]HTTPRunnerResults, numThreads) for i := 0; i < numThreads; i++ { @@ -114,8 +102,6 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) { httpstate[i].sizes = total.sizes.Clone() httpstate[i].headerSizes = total.headerSizes.Clone() httpstate[i].RetCodes = make(map[int]int64) - httpstate[i].AbortOn = total.AbortOn - httpstate[i].aborter = total.aborter } if o.Profiler != "" { @@ -137,13 +123,12 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) { runtime.GC() // get up-to-date statistics pprof.WriteHeapProfile(fm) // nolint:gas,errcheck fm.Close() // nolint:gas,errcheck - _, _ = fmt.Fprintf(out, "Wrote profile data to %s.{cpu|mem}\n", o.Profiler) + fmt.Fprintf(out, "Wrote profile data to %s.{cpu|mem}\n", o.Profiler) } - // Numthreads may have reduced but it should be ok to accumulate 0s from - // unused ones. We also must cleanup all the created clients. + // Numthreads may have reduced + numThreads = r.Options().NumThreads keys := []int{} for i := 0; i < numThreads; i++ { - total.SocketCount += httpstate[i].client.Close() // Q: is there some copying each time stats[i] is used? for k := range httpstate[i].RetCodes { if _, exists := total.RetCodes[k]; !exists { @@ -154,13 +139,10 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) { total.sizes.Transfer(httpstate[i].sizes) total.headerSizes.Transfer(httpstate[i].headerSizes) } - // Cleanup state: - r.Options().ReleaseRunners() sort.Ints(keys) totalCount := float64(total.DurationHistogram.Count) - _, _ = fmt.Fprintf(out, "Sockets used: %d (for perfect keepalive, would be %d)\n", total.SocketCount, r.Options().NumThreads) for _, k := range keys { - _, _ = fmt.Fprintf(out, "Code %3d : %d (%.1f %%)\n", k, total.RetCodes[k], 100.*float64(total.RetCodes[k])/totalCount) + fmt.Fprintf(out, "Code %3d : %d (%.1f %%)\n", k, total.RetCodes[k], 100.*float64(total.RetCodes[k])/totalCount) } total.HeaderSizes = total.headerSizes.Export() total.Sizes = total.sizes.Export() diff --git a/vendor/istio.io/fortio/fnet/network.go b/vendor/istio.io/fortio/fnet/network.go new file mode 100644 index 0000000000..5d101e8f1c --- /dev/null +++ b/vendor/istio.io/fortio/fnet/network.go @@ -0,0 +1,28 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fnet // import "istio.io/fortio/fnet" + +import ( + "strings" +) + +// NormalizePort parses port and returns host:port if port is in the form +// of host:port or :port if port is in the form of port. +func NormalizePort(port string) string { + if strings.ContainsAny(port, ":") { + return port + } + return ":" + port +} diff --git a/vendor/fortio.org/fortio/log/logger.go b/vendor/istio.io/fortio/log/logger.go similarity index 99% rename from vendor/fortio.org/fortio/log/logger.go rename to vendor/istio.io/fortio/log/logger.go index c0844524b9..b42c369234 100644 --- a/vendor/fortio.org/fortio/log/logger.go +++ b/vendor/istio.io/fortio/log/logger.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package log // import "fortio.org/fortio/log" +package log // import "istio.io/fortio/log" import ( "flag" diff --git a/vendor/fortio.org/fortio/periodic/periodic.go b/vendor/istio.io/fortio/periodic/periodic.go similarity index 90% rename from vendor/fortio.org/fortio/periodic/periodic.go rename to vendor/istio.io/fortio/periodic/periodic.go index 4c55b1f525..ddf247d581 100644 --- a/vendor/fortio.org/fortio/periodic/periodic.go +++ b/vendor/istio.io/fortio/periodic/periodic.go @@ -20,7 +20,7 @@ // is also ../histogram to use the stats from the command line and ../echosrv // as a very light http server that can be used to test proxies etc like // the Istio components. -package periodic // import "fortio.org/fortio/periodic" +package periodic // import "istio.io/fortio/periodic" import ( "fmt" @@ -31,9 +31,13 @@ import ( "sync" "time" - "fortio.org/fortio/log" - "fortio.org/fortio/stats" - "fortio.org/fortio/version" + "istio.io/fortio/log" + "istio.io/fortio/stats" +) + +const ( + // Version is the overall package version (used to version json output too). + Version = "0.6.8" ) // DefaultRunnerOptions are the default values for options (do not mutate!). @@ -66,13 +70,6 @@ func (r *RunnerOptions) MakeRunners(rr Runnable) { } } -// ReleaseRunners clear the runners state. -func (r *RunnerOptions) ReleaseRunners() { - for idx := range r.Runners { - r.Runners[idx] = nil - } -} - // Aborter is the object controlling Abort() of the runs. type Aborter struct { sync.Mutex @@ -100,8 +97,6 @@ func NewAborter() *Aborter { // RunnerOptions are the parameters to the PeriodicRunner. type RunnerOptions struct { - // Type of run (to be copied into results) - RunType string // Array of objects to run in each thread (use MakeRunners() to clone the same one) Runners []Runnable // At which (target) rate to run the Runners across NumThreads. @@ -130,7 +125,6 @@ type RunnerOptions struct { // RunnerResults encapsulates the actual QPS observed and duration histogram. type RunnerResults struct { - RunType string Labels string StartTime time.Time RequestedQPS string @@ -266,18 +260,14 @@ func (r *RunnerOptions) Abort() { } } -// internal version, returning the concrete implementation. logical std::move +// internal version, returning the concrete implementation. func newPeriodicRunner(opts *RunnerOptions) *periodicRunner { r := &periodicRunner{*opts} // by default just copy the input params - opts.ReleaseRunners() - opts.Stop = nil r.Normalize() return r } // NewPeriodicRunner constructs a runner from input parameters/options. -// The options will be moved and normalized to the returned object, do -// not use the original options after this call, call Options() instead. // Abort() must be called if Run() is not called. func NewPeriodicRunner(params *RunnerOptions) PeriodicRunner { return newPeriodicRunner(params) @@ -327,20 +317,20 @@ func (r *periodicRunner) Run() RunnerResults { leftOver = r.Exactly - totalCalls if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] : exactly %d, %d calls each (total %d + %d)\n", + fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] : exactly %d, %d calls each (total %d + %d)\n", r.QPS, r.NumThreads, runtime.GOMAXPROCS(0), r.Exactly, numCalls, totalCalls, leftOver) } } else { if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] for %v : %d calls each (total %d)\n", + fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] for %v : %d calls each (total %d)\n", r.QPS, r.NumThreads, runtime.GOMAXPROCS(0), r.Duration, numCalls, totalCalls) } } } else { // Always print that as we need ^C to interrupt, in that case the user need to notice // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] until interrupted\n", + fmt.Fprintf(r.Out, "Starting at %g qps with %d thread(s) [gomax %d] until interrupted\n", r.QPS, r.NumThreads, runtime.GOMAXPROCS(0)) numCalls = 0 } @@ -348,12 +338,12 @@ func (r *periodicRunner) Run() RunnerResults { if !useExactly && !hasDuration { // Always log something when waiting for ^C // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Starting at max qps with %d thread(s) [gomax %d] until interrupted\n", + fmt.Fprintf(r.Out, "Starting at max qps with %d thread(s) [gomax %d] until interrupted\n", r.NumThreads, runtime.GOMAXPROCS(0)) } else { if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Starting at max qps with %d thread(s) [gomax %d] ", + fmt.Fprintf(r.Out, "Starting at max qps with %d thread(s) [gomax %d] ", r.NumThreads, runtime.GOMAXPROCS(0)) } if useExactly { @@ -362,13 +352,13 @@ func (r *periodicRunner) Run() RunnerResults { leftOver = r.Exactly % int64(r.NumThreads) if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "for %s (%d per thread + %d)\n", requestedDuration, numCalls, leftOver) + fmt.Fprintf(r.Out, "for %s (%d per thread + %d)\n", requestedDuration, numCalls, leftOver) } } else { requestedDuration = fmt.Sprint(r.Duration) if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "for %s\n", requestedDuration) + fmt.Fprintf(r.Out, "for %s\n", requestedDuration) } } } @@ -419,7 +409,7 @@ func (r *periodicRunner) Run() RunnerResults { actualQPS := float64(functionDuration.Count) / elapsed.Seconds() if log.Log(log.Warning) { // nolint: gas - _, _ = fmt.Fprintf(r.Out, "Ended after %v : %d calls. qps=%.5g\n", elapsed, functionDuration.Count, actualQPS) + fmt.Fprintf(r.Out, "Ended after %v : %d calls. qps=%.5g\n", elapsed, functionDuration.Count, actualQPS) } if useQPS { percentNegative := 100. * float64(sleepTime.Hdata[0]) / float64(sleepTime.Count) @@ -428,7 +418,7 @@ func (r *periodicRunner) Run() RunnerResults { // user. if percentNegative > 5 { sleepTime.Print(r.Out, "Aggregated Sleep Time", []float64{50}) - _, _ = fmt.Fprintf(r.Out, "WARNING %.2f%% of sleep were falling behind\n", percentNegative) // nolint: gas + fmt.Fprintf(r.Out, "WARNING %.2f%% of sleep were falling behind\n", percentNegative) // nolint: gas } else { if log.Log(log.Verbose) { sleepTime.Print(r.Out, "Aggregated Sleep Time", []float64{50}) @@ -441,14 +431,14 @@ func (r *periodicRunner) Run() RunnerResults { if useExactly && actualCount != r.Exactly { requestedDuration += fmt.Sprintf(", interrupted after %d", actualCount) } - result := RunnerResults{r.RunType, r.Labels, start, requestedQPS, requestedDuration, - actualQPS, elapsed, r.NumThreads, version.Short(), functionDuration.Export().CalcPercentiles(r.Percentiles), r.Exactly} + result := RunnerResults{r.Labels, start, requestedQPS, requestedDuration, + actualQPS, elapsed, r.NumThreads, Version, functionDuration.Export().CalcPercentiles(r.Percentiles), r.Exactly} if log.Log(log.Warning) { result.DurationHistogram.Print(r.Out, "Aggregated Function Time") } else { functionDuration.Counter.Print(r.Out, "Aggregated Function Time") for _, p := range result.DurationHistogram.Percentiles { - _, _ = fmt.Fprintf(r.Out, "# target %g%% %.6g\n", p.Percentile, p.Value) // nolint: gas + fmt.Fprintf(r.Out, "# target %g%% %.6g\n", p.Percentile, p.Value) // nolint: gas } } select { diff --git a/vendor/fortio.org/fortio/stats/stats.go b/vendor/istio.io/fortio/stats/stats.go similarity index 86% rename from vendor/fortio.org/fortio/stats/stats.go rename to vendor/istio.io/fortio/stats/stats.go index ea02151bf3..b761143deb 100644 --- a/vendor/fortio.org/fortio/stats/stats.go +++ b/vendor/istio.io/fortio/stats/stats.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package stats // import "fortio.org/fortio/stats" +package stats // import "istio.io/fortio/stats" import ( "bufio" @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "fortio.org/fortio/log" + "istio.io/fortio/log" ) // Counter is a type whose instances record values @@ -78,7 +78,7 @@ func (c *Counter) StdDev() float64 { // Print prints stats. func (c *Counter) Print(out io.Writer, msg string) { - _, _ = fmt.Fprintf(out, "%s : count %d avg %.8g +/- %.4g min %g max %g sum %.9g\n", // nolint(errorcheck) + fmt.Fprintf(out, "%s : count %d avg %.8g +/- %.4g min %g max %g sum %.9g\n", // nolint(errorcheck) msg, c.Count, c.Avg(), c.StdDev(), c.Min, c.Max, c.Sum) } @@ -139,10 +139,7 @@ var ( numBuckets = numValues + 1 // 1 special first bucket is <= 0; and 1 extra last bucket is > 100000 firstValue = float64(histogramBucketValues[0]) lastValue = float64(histogramBucketValues[numValues-1]) - val2Bucket []int // ends at 1000. Remaining values will not be received in constant time. - - maxArrayValue = int32(1000) // Last value looked up as O(1) array, the rest is linear search - maxArrayValueIndex = -1 // Index of maxArrayValue + val2Bucket []int ) // Histogram extends Counter and adds an histogram. @@ -206,47 +203,24 @@ func NewHistogram(Offset float64, Divider float64) *Histogram { return h } -// Val2Bucket values are kept in two different structure -// val2Bucket allows you reach between 0 and 1000 in constant time +// Tradeoff memory for speed (though that also kills the cache so...) +// this creates an array of 100k (max value) entries +// TODO: consider using an interval search for the last N big buckets func init() { - val2Bucket = make([]int, maxArrayValue) - maxArrayValueIndex = -1 - for i, value := range histogramBucketValues { - if value == maxArrayValue { - maxArrayValueIndex = i - break - } - } - if maxArrayValueIndex == -1 { - log.Fatalf("Bug boundary maxArrayValue=%d not found in bucket list %v", maxArrayValue, histogramBucketValues) - } + lastV := int32(lastValue) + val2Bucket = make([]int, lastV+1) idx := 0 - for i := int32(0); i < maxArrayValue; i++ { + for i := int32(0); i <= lastV; i++ { if i >= histogramBucketValues[idx] { idx++ } val2Bucket[i] = idx } - // coding bug detection (aka impossible if it works once) until 1000 - if idx != maxArrayValueIndex { - log.Fatalf("Bug in creating histogram index idx %d vs index %d up to %d", idx, int(maxArrayValue), maxArrayValue) + // coding bug detection (aka impossible if it works once) + if idx != numValues { + log.Fatalf("Bug in creating histogram buckets idx %d vs numbuckets %d (last val %d)", idx, numValues, lastV) } -} -// lookUpIdx looks for scaledValue's index in histogramBucketValues -// TODO: change linear time to O(log(N)) with binary search -func lookUpIdx(scaledValue int) int { - scaledValue32 := int32(scaledValue) - if scaledValue32 < maxArrayValue { //constant - return val2Bucket[scaledValue] - } - for i := maxArrayValueIndex; i < numValues; i++ { - if histogramBucketValues[i] > scaledValue32 { - return i - } - } - log.Fatalf("never reached/bug") - return 0 } // Record records a data point. @@ -273,7 +247,7 @@ func (h *Histogram) record(v float64, count int) { idx = numBuckets - 1 // last bucket is for > last value } else { // else we look it up - idx = lookUpIdx(int(scaledVal)) + idx = val2Bucket[int(scaledVal)] } h.Hdata[idx] += int32(count) } @@ -387,24 +361,25 @@ func (e *HistogramData) CalcPercentiles(percentiles []float64) *HistogramData { // Also calculates the percentile. func (e *HistogramData) Print(out io.Writer, msg string) { if len(e.Data) == 0 { - _, _ = fmt.Fprintf(out, "%s : no data\n", msg) // nolint: gas + fmt.Fprintf(out, "%s : no data\n", msg) // nolint: gas return } // the base counter part: - _, _ = fmt.Fprintf(out, "%s : count %d avg %.8g +/- %.4g min %g max %g sum %.9g\n", + fmt.Fprintf(out, "%s : count %d avg %.8g +/- %.4g min %g max %g sum %.9g\n", // nolint(errorcheck) msg, e.Count, e.Avg, e.StdDev, e.Min, e.Max, e.Sum) - _, _ = fmt.Fprintln(out, "# range, mid point, percentile, count") + fmt.Fprintln(out, "# range, mid point, percentile, count") // nolint: gas sep := ">=" for i, b := range e.Data { if i > 0 { sep = ">" // last interval is inclusive (of max value) } - _, _ = fmt.Fprintf(out, "%s %.6g <= %.6g , %.6g , %.2f, %d\n", sep, b.Start, b.End, (b.Start+b.End)/2., b.Percent, b.Count) + // nolint: gas + fmt.Fprintf(out, "%s %.6g <= %.6g , %.6g , %.2f, %d\n", sep, b.Start, b.End, (b.Start+b.End)/2., b.Percent, b.Count) } // print the information of target percentiles for _, p := range e.Percentiles { - _, _ = fmt.Fprintf(out, "# target %g%% %.6g\n", p.Percentile, p.Value) // nolint: gas + fmt.Fprintf(out, "# target %g%% %.6g\n", p.Percentile, p.Value) // nolint: gas } }