diff --git a/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go b/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go index f749808e8..9bcd52e0e 100644 --- a/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go +++ b/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go @@ -43,6 +43,9 @@ const ( // Client authentication valid for defaults defaultClientAuthValidFor = time.Hour * 24 * 365 * 1 // 1 years defaultClientAuthCAValidFor = time.Hour * 24 * 365 * 15 // 15 years + // TLS curve defaults + defaultTLSCurve = "P256" + defaultClientAuthCurve = "P521" ) var ( @@ -148,11 +151,11 @@ type createCAOptions struct { ecdsaCurve string } -func (o *createCAOptions) ConfigureFlags(f *pflag.FlagSet, defaultFName string, defaultValidFor time.Duration) { +func (o *createCAOptions) ConfigureFlags(f *pflag.FlagSet, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { f.StringVar(&o.certFile, "cert", defaultFName+".crt", "Filename of the generated CA certificate") f.StringVar(&o.keyFile, "key", defaultFName+".key", "Filename of the generated CA private key") f.DurationVar(&o.validFor, "validfor", defaultValidFor, "Lifetime of the certificate until expiration") - f.StringVar(&o.ecdsaCurve, "curve", "P521", "ECDSA curve used for private key") + f.StringVar(&o.ecdsaCurve, "curve", defaultCurve, "ECDSA curve used for private key") } func (o *createCAOptions) CreateCA() { @@ -184,13 +187,13 @@ type createCertificateBaseOptions struct { ecdsaCurve string } -func (o *createCertificateBaseOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { +func (o *createCertificateBaseOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { f.StringVar(&o.caCertFile, "cacert", defaultCAFName+".crt", "File containing TLS CA certificate") f.StringVar(&o.caKeyFile, "cakey", defaultCAFName+".key", "File containing TLS CA private key") f.StringSliceVar(&o.hosts, "host", nil, "Host name to include in the certificate") f.StringSliceVar(&o.emailAddresses, "email", nil, "Email address to include in the certificate") f.DurationVar(&o.validFor, "validfor", defaultValidFor, "Lifetime of the certificate until expiration") - f.StringVar(&o.ecdsaCurve, "curve", "P521", "ECDSA curve used for private key") + f.StringVar(&o.ecdsaCurve, "curve", defaultCurve, "ECDSA curve used for private key") } // Create a certificate from given options. @@ -206,8 +209,8 @@ func (o *createCertificateBaseOptions) CreateCertificate(isClientAuth bool) (str // Create certificate options := certificates.CreateCertificateOptions{ - Hosts: o.hosts, - EmailAddresses: o.emailAddresses, + Hosts: removeEmptyStrings(o.hosts), + EmailAddresses: removeEmptyStrings(o.emailAddresses), ValidFor: o.validFor, ECDSACurve: o.ecdsaCurve, IsClientAuth: isClientAuth, @@ -225,8 +228,8 @@ type createKeyFileOptions struct { keyFile string } -func (o *createKeyFileOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createKeyFileOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.keyFile, "keyfile", defaultFName+".keyfile", "Filename of keyfile to generate") } @@ -247,8 +250,8 @@ type createCertificateOptions struct { keyFile string } -func (o *createCertificateOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createCertificateOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.certFile, "cert", defaultFName+".crt", "Filename of the generated certificate") f.StringVar(&o.keyFile, "key", defaultFName+".key", "Filename of the generated private key") } @@ -272,8 +275,8 @@ type createKeystoreOptions struct { alias string } -func (o *createKeystoreOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createKeystoreOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.keystoreFile, "keystore", defaultFName+".jks", "Filename of the generated keystore") f.StringVar(&o.keystorePassword, "keystore-password", "", "Password of the generated keystore") f.StringVar(&o.alias, "alias", "", "Aliases use to store the certificate under in the keystore") @@ -317,12 +320,12 @@ func AddCommands(cmd *cobra.Command, logFatalFunc func(error, string), showUsage cmdCreateClientAuth.AddCommand(cmdCreateClientAuthKeyFile) createOptions.jwtsecret.ConfigureFlags(cmdCreateJWTSecret.Flags()) - createOptions.tls.ca.ConfigureFlags(cmdCreateTLSCA.Flags(), "tls-ca", defaultTLSCAValidFor) - createOptions.tls.keyFile.ConfigureFlags(cmdCreateTLSKeyFile.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.tls.certificate.ConfigureFlags(cmdCreateTLSCertificate.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.tls.keystore.ConfigureFlags(cmdCreateTLSKeystore.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.clientAuth.ca.ConfigureFlags(cmdCreateClientAuthCA.Flags(), "client-auth-ca", defaultClientAuthCAValidFor) - createOptions.clientAuth.keyFile.ConfigureFlags(cmdCreateClientAuthKeyFile.Flags(), "client-auth-ca", "client-auth", defaultClientAuthValidFor) + createOptions.tls.ca.ConfigureFlags(cmdCreateTLSCA.Flags(), "tls-ca", defaultTLSCAValidFor, defaultTLSCurve) + createOptions.tls.keyFile.ConfigureFlags(cmdCreateTLSKeyFile.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.tls.certificate.ConfigureFlags(cmdCreateTLSCertificate.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.tls.keystore.ConfigureFlags(cmdCreateTLSKeystore.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.clientAuth.ca.ConfigureFlags(cmdCreateClientAuthCA.Flags(), "client-auth-ca", defaultClientAuthCAValidFor, defaultClientAuthCurve) + createOptions.clientAuth.keyFile.ConfigureFlags(cmdCreateClientAuthKeyFile.Flags(), "client-auth-ca", "client-auth", defaultClientAuthValidFor, defaultClientAuthCurve) } // Cobra run function using the usage of the given command @@ -401,3 +404,14 @@ func mustReadFile(filename string, flagName string) string { } return string(content) } + +// removeEmptyStrings returns the given slice without all empty entries removed. +func removeEmptyStrings(slice []string) []string { + result := make([]string, 0, len(slice)) + for _, x := range slice { + if x != "" { + result = append(result, x) + } + } + return result +} diff --git a/deps/github.com/arangodb-helper/go-certificates/create.go b/deps/github.com/arangodb-helper/go-certificates/create.go index 0cd489bee..d8bdf82a8 100644 --- a/deps/github.com/arangodb-helper/go-certificates/create.go +++ b/deps/github.com/arangodb-helper/go-certificates/create.go @@ -44,6 +44,7 @@ const ( ) type CreateCertificateOptions struct { + Subject *pkix.Name // If set, this name is used for the subject of the certificate and CommonName is ignored. CommonName string // Common name set in the certificate. If not specified, defaults to first email address, then first host and if all not set 'ArangoDB'. Hosts []string // Comma-separated hostnames and IPs to generate a certificate for EmailAddresses []string // List of email address to include in the certificate as alternative name @@ -101,14 +102,18 @@ func CreateCertificate(options CreateCertificateOptions, ca *CA) (string, string } else if len(options.Hosts) > 0 { commonName = options.Hosts[0] } + var subject pkix.Name + if options.Subject != nil { + subject = *options.Subject + } else { + subject.CommonName = commonName + subject.Organization = []string{"ArangoDB"} + } template := x509.Certificate{ SerialNumber: serialNumber, - Subject: pkix.Name{ - CommonName: commonName, - Organization: []string{"ArangoDB"}, - }, - NotBefore: notBefore, - NotAfter: notAfter, + Subject: subject, + NotBefore: notBefore, + NotAfter: notAfter, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, diff --git a/deps/github.com/arangodb/go-driver/.travis.yml b/deps/github.com/arangodb/go-driver/.travis.yml index 5ab1337f6..da58d729d 100644 --- a/deps/github.com/arangodb/go-driver/.travis.yml +++ b/deps/github.com/arangodb/go-driver/.travis.yml @@ -7,7 +7,7 @@ language: go env: - TEST_SUITE=run-tests-http - - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.1 + - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.2 - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb:latest - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb-preview:latest diff --git a/deps/github.com/arangodb/go-driver/Makefile b/deps/github.com/arangodb/go-driver/Makefile index 85f2b3216..d93aa276c 100644 --- a/deps/github.com/arangodb/go-driver/Makefile +++ b/deps/github.com/arangodb/go-driver/Makefile @@ -3,7 +3,7 @@ SCRIPTDIR := $(shell pwd) ROOTDIR := $(shell cd $(SCRIPTDIR) && pwd) GOBUILDDIR := $(SCRIPTDIR)/.gobuild -GOVERSION := 1.9.2-alpine +GOVERSION := 1.10.1-alpine TMPDIR := $(GOBUILDDIR) ifndef ARANGODB @@ -292,6 +292,7 @@ __test_go_test: --net=$(TEST_NET) \ -v $(ROOTDIR):/usr/code \ -e GOPATH=/usr/code/.gobuild \ + -e GOCACHE=off \ -e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \ -e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \ -e TEST_CONNECTION=$(TEST_CONNECTION) \ diff --git a/deps/github.com/arangodb/go-driver/agency/agency_connection.go b/deps/github.com/arangodb/go-driver/agency/agency_connection.go index 743be9145..1ff703fb4 100644 --- a/deps/github.com/arangodb/go-driver/agency/agency_connection.go +++ b/deps/github.com/arangodb/go-driver/agency/agency_connection.go @@ -143,11 +143,13 @@ func (c *agencyConnection) doOnce(ctx context.Context, req driver.Request) (driv epReq := req.Clone() result, err := epConn.Do(ctx, epReq) if err == nil { - // Success - results <- result - // Cancel all other requests - cancel() - return + if err = isSuccess(result); err == nil { + // Success + results <- result + // Cancel all other requests + cancel() + return + } } // Check error if statusCode, ok := isArangoError(err); ok { @@ -160,6 +162,10 @@ func (c *agencyConnection) doOnce(ctx context.Context, req driver.Request) (driv return } } + // No permanent error. Are we the only endpoint? + if len(connections) == 1 { + errors <- driver.WithStack(err) + } // No permanent error, try next agent }(epConn) } @@ -180,6 +186,20 @@ func (c *agencyConnection) doOnce(ctx context.Context, req driver.Request) (driv return nil, false, driver.WithStack(fmt.Errorf("All %d servers responded with temporary failure", len(connections))) } +func isSuccess(resp driver.Response) error { + if resp == nil { + return driver.WithStack(fmt.Errorf("Response is nil")) + } + statusCode := resp.StatusCode() + if statusCode >= 200 && statusCode < 300 { + return nil + } + return driver.ArangoError{ + HasError: true, + Code: statusCode, + } +} + // isArangoError checks if the given error is (or is caused by) an ArangoError. // If so it returned the Code and true, otherwise it returns 0, false. func isArangoError(err error) (int, bool) { @@ -224,7 +244,7 @@ func (c *agencyConnection) UpdateEndpoints(endpoints []string) error { for i, ep := range endpoints { config := c.config config.Endpoints = []string{ep} - config.FailOnRedirect = true + config.DontFollowRedirect = true httpConn, err := http.NewConnection(config) if err != nil { return driver.WithStack(err) diff --git a/deps/github.com/arangodb/go-driver/agency/doc.go b/deps/github.com/arangodb/go-driver/agency/doc.go index 23b0a2683..76412a159 100644 --- a/deps/github.com/arangodb/go-driver/agency/doc.go +++ b/deps/github.com/arangodb/go-driver/agency/doc.go @@ -20,20 +20,19 @@ // Author Ewout Prangsma // -package agency +/* +Package agency provides an API to access the ArangoDB agency (it is unlikely that you need this package directly). -// -// The Agency is fault-tolerant and highly-available key-value store -// that is used to store critical, low-level information about -// an ArangoDB cluster. -// -// The API provided in this package gives access to the Agency. -// -// THIS API IS NOT USED FOR NORMAL DATABASE ACCESS. -// -// Reasons for using this API are: -// - You want to make use of an indepent Agency as your own HA key-value store. -// - You want access to low-level information of your database. USE WITH GREAT CARE! -// -// WARNING: Messing around in the Agency can quickly lead to a corrupt database! -// +The Agency is fault-tolerant and highly-available key-value store +that is used to store critical, low-level information about +an ArangoDB cluster. + +THIS API IS NOT USED FOR NORMAL DATABASE ACCESS. + +Reasons for using this API are: +- You want to make use of an indepent Agency as your own HA key-value store. +- You want access to low-level information of your database. USE WITH GREAT CARE! + +WARNING: Messing around in the Agency can quickly lead to a corrupt database! +*/ +package agency diff --git a/deps/github.com/arangodb/go-driver/cluster/doc.go b/deps/github.com/arangodb/go-driver/cluster/doc.go new file mode 100644 index 000000000..53abf2920 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/cluster/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package cluster implements a driver.Connection that provides cluster failover support (it is not intended to be used directly). +*/ +package cluster diff --git a/deps/github.com/arangodb/go-driver/collection_document_impl.go b/deps/github.com/arangodb/go-driver/collection_document_impl.go index b9a83eb0a..bc09aa5f7 100644 --- a/deps/github.com/arangodb/go-driver/collection_document_impl.go +++ b/deps/github.com/arangodb/go-driver/collection_document_impl.go @@ -103,7 +103,7 @@ func (c *collection) CreateDocument(ctx context.Context, document interface{}) ( if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -155,7 +155,7 @@ func (c *collection) CreateDocuments(ctx context.Context, documents interface{}) if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -196,7 +196,7 @@ func (c *collection) UpdateDocument(ctx context.Context, key string, update inte if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -264,7 +264,7 @@ func (c *collection) UpdateDocuments(ctx context.Context, keys []string, updates if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -305,7 +305,7 @@ func (c *collection) ReplaceDocument(ctx context.Context, key string, document i if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -373,7 +373,7 @@ func (c *collection) ReplaceDocuments(ctx context.Context, keys []string, docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -407,7 +407,7 @@ func (c *collection) RemoveDocument(ctx context.Context, key string) (DocumentMe if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -456,7 +456,7 @@ func (c *collection) RemoveDocuments(ctx context.Context, keys []string) (Docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/context.go b/deps/github.com/arangodb/go-driver/context.go index 162991b68..a47815dba 100644 --- a/deps/github.com/arangodb/go-driver/context.go +++ b/deps/github.com/arangodb/go-driver/context.go @@ -359,16 +359,6 @@ func applyContextSettings(ctx context.Context, req Request) contextSettings { return result } -// okStatus returns one of the given status codes depending on the WaitForSync field value. -// If WaitForSync==true, statusWithWaitForSync is returned, otherwise statusWithoutWaitForSync is returned. -func (cs contextSettings) okStatus(statusWithWaitForSync, statusWithoutWaitForSync int) int { - if cs.WaitForSync { - return statusWithWaitForSync - } else { - return statusWithoutWaitForSync - } -} - // contextOrBackground returns the given context if it is not nil. // Returns context.Background() otherwise. func contextOrBackground(ctx context.Context) context.Context { diff --git a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go index c09d5d7b7..100526953 100644 --- a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go @@ -103,7 +103,7 @@ func (c *edgeCollection) createDocument(ctx context.Context, document interface{ if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -318,7 +318,7 @@ func (c *edgeCollection) replaceDocument(ctx context.Context, key string, docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -433,7 +433,7 @@ func (c *edgeCollection) removeDocument(ctx context.Context, key string) (Docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/encode-go_1_8.go b/deps/github.com/arangodb/go-driver/encode-go_1_8.go index 1fc2d8348..1f2247177 100644 --- a/deps/github.com/arangodb/go-driver/encode-go_1_8.go +++ b/deps/github.com/arangodb/go-driver/encode-go_1_8.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build "go1.8" +// +build go1.8 package driver @@ -33,5 +33,6 @@ func pathEscape(s string) string { // pathUnescape unescapes the given value for use in a URL path. func pathUnescape(s string) string { - return url.PathUnescape(s) + r, _ := url.PathUnescape(s) + return r } diff --git a/deps/github.com/arangodb/go-driver/encode.go b/deps/github.com/arangodb/go-driver/encode.go index a490ab94e..af8d4bb20 100644 --- a/deps/github.com/arangodb/go-driver/encode.go +++ b/deps/github.com/arangodb/go-driver/encode.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build !"go1.8" +// +build !go1.8 package driver diff --git a/deps/github.com/arangodb/go-driver/http/connection.go b/deps/github.com/arangodb/go-driver/http/connection.go index 92ee9f4b1..e4c428894 100644 --- a/deps/github.com/arangodb/go-driver/http/connection.go +++ b/deps/github.com/arangodb/go-driver/http/connection.go @@ -43,6 +43,7 @@ import ( const ( DefaultMaxIdleConnsPerHost = 64 + DefaultConnLimit = 32 keyRawResponse driver.ContextKey = "arangodb-rawResponse" keyResponse driver.ContextKey = "arangodb-response" @@ -76,6 +77,10 @@ type ConnectionConfig struct { cluster.ConnectionConfig // ContentType specified type of content encoding to use. ContentType driver.ContentType + // ConnLimit is the upper limit to the number of connections to a single server. + // The default is 32 (DefaultConnLimit). + // Set this value to -1 if you do not want any upper limit. + ConnLimit int } // NewConnection creates a new HTTP connection based on the given configuration settings. @@ -95,6 +100,9 @@ func NewConnection(config ConnectionConfig) (driver.Connection, error) { // newHTTPConnection creates a new HTTP connection for a single endpoint and the remainder of the given configuration settings. func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connection, error) { + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } endpoint = util.FixupEndpointURLScheme(endpoint) u, err := url.Parse(endpoint) if err != nil { @@ -154,10 +162,19 @@ func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connect } } } + var connPool chan int + if config.ConnLimit > 0 { + connPool = make(chan int, config.ConnLimit) + // Fill with available tokens + for i := 0; i < config.ConnLimit; i++ { + connPool <- i + } + } c := &httpConnection{ endpoint: *u, contentType: config.ContentType, client: httpClient, + connPool: connPool, } return c, nil } @@ -167,6 +184,7 @@ type httpConnection struct { endpoint url.URL contentType driver.ContentType client *http.Client + connPool chan int } // String returns the endpoint as string @@ -225,6 +243,22 @@ func (c *httpConnection) Do(ctx context.Context, req driver.Request) (driver.Res if err != nil { return nil, driver.WithStack(err) } + + // Block on too many concurrent connections + if c.connPool != nil { + select { + case t := <-c.connPool: + // Ok, we're allowed to continue + defer func() { + // Give back token + c.connPool <- t + }() + case <-rctx.Done(): + // Context cancelled or expired + return nil, driver.WithStack(rctx.Err()) + } + } + resp, err := c.client.Do(r) if err != nil { return nil, driver.WithStack(err) diff --git a/deps/github.com/arangodb/go-driver/http/doc.go b/deps/github.com/arangodb/go-driver/http/doc.go new file mode 100644 index 000000000..0cffc2b01 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/http/doc.go @@ -0,0 +1,69 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package http implements driver.Connection using an HTTP connection. + +This connection uses HTTP or HTTPS to connect to the ArangoDB database and +encodes its content as JSON or Velocypack, depending on the value +of the `ContentType` fields in the `http.ConnectionConfig`. + +Creating an Insecure Connection + +To create an HTTP connection, use code like this. + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure HTTPS connection, use code like this. + + // Create an HTTPS connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package http diff --git a/deps/github.com/arangodb/go-driver/jwt/doc.go b/deps/github.com/arangodb/go-driver/jwt/doc.go index 4a9b04ce4..7d226ee37 100644 --- a/deps/github.com/arangodb/go-driver/jwt/doc.go +++ b/deps/github.com/arangodb/go-driver/jwt/doc.go @@ -20,12 +20,38 @@ // Author Ewout Prangsma // -package jwt +/* +Package jwt provides a helper function used to access ArangoDB +servers using a JWT secret. -// -// This package provides a help function used to access ArangoDB -// servers using a JWT secret. -// -// Authenticating with a JWT secret results in "super-user" access -// to the database. -// +Authenticating with a JWT secret results in "super-user" access +to the database. + +To use a JWT secret to access your database, use code like this: + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + + // Prepare authentication + hdr, err := CreateArangodJwtAuthorizationHeader("yourJWTSecret", "yourUniqueServerID") + if err != nil { + // Handle error + } + auth := driver.RawAuthentication(hdr) + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + Authentication: auth, + }) + if err != nil { + // Handle error + } + +*/ +package jwt diff --git a/deps/github.com/arangodb/go-driver/test/concurrency_test.go b/deps/github.com/arangodb/go-driver/test/concurrency_test.go new file mode 100644 index 000000000..3a5db3cf4 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/concurrency_test.go @@ -0,0 +1,184 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "strconv" + "sync" + "testing" + + driver "github.com/arangodb/go-driver" +) + +// TestConcurrentCreateSmallDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateSmallDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateSmallDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + for i := 0; i < limit; i++ { + ctx := context.Background() + doc := UserDoc{ + "Jan", + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 1000) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() +} + +// TestConcurrentCreateBigDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateBigDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateBigDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + data := make([]byte, 1024) + for i := 0; i < limit; i++ { + rand.Read(data) + ctx := context.Background() + doc := UserDoc{ + "Jan" + strconv.Itoa(i) + hex.EncodeToString(data), + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 100) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() +} diff --git a/deps/github.com/arangodb/go-driver/test/cursor_test.go b/deps/github.com/arangodb/go-driver/test/cursor_test.go index ec028b1d8..459db1fd2 100644 --- a/deps/github.com/arangodb/go-driver/test/cursor_test.go +++ b/deps/github.com/arangodb/go-driver/test/cursor_test.go @@ -232,7 +232,8 @@ func TestCreateCursor(t *testing.T) { // Test stream query cursors. The goroutines are technically only // relevant for the MMFiles engine, but don't hurt on rocksdb either func TestCreateStreamCursor(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() c := createClientFromEnv(t, true) version, err := c.Version(nil) @@ -293,12 +294,10 @@ func TestCreateStreamCursor(t *testing.T) { cursors = append(cursors, cursor) } - out := make(chan bool) - defer close(out) - // start a write query on the same collection inbetween // contrary to normal cursors which are executed right // away this will block until all read cursors are resolved + testReady := make(chan bool) go func() { query = "FOR doc IN 1..5 LET y = SLEEP(0.01) INSERT {name:'Peter', age:0} INTO cursor_stream_test" cursor, err := db.Query(ctx2, query, nil) // should not return immediately @@ -316,7 +315,7 @@ func TestCreateStreamCursor(t *testing.T) { t.Errorf("Failed to read document, err: %s", describe(err)) } } - out <- true // signal write done + testReady <- true // signal write done }() readCount := 0 @@ -331,17 +330,22 @@ func TestCreateStreamCursor(t *testing.T) { readCount++ } } - out <- false // signal read done + testReady <- false // signal read done }() writeDone := false readDone := false + deadline := time.Now().Add(time.Second * 30) for { - done := <-out - if done { - writeDone = true - } else { - readDone = true + select { + case <-time.After(time.Until(deadline)): + t.Fatal("Timeout") + case v := <-testReady: + if v { + writeDone = true + } else { + readDone = true + } } // On MMFiles the read-cursors have to finish first if writeDone && !readDone && info.Type == driver.EngineTypeMMFiles { @@ -349,6 +353,7 @@ func TestCreateStreamCursor(t *testing.T) { } if writeDone && readDone { + close(testReady) break } } diff --git a/deps/github.com/arangodb/go-driver/test/doc.go b/deps/github.com/arangodb/go-driver/test/doc.go new file mode 100644 index 000000000..45574733e --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package test implements add tests for the go-driver. +*/ +package test diff --git a/deps/github.com/arangodb/go-driver/test/document_create_test.go b/deps/github.com/arangodb/go-driver/test/document_create_test.go index 3d5b2e178..c9a2faddd 100644 --- a/deps/github.com/arangodb/go-driver/test/document_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_create_test.go @@ -156,3 +156,35 @@ func TestCreateDocumentNil(t *testing.T) { t.Fatalf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// and then checks that it exists. +func TestCreateDocumentInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Jan", + 40, + } + meta, err := col.CreateDocument(nil, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_remove_test.go b/deps/github.com/arangodb/go-driver/test/document_remove_test.go index 8b3b227d1..bd01d1996 100644 --- a/deps/github.com/arangodb/go-driver/test/document_remove_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_remove_test.go @@ -169,3 +169,36 @@ func TestRemoveDocumentKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// removes it and then checks the removal has succeeded. +func TestRemoveDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + if _, err := col.RemoveDocument(ctx, meta.Key); err != nil { + t.Fatalf("Failed to remove document '%s': %s", meta.Key, describe(err)) + } + // Should not longer exist + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError, got %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(ctx, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if found { + t.Errorf("DocumentExists returned true for '%s', expected false", meta.Key) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_replace_test.go b/deps/github.com/arangodb/go-driver/test/document_replace_test.go index f6d7bbeb9..d033e219c 100644 --- a/deps/github.com/arangodb/go-driver/test/document_replace_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_replace_test.go @@ -214,3 +214,38 @@ func TestReplaceDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// replaces it and then checks the replacement has succeeded. +func TestReplaceDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Replacement doc + replacement := Account{ + ID: "foo", + User: &UserDoc{}, + } + if _, err := col.ReplaceDocument(ctx, meta.Key, replacement); err != nil { + t.Fatalf("Failed to replace document '%s': %s", meta.Key, describe(err)) + } + // Read replaces document + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacement, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", replacement, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_update_test.go b/deps/github.com/arangodb/go-driver/test/document_update_test.go index 6c3a7fe5d..ab70f4004 100644 --- a/deps/github.com/arangodb/go-driver/test/document_update_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_update_test.go @@ -296,3 +296,38 @@ func TestUpdateDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// updates it and then checks the update has succeeded. +func TestUpdateDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Update document + update := map[string]interface{}{ + "name": "Updated", + } + if _, err := col.UpdateDocument(ctx, meta.Key, update); err != nil { + t.Fatalf("Failed to update document '%s': %s", meta.Key, describe(err)) + } + // Read updated document + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc.Name = "Updated" + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_create_test.go b/deps/github.com/arangodb/go-driver/test/documents_create_test.go index e10b36075..30ae1e4af 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_create_test.go @@ -167,3 +167,47 @@ func TestCreateDocumentsNonSlice(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentsInWaitForSyncCollection creates a few documents in a collection with waitForSync enabled and then checks that it exists. +func TestCreateDocumentsInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Jan", + 40, + }, + UserDoc{ + "Foo", + 41, + }, + UserDoc{ + "Frank", + 42, + }, + } + metas, errs, err := col.CreateDocuments(nil, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if len(metas) != len(docs) { + t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) + } else { + for i := 0; i < len(docs); i++ { + if err := errs[i]; err != nil { + t.Errorf("Expected no error at index %d, got %s", i, describe(err)) + } + + // Document must exists now + var readDoc UserDoc + if _, err := col.ReadDocument(nil, metas[i].Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", metas[i].Key, describe(err)) + } + if !reflect.DeepEqual(docs[i], readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], readDoc) + } + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_import_test.go b/deps/github.com/arangodb/go-driver/test/documents_import_test.go index 1d0840412..414fb7933 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_import_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_import_test.go @@ -559,3 +559,47 @@ func TestImportDocumentsOverwriteNo(t *testing.T) { } } } + +// TestImportDocumentsWithKeysInWaitForSyncCollection imports documents into a collection with waitForSync enabled +// and then checks that it exists. +func TestImportDocumentsWithKeysInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestImportDocumentsWithKeysInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDocWithKey{ + UserDocWithKey{ + "jan", + "Jan", + 40, + }, + UserDocWithKey{ + "foo", + "Foo", + 41, + }, + UserDocWithKey{ + "frank", + "Frank", + 42, + }, + } + + var raw []byte + ctx := driver.WithRawResponse(nil, &raw) + stats, err := col.ImportDocuments(ctx, docs, nil) + if err != nil { + t.Fatalf("Failed to import documents: %s %#v", describe(err), err) + } else { + if stats.Created != int64(len(docs)) { + t.Errorf("Expected %d created documents, got %d (json %s)", len(docs), stats.Created, formatRawResponse(raw)) + } + if stats.Errors != 0 { + t.Errorf("Expected %d error documents, got %d (json %s)", 0, stats.Errors, formatRawResponse(raw)) + } + if stats.Empty != 0 { + t.Errorf("Expected %d empty documents, got %d (json %s)", 0, stats.Empty, formatRawResponse(raw)) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_remove_test.go b/deps/github.com/arangodb/go-driver/test/documents_remove_test.go index c32391817..a52f69365 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_remove_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_remove_test.go @@ -227,3 +227,36 @@ func TestRemoveDocumentsKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestRemoveDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// removes them and then checks the removal has succeeded. +func TestRemoveDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + if _, _, err := col.RemoveDocuments(ctx, metas.Keys()); err != nil { + t.Fatalf("Failed to remove documents: %s", describe(err)) + } + // Should not longer exist + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError at %d, got %s", i, describe(err)) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_replace_test.go b/deps/github.com/arangodb/go-driver/test/documents_replace_test.go index 2240a9eb4..19748560a 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_replace_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_replace_test.go @@ -329,3 +329,54 @@ func TestReplaceDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentsInWaitForSyncCollection creates documents into a collection with waitForSync enabled, +// replaces them and then checks the replacements have succeeded. +func TestReplaceDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Pioter", + 45, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Replacement docs + replacements := []Account{ + Account{ + ID: "foo", + User: &UserDoc{}, + }, + Account{ + ID: "foo2", + User: &UserDoc{}, + }, + } + if _, _, err := col.ReplaceDocuments(ctx, metas.Keys(), replacements); err != nil { + t.Fatalf("Failed to replace documents: %s", describe(err)) + } + // Read replaced documents + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacements[i], readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, replacements[i], readDoc) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_update_test.go b/deps/github.com/arangodb/go-driver/test/documents_update_test.go index f601b3758..a4ccad15a 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_update_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_update_test.go @@ -452,3 +452,54 @@ func TestUpdateDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// updates them and then checks the updates have succeeded. +func TestUpdateDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Otto", + 43, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Update documents + updates := []map[string]interface{}{ + map[string]interface{}{ + "name": "Updated1", + }, + map[string]interface{}{ + "name": "Updated2", + }, + } + if _, _, err := col.UpdateDocuments(ctx, metas.Keys(), updates); err != nil { + t.Fatalf("Failed to update documents: %s", describe(err)) + } + // Read updated documents + for i, meta := range metas { + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc := docs[i] + doc.Name = fmt.Sprintf("Updated%d", i+1) + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, doc, readDoc) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/util.go b/deps/github.com/arangodb/go-driver/test/util.go index 3d94ce884..40ea6d031 100644 --- a/deps/github.com/arangodb/go-driver/test/util.go +++ b/deps/github.com/arangodb/go-driver/test/util.go @@ -26,6 +26,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" + "strconv" + "strings" "testing" driver "github.com/arangodb/go-driver" @@ -81,3 +84,16 @@ func formatRawResponse(raw []byte) string { } return hex.EncodeToString(raw) } + +// getIntFromEnv looks for an environment variable with given key. +// If found, it parses the value to an int, if success that value is returned. +// In all other cases, the given default value is returned. +func getIntFromEnv(envKey string, defaultValue int) int { + v := strings.TrimSpace(os.Getenv(envKey)) + if v != "" { + if result, err := strconv.Atoi(v); err == nil { + return result + } + } + return defaultValue +} diff --git a/deps/github.com/arangodb/go-driver/util/doc.go b/deps/github.com/arangodb/go-driver/util/doc.go new file mode 100644 index 000000000..b16feff9f --- /dev/null +++ b/deps/github.com/arangodb/go-driver/util/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package util provides some helper methods for the go-driver (it is unlikely that you need this package directly). +*/ +package util diff --git a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go index 233231dc1..323e2c5b0 100644 --- a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go @@ -102,7 +102,7 @@ func (c *vertexCollection) createDocument(ctx context.Context, document interfac if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -317,7 +317,7 @@ func (c *vertexCollection) replaceDocument(ctx context.Context, key string, docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -432,7 +432,7 @@ func (c *vertexCollection) removeDocument(ctx context.Context, key string) (Docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/vst/doc.go b/deps/github.com/arangodb/go-driver/vst/doc.go new file mode 100644 index 000000000..2337aea23 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/vst/doc.go @@ -0,0 +1,68 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package vst implements driver.Connection using a VelocyStream connection. + +This connection uses VelocyStream (with optional TLS) to connect to the ArangoDB database. +It encodes its contents as Velocypack. + +Creating an Insecure Connection + +To create a VST connection, use code like this. + + // Create a VST connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure VST connection, use code like this. + + // Create a VST over TLS connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package vst diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go b/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go index 974b1dbeb..6458c344b 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go @@ -24,7 +24,6 @@ package protocol import ( "encoding/binary" - "fmt" "io" driver "github.com/arangodb/go-driver" @@ -45,7 +44,7 @@ func readChunkVST1_0(r io.Reader) (chunk, error) { if (1 == (chunkX & 0x1)) && ((chunkX >> 1) > 1) { // First chunk, numberOfChunks>1 -> read messageLength - fmt.Println("Reading maxHdr") + //fmt.Println("Reading maxHdr") if err := readBytes(hdr[minChunkHeaderSize:], r); err != nil { return chunk{}, driver.WithStack(err) } diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go index 927f1aae2..ce98796a0 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go @@ -103,6 +103,12 @@ func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, err return c, nil } +// load returns an indication of the amount of work this connection has. +// 0 means no work at all, >0 means some work. +func (c *Connection) load() int { + return c.msgStore.Size() +} + // Close the connection to the server func (c *Connection) Close() error { if !c.closing { @@ -111,10 +117,7 @@ func (c *Connection) Close() error { return driver.WithStack(err) } c.msgStore.ForEach(func(m *Message) { - if m.response != nil { - close(m.response) - m.response = nil - } + m.closeResponseChan() }) } return nil @@ -140,6 +143,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M } // Prepare for receiving a response m := c.msgStore.Add(msgID) + responseChan := m.responseChan //panic(fmt.Sprintf("chunks: %d, messageParts: %d, first: %s", len(chunks), len(messageParts), hex.EncodeToString(messageParts[0]))) @@ -168,7 +172,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M if err != nil { return nil, driver.WithStack(err) } - return m.response, nil + return responseChan, nil case <-ctx.Done(): return nil, ctx.Err() } @@ -252,10 +256,7 @@ func (c *Connection) processChunk(chunk chunk) { //fmt.Println("Chunk: " + hex.EncodeToString(chunk.Data) + "\nMessage: " + hex.EncodeToString(m.Data)) // Notify listener - if m.response != nil { - m.response <- *m - close(m.response) - } + m.notifyListener() } } diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/doc.go b/deps/github.com/arangodb/go-driver/vst/protocol/doc.go new file mode 100644 index 000000000..84f503dca --- /dev/null +++ b/deps/github.com/arangodb/go-driver/vst/protocol/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package protocol implements the VelocyStream protocol (it is not intended to be used directly). +*/ +package protocol diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/message.go b/deps/github.com/arangodb/go-driver/vst/protocol/message.go index efbaccc8e..147673812 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/message.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/message.go @@ -22,16 +22,41 @@ package protocol -import "sort" +import ( + "sort" + "sync/atomic" +) // Message is what is send back to the client in response to a request. type Message struct { ID uint64 Data []byte - chunks []chunk - numberOfChunks uint32 - response chan Message + chunks []chunk + numberOfChunks uint32 + responseChanClosed int32 + responseChan chan Message +} + +// closes the response channel if needed. +func (m *Message) closeResponseChan() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + close(ch) + } + } +} + +// notifyListener pushes itself onto its response channel and closes the response channel afterwards. +func (m *Message) notifyListener() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + ch <- *m + close(ch) + } + } } // addChunk adds the given chunks to the list of chunks of the message. diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go b/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go index 5c60d4bf5..e45da2cc1 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go @@ -32,6 +32,14 @@ type messageStore struct { messages map[uint64]*Message } +// Size returns the number of messages in this store. +func (s *messageStore) Size() int { + s.mutex.RLock() + defer s.mutex.RUnlock() + + return len(s.messages) +} + // Get returns the message with given id, or nil if not found func (s *messageStore) Get(id uint64) *Message { s.mutex.RLock() @@ -58,8 +66,8 @@ func (s *messageStore) Add(id uint64) *Message { } m := &Message{ - ID: id, - response: make(chan Message), + ID: id, + responseChan: make(chan Message), } s.messages[id] = m return m diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go index a943bc60f..b40228a38 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go @@ -33,6 +33,7 @@ import ( const ( DefaultIdleConnTimeout = time.Minute + DefaultConnLimit = 3 ) // TransportConfig contains configuration options for Transport. @@ -43,6 +44,11 @@ type TransportConfig struct { // Zero means no limit. IdleConnTimeout time.Duration + // ConnLimit is the upper limit to the number of connections to a single server. + // Due to the nature of the VST protocol, this value does not have to be high. + // The default is 3 (DefaultConnLimit). + ConnLimit int + // Version specifies the version of the Velocystream protocol Version Version } @@ -63,6 +69,9 @@ func NewTransport(hostAddr string, tlsConfig *tls.Config, config TransportConfig if config.IdleConnTimeout == 0 { config.IdleConnTimeout = DefaultIdleConnTimeout } + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } return &Transport{ TransportConfig: config, hostAddr: hostAddr, @@ -91,13 +100,16 @@ func (c *Transport) CloseIdleConnections() (closed, remaining int) { c.connMutex.Lock() defer c.connMutex.Unlock() - for i, conn := range c.connections { + for i := 0; i < len(c.connections); { + conn := c.connections[i] if conn.IsClosed() || conn.IsIdle(c.IdleConnTimeout) { // Remove connection from list c.connections = append(c.connections[:i], c.connections[i+1:]...) // Close connection go conn.Close() closed++ + } else { + i++ } } @@ -141,6 +153,7 @@ func (c *Transport) getConnection(ctx context.Context) (*Connection, error) { // Invoke callback if cb := c.onConnectionCreated; cb != nil { if err := cb(ctx, conn); err != nil { + conn.Close() return nil, driver.WithStack(err) } } @@ -154,15 +167,35 @@ func (c *Transport) getAvailableConnection() *Connection { c.connMutex.Lock() defer c.connMutex.Unlock() + // Select the connection with the least amount of traffic + var bestConn *Connection + bestConnLoad := 0 + activeConnCount := 0 for _, conn := range c.connections { if !conn.IsClosed() { - conn.updateLastActivity() - return conn + activeConnCount++ + connLoad := conn.load() + if bestConn == nil || connLoad < bestConnLoad { + bestConn = conn + bestConnLoad = connLoad + } + } } - // No connections available - return nil + if bestConn == nil { + // No connections available + return nil + } + + // Is load is >0 AND the number of connections is below the limit, create a new one + if bestConnLoad > 0 && activeConnCount < c.ConnLimit { + return nil + } + + // Use the best connection found + bestConn.updateLastActivity() + return bestConn } // createConnection creates a new connection. diff --git a/deps/github.com/pkg/errors/.travis.yml b/deps/github.com/pkg/errors/.travis.yml index 7ca408d1b..15e5a1926 100644 --- a/deps/github.com/pkg/errors/.travis.yml +++ b/deps/github.com/pkg/errors/.travis.yml @@ -7,6 +7,7 @@ go: - 1.7.x - 1.8.x - 1.9.x + - 1.10.x - tip script: