regression tests for fixes in release/0.84.1

.circleci/config.yml

@@ -1,10 +1,9 @@
 version: 2
 jobs:
-  postgresql:
+  postgresql-ssl:
     docker:
       - image: cossacklabs/android-build
-      # postgresql image with ssl support
-      - image: nimbustech/postgres-ssl:9.5
+      - image: cossacklabs/postgresql-ssl:11
         environment:
           POSTGRES_PASSWORD: test
           POSTGRES_USER: test
@@ -16,6 +15,7 @@ jobs:
       TEST_DB_PORT: 5432
       GOPATH_FOLDER: gopath
       TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "on"
     steps:
     # prepare
       - checkout
@@ -40,12 +40,41 @@ jobs:
       # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
       - run: .circleci/integration.sh
       - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
-  mysql:
+
+  postgresql:
+    docker:
+      - image: cossacklabs/android-build
+      - image: postgres:11-alpine
+        environment:
+          POSTGRES_PASSWORD: test
+          POSTGRES_USER: test
+          POSTGRES_DB: test
+    environment:
+      GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
+      FILEPATH_ERROR_FLAG: /tmp/test_fail
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_DB_PORT: 5432
+      GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "off"
+    steps:
+      # prepare
+      - checkout
+      - run: .circleci/prepare.sh
+      - run: sudo apt-get install -y postgresql-client
+      - run: pg_isready -U${POSTGRES_USER} -d${POSTGRES_DB} -h127.0.0.1
+      # generate test data for integration tests
+      - run: python3 tests/generate_random_data.py
+      # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
+      - run: .circleci/integration.sh
+      - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+
+  mariadb-ssl:
     docker:
     - image: cossacklabs/android-build
     # use the same credentials for mysql db as for postgresql (which support was added first)
     # has latest tag on 2018.03.29
-    - image: mysql:5.7.21
+    - image: cossacklabs/mariadb-ssl:10.3
       environment:
         MYSQL_DATABASE: test
         MYSQL_USER: test
@@ -59,6 +88,7 @@ jobs:
       TEST_DB_PORT: 3306
       GOPATH_FOLDER: gopath
       TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "on"
     steps:
     # prepare
     - checkout
@@ -81,12 +111,46 @@ jobs:
     # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
     - run: .circleci/integration.sh
     - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+
   mariadb:
+    docker:
+      - image: cossacklabs/android-build
+      # use the same credentials for mysql db as for postgresql (which support was added first)
+      # has latest tag on 2018.03.29
+      - image: mariadb:latest
+        environment:
+          MYSQL_DATABASE: test
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_ROOT_PASSWORD: root
+    environment:
+      GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
+      FILEPATH_ERROR_FLAG: /tmp/test_fail
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_MYSQL: true
+      TEST_DB_PORT: 3306
+      GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "off"
+    steps:
+      # prepare
+      - checkout
+      - run: .circleci/prepare.sh
+      - run: sudo apt-get install -y mysql-client
+      - run: mysqladmin ping -h127.0.0.1
+      # testing
+      # generate test data for integration tests
+      - run: python3 tests/generate_random_data.py
+      # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
+      - run: .circleci/integration.sh
+      - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+
+  mysql:
     docker:
     - image: cossacklabs/android-build
     # use the same credentials for mysql db as for postgresql (which support was added first)
     # has latest tag on 2018.03.29
-    - image: mariadb:latest
+    - image: cossacklabs/mysql-ssl:5.7.25
       environment:
         MYSQL_DATABASE: test
         MYSQL_USER: test
@@ -100,6 +164,7 @@ jobs:
       TEST_DB_PORT: 3306
       GOPATH_FOLDER: gopath
       TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "on"
     steps:
     # prepare
     - checkout
@@ -122,11 +187,44 @@ jobs:
     # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
     - run: .circleci/integration.sh
     - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+
+  mysql-ssl:
+    docker:
+      - image: cossacklabs/android-build
+      - image: mysql:5.7.25
+        environment:
+          MYSQL_DATABASE: test
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_ROOT_PASSWORD: root
+    environment:
+      GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
+      FILEPATH_ERROR_FLAG: /tmp/test_fail
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_MYSQL: true
+      TEST_DB_PORT: 3306
+      GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+      TEST_TLS: "off"
+    steps:
+      # prepare
+      - checkout
+      - run: .circleci/prepare.sh
+      - run: sudo apt-get install -y mysql-client
+      - run: mysqladmin ping -h127.0.0.1
+      # testing
+      # generate test data for integration tests
+      - run: python3 tests/generate_random_data.py
+      # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
+      - run: .circleci/integration.sh
+      - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
 workflows:
   version: 2
   tests:
     jobs:
       - postgresql
+      - postgresql-ssl
       - mysql
-      # has errors with tls connections
-      #- mariadb
+      - mysql-ssl
+      - mariadb
+      - mariadb-ssl

.circleci/integration.sh

@@ -8,6 +8,10 @@ export TEST_DB_USER_PASSWORD=test
 export TEST_DB_NAME=test
 
 cd $HOME/project
+# set correct permissions for ssl keys here because git by default recognize changing only executable bit
+# http://git.661346.n2.nabble.com/file-mode-td6467904.html#a6469081
+# https://stackoverflow.com/questions/11230171/git-is-changing-my-files-permissions-when-i-push-to-server/11231682#11231682
+find tests/ssl -name "*.key" -type f -exec chmod 0600 {} \;
 for version in $VERSIONS; do
     echo "-------------------- Testing Go version $version"
 
@@ -18,18 +22,11 @@ for version in $VERSIONS; do
     export PATH=$GOROOT/bin/:$PATH;
     export GOPATH=$HOME/$GOPATH_FOLDER;
 
-    export TEST_TLS=on
 
-    echo "--------------------  Testing with TEST_TLS=on"
+    echo "--------------------  Testing with TEST_TLS=${TEST_TLS}"
 
     python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "golang-$version" >> "$FILEPATH_ERROR_FLAG";
+    if [ "$?" != "0" ]; then echo "golang-$version test_tls=${TEST_TLS}" >> "$FILEPATH_ERROR_FLAG";
     fi
 
-    export TEST_TLS=off
-
-    echo "--------------------  Testing with TEST_TLS=off"
-    python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "golang-$version" >> "$FILEPATH_ERROR_FLAG";
-    fi
 done

.circleci/prepare.sh

@@ -9,4 +9,4 @@ pip3 install -r $HOME/project/tests/requirements.txt -r $HOME/project/wrappers/p
 # install from sources because pip install git+https://github.com/mysql/mysql-connector-python not support recursive submodules
 git clone https://github.com/Lagovas/mysql-connector-python; cd mysql-connector-python; sudo python3 setup.py clean build_py install_lib
 cd $HOME && GOPATH=$HOME/$GOPATH_FOLDER go get -u -v golang.org/x/lint/golint
-sudo ldconfig
+sudo ldconfig

cmd/acra-server/acra-server.go

@@ -56,14 +56,6 @@ var restartSignalsChannel chan os.Signal
 var errorSignalChannel chan os.Signal
 var authPath *string
 
-// For testing purposes only, allows to skip checking TLS certificate when connecting to database.
-const (
-	testMode = "true"
-)
-
-// TestOnly is set in compile time for running integration tests
-var TestOnly = "false"
-
 // Constants used by AcraServer.
 const (
 	defaultAcraserverWaitTimeout = 10
@@ -234,12 +226,7 @@ func main() {
 				Errorln("Configuration error: can't get config for TLS")
 			os.Exit(1)
 		}
-		// need for testing with mysql docker container that always generate new certificates
-		if TestOnly == testMode {
-			tlsConfig.InsecureSkipVerify = true
-			tlsConfig.ClientAuth = tls.NoClientCert
-			log.Warningln("Skip verifying TLS certificate, use for tests only!")
-		}
+		log.Infoln("Loaded tls config")
 	}
 	if *useTLS {
 		log.Println("Selecting transport: use TLS transport wrapper")

decryptor/mysql/response_proxy.go

@@ -342,7 +342,7 @@ func (handler *Handler) ProxyClientConnection(errCh chan<- error) {
 			handler.setQueryHandler(handler.QueryResponseHandler)
 			break
 		case CommandStatementClose, CommandStatementSendLongData, CommandStatementReset:
-			fallthrough
+			clientLog.Debugln("Close|SendLongData|Reset command")
 		default:
 			clientLog.Debugf("Command %d not supported now", cmd)
 		}
@@ -381,7 +381,7 @@ func (handler *Handler) processTextDataRow(rowData []byte, fields []*ColumnDescr
 		}
 		if handler.isFieldToDecrypt(fields[i]) {
 			decryptedValue, err := handler.decryptor.DecryptBlock(value)
-			if err == nil && len(decryptedValue) != len(value) {
+			if err == nil && decryptedValue != nil && len(decryptedValue) != len(value) {
 				fieldLogger.Debugln("Update with decrypted value")
 				output = append(output, PutLengthEncodedString(decryptedValue)...)
 			} else {
@@ -443,7 +443,7 @@ func (handler *Handler) processBinaryDataRow(rowData []byte, fields []*ColumnDes
 			if err != nil {
 				handler.logger.Debugln("Leave value as is")
 			}
-			if len(value) != len(decryptedValue) {
+			if decryptedValue != nil && err == nil && len(value) != len(decryptedValue) {
 				output = append(output, PutLengthEncodedString(decryptedValue)...)
 			} else {
 				output = append(output, rowData[pos:pos+n]...)

decryptor/postgresql/dataProcessor.go

@@ -20,16 +20,18 @@ import (
 	"encoding/hex"
 	"github.com/cossacklabs/acra/decryptor/base"
 	"github.com/cossacklabs/acra/utils"
+	"github.com/sirupsen/logrus"
 )
 
 // NewEncodeDecodeWrapper encode/decode data to/from escaped format (hex/octal)
 func NewEncodeDecodeWrapper(processor base.DataProcessor) base.DataProcessor {
 	return base.ProcessorFunc(func(data []byte, ctx *base.DataProcessorContext) ([]byte, error) {
-		data, err := utils.DecodeEscaped(data)
+		decodedData, err := utils.DecodeEscaped(data)
 		if err != nil {
-			return data, err
+			logrus.WithError(err).Debugln("Data is not in hex/escape format, process as binary data (used in prepared statements)")
+			decodedData = data
 		}
-		data, err = processor.Process(data, ctx)
+		data, err = processor.Process(decodedData, ctx)
 		if err != nil {
 			return data, err
 		}

tests/acra-writer.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID8zCCAtugAwIBAgIQLMpIn50GP4xfg4bUVlNKADANBgkqhkiG9w0BAQsFADCB
+gDEgMB4GA1UEAwwXZXhhbXBsZS5jb3NzYWNrbGFicy5jb20xCzAJBgNVBAYTAkdC
+MSAwHgYDVQQKDBdleGFtcGxlLmNvc3NhY2tsYWJzLmNvbTELMAkGA1UECwwCSFEx
+DzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMCAXDTE4MDUyNTEyMjcz
+MFoYDzIwNjgwNTEyMTIyNzMwWjB0MRQwEgYDVQQDDAthY3JhLXdyaXRlcjELMAkG
+A1UEBhMCR0IxIDAeBgNVBAoMF2V4YW1wbGUuY29zc2Fja2xhYnMuY29tMQswCQYD
+VQQLDAJIUTEPMA0GA1UECAwGTG9uZG9uMQ8wDQYDVQQHDAZMb25kb24wggEiMA0G
+CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtdCZkoGRBFFBHZDSghU70gEfhZka1
+qN8fUh/rJUHQXm7eye11qAPq/6TlWvvD9q65YCxM9KlXKTy9TMJKrLPMDBzdarnN
+OTrXktRJG94N7aILZG9DOAH5sKAZOeaQVGUzyShShriABo9/FkMUHJv8C2IySWDt
++CPRjUBqIqVgNgCDLBJW1wyqrr9tIIbT+ShHGv5b57QJU5EO0q+SCNlscvM79HHy
+59TIOrDd1I7DBAWz5eSYj58WZJHpPu6qscqlkZj8/5rRl2DiWu0SLrROGJeP2i52
+etuIVHgwrVcBzXQrRbCu9osmjkfsnCIMb7ICpGbOy5PsUFyoapZ6x1P5AgMBAAGj
+cjBwMB0GA1UdDgQWBBSFgL110VC0kj08M6oGqfnYK0v3gTAfBgNVHSMEGDAWgBTm
+xI2vIRpx1x9skyGkezxpOQj+GjALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB
+BQUHAwIwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAQEADPDkqxgzAEEB
+TE2UhCl8MBHOqm4duZWCJbmCcV1jB02gtW6DejU3h0brL16yhHoQJm7WQ07c66dk
+Nr4iHr92TQV3dv4Fl9cu8Q7FLnsqGw+EayTCgalLlTiiKc0FrKnVr6XNzjVWBYCZ
+Zbi1ydkqXiUO7ffymqy34jrDd1fwdtNyLqwQdf6GYsZ5e0p0REKrKOxdnGzuyP01
+/b4x34AB6AhxOOk+v7shQKGQp2pBDPxbaKstgBIGHiSK4nPAF4Po6zid/aeyJ3IN
+mdW8w5cIz5iQ2PRws8d/GoMlgJ+eZ1F4tKLTTbyuoEV0YwmP8hsGA0uSVd0Zsldf
+GpIHDMX7EQ==
+-----END CERTIFICATE-----

tests/acra-writer.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA7XQmZKBkQRRQR2Q0oIVO9IBH4WZGtajfH1If6yVB0F5u3snt
+dagD6v+k5Vr7w/auuWAsTPSpVyk8vUzCSqyzzAwc3Wq5zTk615LUSRveDe2iC2Rv
+QzgB+bCgGTnmkFRlM8koUoa4gAaPfxZDFByb/AtiMklg7fgj0Y1AaiKlYDYAgywS
+VtcMqq6/bSCG0/koRxr+W+e0CVORDtKvkgjZbHLzO/Rx8ufUyDqw3dSOwwQFs+Xk
+mI+fFmSR6T7uqrHKpZGY/P+a0Zdg4lrtEi60ThiXj9oudnrbiFR4MK1XAc10K0Ww
+rvaLJo5H7JwiDG+yAqRmzsuT7FBcqGqWesdT+QIDAQABAoIBAGblMQnj0u28jrkQ
+/BkfC0K6H+vVdQidH+qjnL85ytYvRZ33JJYac3AuzVax4W1ehQJBYZELaCIj9XUW
+QQ843USrbwqnh9IdXZkywBjaIZQpiFGleO1I5No9JnMn+vhKCfyxy25PgL3Bmgb5
+L6IUb6d9i1S9JWqZX3Qb4HDdiIoypqdqfPhi4avGcZaiTqtQMkuQDGMWR4jWkQ+5
++6safjAYhuJO0saKSWzS3KlJ5G8dh2/6Fv7SKXn8JBmmV9Y4FF96AIclb30KEXsL
+aFf+r4QhKt6Z8OCPEx5haMwuYXPP2mctnhhIoug9lS5DrZYItu+pZguK6pAmOqII
+wYb4zHECgYEA/maQMuugazc0unKwP6APWZUTvndqqeDJYaVDwcwTBQujWDkJq4CD
+tCt8//ZrEfCurRt89ZVyuNAAD7bdKHjGBdrlxqGS4QE8XEAokw5SVHwldtrYXqRx
+nQ577q2j0GihSf8A9Vrm70PcAads/fwfenpMJ7iPb65dGQsRPjKrSVUCgYEA7vJP
+4LoeJ/L1EBC20CcrDFQ9SEih5ynT83ImDTuWdq+pmdmbUk1Qd6CWaTFqUcTNl9zl
+hZ3X+1KAnV47pd1CKpq2D+ELpy1FXq1rrgH1YnjWELoDA/I8aaadmXGqM6ezc3ET
+QhzSuVPixKdOo3eEOvRlt3KbGSv1mc7X9MGREBUCgYAPcBfmDyhv+Syo99a8NbFY
+xVQqZcLCRlLSKx4+f2zUDI3AB84Pa0ASyeTO/5wlvY5MEPP3TzdkAbNPQNCQ6StT
+JaJ1+E4ae2G3U2li6yaR2+rGGKeyrpX6auuYY08ikYbgH6pmOTr0YBDUxN03WHps
+i0ldy7RgmZsqruI//QEp9QKBgG2+dNluIfOPLubWBmDL3IWpGwxb76BQIbyqLLff
+KlgQjgDPZPsD73VCGFLwEzjC/vpExlX8DPqtBc1BPnwLhxwyOnRIlCWQMVAurkoZ
+NC6p+Qdtp3zkgMK1d6TGA7n9x/m5cXLn8hCUao6PmRqTBfRd02nSBVImUoL18VHE
+pW+RAoGAKQSyBT4dQGsBzoKj3bQBdeufhHcB6ahtjVVwPyqeEXF7tBEn/tk9/C/u
+IRmni64ME7t/+aUGU/TaPIi7emrlfSc8xT60OGHdK/tFAUhVIfk3sCuEn82YjsBQ
+kKA+K7olthFbK4eZVlNaoT9+kHVSBpCBq8Yx7q7fTKJ4OTmyXAg=
+-----END RSA PRIVATE KEY-----