Merge remote-tracking branch 'origin/rel/7.4' into fwd/7.4-master

circle.yml

@@ -2,7 +2,7 @@
 general:
   branches:
     ignore:
-      - /^ultron\/.*/ # Ignore ultron/* branches
+      - /^ultron\/.*/  # Ignore ultron/* branches
 
 machine:
   node:
@@ -13,7 +13,9 @@ machine:
     CXX: g++-4.9
 
 dependencies:
-  pre:
+  override:
+    - rm -rf node_modules
+    - npm install
     - sudo pip install yamllint
 
 test:

eve/main.yml

@@ -0,0 +1,43 @@
+---
+version: 0.2
+
+branches:
+  default:
+    stage: pre-merge
+
+stages:
+  pre-merge:
+    worker: &master-worker
+      type: docker
+      path: eve/workers/master
+      volumes:
+        - '/home/eve/workspace'
+    steps:
+      - Git:
+          name: fetch source
+          repourl: '%(prop:git_reference)s'
+          shallow: True
+          retryFetch: True
+          haltOnFailure: True
+      - ShellCommand:
+          name: install dependencies
+          command: npm install
+      - ShellCommand:
+          name: run lint yml
+          command: npm run --silent lint_yml
+      - ShellCommand:
+          name: run lint
+          command: npm run --silent lint -- --max-warnings 0
+      - ShellCommand:
+          name: run lint_md
+          command: npm run --silent lint_md
+      - ShellCommand:
+          name: run test
+          command: npm run --silent test
+      - ShellCommand:
+          name: run ft_test
+          command: npm run ft_test
+      - ShellCommand:
+          name: run executables tests
+          command: npm install && npm test
+          workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/'

eve/workers/master/Dockerfile

@@ -0,0 +1,55 @@
+FROM ubuntu:trusty
+
+#
+# Install apt packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY buildbot_worker_packages.list arsenal_packages.list /tmp/
+RUN apt-get update -q && apt-get -qy install curl apt-transport-https \
+    && apt-get install -qy software-properties-common python-software-properties \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo "deb https://deb.nodesource.com/node_6.x trusty main" > /etc/apt/sources.list.d/nodesource.list \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update -q \
+    && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \
+    && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \
+    && pip install pip==9.0.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/*_packages.list
+
+#
+# Install usefull nodejs dependencies
+#
+
+RUN npm install mocha -g
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION=0.9.12
+RUN pip install yamllint
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+USER eve
+ENV HOME /home/eve
+#
+# Setup nodejs environmnent
+#
+
+ENV CXX=g++-4.9
+ENV LANG C.UTF-8
+
+
+WORKDIR /home/eve/workspace
+CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \
+    && sudo service redis-server start \
+    && buildbot-worker start --nodaemon

eve/workers/master/arsenal_packages.list

@@ -0,0 +1,3 @@
+nodejs
+redis-server
+g++-4.9

eve/workers/master/buildbot_worker_packages.list

@@ -0,0 +1,9 @@
+ca-certificates
+git
+libffi-dev
+libssl-dev
+python2.7
+python2.7-dev
+python-pip
+software-properties-common
+sudo

lib/auth/v4/awsURIencode.js

@@ -32,7 +32,7 @@ function _toHexUTF8(char) {
     return res;
 }
 
-function awsURIencode(input, encodeSlash) {
+function awsURIencode(input, encodeSlash, noEncodeStar) {
     const encSlash = encodeSlash === undefined ? true : encodeSlash;
     let encoded = '';
     for (let i = 0; i < input.length; i++) {
@@ -47,6 +47,8 @@ function awsURIencode(input, encodeSlash) {
             encoded = encoded.concat('%20');
         } else if (ch === '/') {
             encoded = encoded.concat(encSlash ? '%2F' : ch);
+        } else if (ch === '*') {
+            encoded = encoded.concat(noEncodeStar ? '*' : '%2A');
         } else {
             encoded = encoded.concat(_toHexUTF8(ch));
         }

lib/auth/v4/createCanonicalRequest.js

@@ -27,8 +27,16 @@ function createCanonicalRequest(params) {
             payloadChecksum = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b' +
                 '934ca495991b7852b855';
         } else if (pHttpVerb === 'POST') {
+            let notEncodeStar = false;
+            // The java sdk does not encode the '*' parameter to compute the
+            // signature, if the user-agent is recognized, we need to keep
+            // the plain '*' as well.
+            if (/aws-sdk-java\/[0-9.]+/.test(pHeaders['user-agent'])) {
+                notEncodeStar = true;
+            }
             let payload = queryString.stringify(pQuery, null, null, {
-                encodeURIComponent: awsURIencode,
+                encodeURIComponent: input => awsURIencode(input, false,
+                    notEncodeStar),
             });
             payload = payload.replace(/%20/g, '+');
             payloadChecksum = crypto.createHash('sha256')

lib/executables/pensieveCreds/package.json

@@ -7,8 +7,8 @@
     "test": "mocha --recursive --timeout 5500 tests/unit"
   },
   "dependencies": {
+    "mocha": "2.5.3",
     "async": "^2.6.0",
     "node-forge": "^0.7.1"
   }
 }
-

tests/unit/auth/v4/createCanonicalRequest.js

@@ -1,7 +1,8 @@
 'use strict'; // eslint-disable-line strict
 
 const assert = require('assert');
-
+const awsURIencode =
+    require('../../../../lib/auth/v4/awsURIencode');
 const createCanonicalRequest =
     require('../../../../lib/auth/v4/createCanonicalRequest');
 
@@ -45,6 +46,50 @@ describe('createCanonicalRequest function', () => {
         assert.strictEqual(actualOutput, expectedOutput);
     });
 
+    const msg = 'S3C-820: aws java sdk should not encode * ' +
+        'character for signature';
+    it(msg, () => {
+        const doc = JSON.stringify({
+            Statement: [{
+                Action: 's3:*',
+            }],
+        });
+        const params = {
+            pHttpVerb: 'POST',
+            pResource: '/',
+            pQuery: {
+                PolicyDocument: doc,
+            },
+            pHeaders: {
+                'host': 'examplebucket.s3.amazonaws.com',
+                'x-amz-date': '20130524T000000Z',
+                'user-agent': 'aws-sdk-java/1.11',
+                'authorization': 'AWS4-HMAC-SHA256 Credential' +
+                    '=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/' +
+                    's3/aws4_request,SignedHeaders=host;user-agent' +
+                    'x-amz-content-sha256;x-amz-date,Signature=' +
+                    'f0e8bdb87c964420e857bd35b5d6ed310bd44f' +
+                    '0170aba48dd91039c6036bdb41',
+                'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c' +
+                    '8996fb92427ae41e4649b934ca495991b7852b855',
+            },
+            pSignedHeaders: 'host;user-agent;x-amz-content-sha256;x-amz-date',
+        };
+        const expectedOutput = 'POST\n' +
+            '/\n' +
+            `PolicyDocument=${awsURIencode(doc)}\n` +
+            'host:examplebucket.s3.amazonaws.com\n' +
+            'user-agent:aws-sdk-java/1.11\n' +
+            'x-amz-content-sha256:e3b0c44298fc1c149afbf4c' +
+            '8996fb92427ae41e4649b934ca495991b7852b855\n' +
+            'x-amz-date:20130524T000000Z\n\n' +
+            'host;user-agent;x-amz-content-sha256;x-amz-date\n' +
+            '25775fcf6b536b361aadce0c5f1afb46eb945dbdd6c3a7723b18300234a89588';
+        const actualOutput = createCanonicalRequest(params);
+        assert.strictEqual(actualOutput, expectedOutput);
+    });
+
+
     // Example taken from: http://docs.aws.amazon.com/AmazonS3/
     // latest/API/sig-v4-header-based-auth.html
     it('should construct a canonical request in accordance ' +