diff --git a/circle.yml b/circle.yml index b678ef486..b43074834 100644 --- a/circle.yml +++ b/circle.yml @@ -2,7 +2,7 @@ general: branches: ignore: - - /^ultron\/.*/ # Ignore ultron/* branches + - /^ultron\/.*/ # Ignore ultron/* branches machine: node: @@ -13,7 +13,9 @@ machine: CXX: g++-4.9 dependencies: - pre: + override: + - rm -rf node_modules + - npm install - sudo pip install yamllint test: diff --git a/eve/main.yml b/eve/main.yml new file mode 100644 index 000000000..9a2e191cc --- /dev/null +++ b/eve/main.yml @@ -0,0 +1,43 @@ +--- +version: 0.2 + +branches: + default: + stage: pre-merge + +stages: + pre-merge: + worker: &master-worker + type: docker + path: eve/workers/master + volumes: + - '/home/eve/workspace' + steps: + - Git: + name: fetch source + repourl: '%(prop:git_reference)s' + shallow: True + retryFetch: True + haltOnFailure: True + - ShellCommand: + name: install dependencies + command: npm install + - ShellCommand: + name: run lint yml + command: npm run --silent lint_yml + - ShellCommand: + name: run lint + command: npm run --silent lint -- --max-warnings 0 + - ShellCommand: + name: run lint_md + command: npm run --silent lint_md + - ShellCommand: + name: run test + command: npm run --silent test + - ShellCommand: + name: run ft_test + command: npm run ft_test + - ShellCommand: + name: run executables tests + command: npm install && npm test + workdir: '%(prop:builddir)s/build/lib/executables/pensieveCreds/' diff --git a/eve/workers/master/Dockerfile b/eve/workers/master/Dockerfile new file mode 100644 index 000000000..3c307b393 --- /dev/null +++ b/eve/workers/master/Dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:trusty + +# +# Install apt packages needed by the buildchain +# +ENV LANG C.UTF-8 +COPY buildbot_worker_packages.list arsenal_packages.list /tmp/ +RUN apt-get update -q && apt-get -qy install curl apt-transport-https \ + && apt-get install -qy software-properties-common python-software-properties \ + && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ + && echo "deb https://deb.nodesource.com/node_6.x trusty main" > /etc/apt/sources.list.d/nodesource.list \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt-get update -q \ + && cat /tmp/buildbot_worker_packages.list | xargs apt-get install -qy \ + && cat /tmp/arsenal_packages.list | xargs apt-get install -qy \ + && pip install pip==9.0.1 \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /tmp/*_packages.list + +# +# Install usefull nodejs dependencies +# + +RUN npm install mocha -g + +# +# Add user eve +# + +RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \ + && adduser eve sudo \ + && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers + +# +# Run buildbot-worker on startup +# + +ARG BUILDBOT_VERSION=0.9.12 +RUN pip install yamllint +RUN pip install buildbot-worker==$BUILDBOT_VERSION + +USER eve +ENV HOME /home/eve +# +# Setup nodejs environmnent +# + +ENV CXX=g++-4.9 +ENV LANG C.UTF-8 + + +WORKDIR /home/eve/workspace +CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \ + && sudo service redis-server start \ + && buildbot-worker start --nodaemon diff --git a/eve/workers/master/arsenal_packages.list b/eve/workers/master/arsenal_packages.list new file mode 100644 index 000000000..ee70ccbb3 --- /dev/null +++ b/eve/workers/master/arsenal_packages.list @@ -0,0 +1,3 @@ +nodejs +redis-server +g++-4.9 diff --git a/eve/workers/master/buildbot_worker_packages.list b/eve/workers/master/buildbot_worker_packages.list new file mode 100644 index 000000000..06bcf6fc0 --- /dev/null +++ b/eve/workers/master/buildbot_worker_packages.list @@ -0,0 +1,9 @@ +ca-certificates +git +libffi-dev +libssl-dev +python2.7 +python2.7-dev +python-pip +software-properties-common +sudo diff --git a/lib/auth/v4/awsURIencode.js b/lib/auth/v4/awsURIencode.js index c1ba94311..1de515e81 100644 --- a/lib/auth/v4/awsURIencode.js +++ b/lib/auth/v4/awsURIencode.js @@ -32,7 +32,7 @@ function _toHexUTF8(char) { return res; } -function awsURIencode(input, encodeSlash) { +function awsURIencode(input, encodeSlash, noEncodeStar) { const encSlash = encodeSlash === undefined ? true : encodeSlash; let encoded = ''; for (let i = 0; i < input.length; i++) { @@ -47,6 +47,8 @@ function awsURIencode(input, encodeSlash) { encoded = encoded.concat('%20'); } else if (ch === '/') { encoded = encoded.concat(encSlash ? '%2F' : ch); + } else if (ch === '*') { + encoded = encoded.concat(noEncodeStar ? '*' : '%2A'); } else { encoded = encoded.concat(_toHexUTF8(ch)); } diff --git a/lib/auth/v4/createCanonicalRequest.js b/lib/auth/v4/createCanonicalRequest.js index fccaa84cc..bd52e86d1 100644 --- a/lib/auth/v4/createCanonicalRequest.js +++ b/lib/auth/v4/createCanonicalRequest.js @@ -27,8 +27,16 @@ function createCanonicalRequest(params) { payloadChecksum = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b' + '934ca495991b7852b855'; } else if (pHttpVerb === 'POST') { + let notEncodeStar = false; + // The java sdk does not encode the '*' parameter to compute the + // signature, if the user-agent is recognized, we need to keep + // the plain '*' as well. + if (/aws-sdk-java\/[0-9.]+/.test(pHeaders['user-agent'])) { + notEncodeStar = true; + } let payload = queryString.stringify(pQuery, null, null, { - encodeURIComponent: awsURIencode, + encodeURIComponent: input => awsURIencode(input, false, + notEncodeStar), }); payload = payload.replace(/%20/g, '+'); payloadChecksum = crypto.createHash('sha256') diff --git a/lib/executables/pensieveCreds/package.json b/lib/executables/pensieveCreds/package.json index 21250a4cb..b3d92a66d 100644 --- a/lib/executables/pensieveCreds/package.json +++ b/lib/executables/pensieveCreds/package.json @@ -7,8 +7,8 @@ "test": "mocha --recursive --timeout 5500 tests/unit" }, "dependencies": { + "mocha": "2.5.3", "async": "^2.6.0", "node-forge": "^0.7.1" } } - diff --git a/tests/unit/auth/v4/createCanonicalRequest.js b/tests/unit/auth/v4/createCanonicalRequest.js index e965236b1..7e4be3cac 100644 --- a/tests/unit/auth/v4/createCanonicalRequest.js +++ b/tests/unit/auth/v4/createCanonicalRequest.js @@ -1,7 +1,8 @@ 'use strict'; // eslint-disable-line strict const assert = require('assert'); - +const awsURIencode = + require('../../../../lib/auth/v4/awsURIencode'); const createCanonicalRequest = require('../../../../lib/auth/v4/createCanonicalRequest'); @@ -45,6 +46,50 @@ describe('createCanonicalRequest function', () => { assert.strictEqual(actualOutput, expectedOutput); }); + const msg = 'S3C-820: aws java sdk should not encode * ' + + 'character for signature'; + it(msg, () => { + const doc = JSON.stringify({ + Statement: [{ + Action: 's3:*', + }], + }); + const params = { + pHttpVerb: 'POST', + pResource: '/', + pQuery: { + PolicyDocument: doc, + }, + pHeaders: { + 'host': 'examplebucket.s3.amazonaws.com', + 'x-amz-date': '20130524T000000Z', + 'user-agent': 'aws-sdk-java/1.11', + 'authorization': 'AWS4-HMAC-SHA256 Credential' + + '=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/' + + 's3/aws4_request,SignedHeaders=host;user-agent' + + 'x-amz-content-sha256;x-amz-date,Signature=' + + 'f0e8bdb87c964420e857bd35b5d6ed310bd44f' + + '0170aba48dd91039c6036bdb41', + 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c' + + '8996fb92427ae41e4649b934ca495991b7852b855', + }, + pSignedHeaders: 'host;user-agent;x-amz-content-sha256;x-amz-date', + }; + const expectedOutput = 'POST\n' + + '/\n' + + `PolicyDocument=${awsURIencode(doc)}\n` + + 'host:examplebucket.s3.amazonaws.com\n' + + 'user-agent:aws-sdk-java/1.11\n' + + 'x-amz-content-sha256:e3b0c44298fc1c149afbf4c' + + '8996fb92427ae41e4649b934ca495991b7852b855\n' + + 'x-amz-date:20130524T000000Z\n\n' + + 'host;user-agent;x-amz-content-sha256;x-amz-date\n' + + '25775fcf6b536b361aadce0c5f1afb46eb945dbdd6c3a7723b18300234a89588'; + const actualOutput = createCanonicalRequest(params); + assert.strictEqual(actualOutput, expectedOutput); + }); + + // Example taken from: http://docs.aws.amazon.com/AmazonS3/ // latest/API/sig-v4-header-based-auth.html it('should construct a canonical request in accordance ' +