diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 65d0125880..fbd95f4778 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -460,6 +460,7 @@ def generateStepForWindows(Map params = [:]){ "ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}", "CASSANDRA_HOST=${linuxIp}", "ES_HOST=${linuxIp}", + "LOCALSTACK_HOST=${linuxIp}", "MEMCACHED_HOST=${linuxIp}", "MONGODB_HOST=${linuxIp}", "MSSQL_HOST=${linuxIp}", diff --git a/.ci/docker/docker-compose-all.yml b/.ci/docker/docker-compose-all.yml index 145221ee6a..7727d2c7c5 100644 --- a/.ci/docker/docker-compose-all.yml +++ b/.ci/docker/docker-compose-all.yml @@ -33,6 +33,10 @@ services: extends: file: docker-compose.yml service: memcached + localstack: + extends: + file: docker-compose.yml + service: localstack node_tests: extends: file: docker-compose-node-test.yml @@ -54,6 +58,8 @@ services: condition: service_healthy memcached: condition: service_healthy + localstack: + condition: service_healthy volumes: nodepgdata: @@ -68,3 +74,5 @@ volumes: driver: local nodecassandradata: driver: local + nodelocalstackdata: + driver: local diff --git a/.ci/docker/docker-compose-edge.yml b/.ci/docker/docker-compose-edge.yml index 8a4535dea0..feb27dd063 100644 --- a/.ci/docker/docker-compose-edge.yml +++ b/.ci/docker/docker-compose-edge.yml @@ -9,6 +9,10 @@ services: extends: file: docker-compose.yml service: elasticsearch + localstack: + extends: + file: docker-compose.yml + service: localstack memcached: extends: file: docker-compose.yml @@ -42,6 +46,8 @@ services: condition: service_healthy elasticsearch: condition: service_healthy + localstack: + condition: service_healthy memcached: condition: service_healthy mongodb: @@ -64,6 +70,8 @@ volumes: driver: local nodemysqldata: driver: local + nodelocalstackdata: + driver: local nodeesdata: driver: local nodecassandradata: diff --git a/.ci/docker/docker-compose-localstack.yml b/.ci/docker/docker-compose-localstack.yml new file mode 100644 index 0000000000..a13d7cc0ff --- /dev/null +++ b/.ci/docker/docker-compose-localstack.yml @@ -0,0 +1,14 @@ +version: '2.1' + +services: + localstack: + extends: + file: docker-compose.yml + service: localstack + node_tests: + extends: + file: docker-compose-node-test.yml + service: node_tests + depends_on: + localstack: + condition: service_healthy diff --git a/.ci/docker/docker-compose-node-edge-test.yml b/.ci/docker/docker-compose-node-edge-test.yml index f162da3d37..ad6bac01f7 100644 --- a/.ci/docker/docker-compose-node-edge-test.yml +++ b/.ci/docker/docker-compose-node-edge-test.yml @@ -22,6 +22,7 @@ services: PGHOST: 'postgres' PGUSER: 'postgres' MEMCACHED_HOST: 'memcached' + LOCALSTACK_HOST: 'localstack' NODE_VERSION: ${NODE_VERSION} NVM_NODEJS_ORG_MIRROR: ${NVM_NODEJS_ORG_MIRROR} ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS} diff --git a/.ci/docker/docker-compose-node-test.yml b/.ci/docker/docker-compose-node-test.yml index 94353c5c4e..a2e5403fca 100644 --- a/.ci/docker/docker-compose-node-test.yml +++ b/.ci/docker/docker-compose-node-test.yml @@ -20,6 +20,7 @@ services: PGHOST: 'postgres' PGUSER: 'postgres' MEMCACHED_HOST: 'memcached' + LOCALSTACK_HOST: 'localstack' NODE_VERSION: ${NODE_VERSION} TAV: ${TAV_MODULE} ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS} diff --git a/.ci/docker/docker-compose.yml b/.ci/docker/docker-compose.yml index c18847b987..76d24116af 100644 --- a/.ci/docker/docker-compose.yml +++ b/.ci/docker/docker-compose.yml @@ -114,6 +114,22 @@ services: timeout: 10s retries: 5 + localstack: + # https://hub.docker.com/r/localstack/localstack/tags + image: localstack/localstack:0.12.12 + environment: + - LOCALSTACK_SERVICES=s3 + - DATA_DIR=/var/lib/localstack + ports: + - "4566:4566" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4566/health"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - nodelocalstackdata:/var/lib/localstack + volumes: nodepgdata: driver: local @@ -127,3 +143,5 @@ volumes: driver: local nodecassandradata: driver: local + nodelocalstackdata: + driver: local diff --git a/.ci/scripts/docker-test.sh b/.ci/scripts/docker-test.sh index 5bc312476a..53fe0c0624 100755 --- a/.ci/scripts/docker-test.sh +++ b/.ci/scripts/docker-test.sh @@ -15,7 +15,7 @@ fi # Skip for node v8 because it results in this warning: # openssl config failed: error:25066067:DSO support routines:DLFCN_LOAD:could not load the shared library if [[ $major_node_version -gt 8 ]]; then - export NODE_OPTIONS="${NODE_OPTIONS:+${NODE_OPTIONS}} --openssl-config=./test/openssl-config-for-testing.cnf" + export NODE_OPTIONS="${NODE_OPTIONS:+${NODE_OPTIONS}} --openssl-config=$(pwd)/test/openssl-config-for-testing.cnf" fi # Workaround to git <2.7 diff --git a/.ci/scripts/test.sh b/.ci/scripts/test.sh index 2881192c7e..26af1d4ee6 100755 --- a/.ci/scripts/test.sh +++ b/.ci/scripts/test.sh @@ -200,6 +200,9 @@ elif [[ -n "${TAV_MODULE}" ]]; then memcached) DOCKER_COMPOSE_FILE=docker-compose-memcached.yml ;; + aws-sdk) + DOCKER_COMPOSE_FILE=docker-compose-localstack.yml + ;; *) # Just the "node_tests" container. No additional services needed for testing. DOCKER_COMPOSE_FILE=docker-compose-node-test.yml @@ -210,6 +213,8 @@ else DOCKER_COMPOSE_FILE=docker-compose-all.yml fi +ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS:-true} + set +e NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR} \ ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS} \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aae1a02604..c7945f482e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,6 +105,16 @@ jobs: volumes: - nodeesdata:/usr/share/elasticsearch/data + localstack: + image: localstack/localstack:0.12.12 + env: + LOCALSTACK_SERVICES: 's3' + DATA_DIR: '/var/lib/localstack' + ports: + - "4566:4566" + volumes: + - nodelocalstackdata:/var/lib/localstack + strategy: matrix: node: diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 46d2ff5fdc..4ffbbc6aa9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -36,6 +36,9 @@ Notes: [float] ===== Features +* Add instrumentation of all AWS S3 methods when using the + https://www.npmjs.com/package/aws-sdk[JavaScript AWS SDK v2] (`aws-sdk`). + [float] ===== Bug fixes diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index 4dfbd148e3..4f2d649241 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -72,7 +72,7 @@ The Node.js agent will automatically instrument the following modules to give yo [options="header"] |======================================================================= |Module |Version |Note -|https://www.npmjs.com/package/aws-sdk[aws-sdk] |>1 <3 |Will instrument SQS send/receive/delete messages +|https://www.npmjs.com/package/aws-sdk[aws-sdk] |>1 <3 |Will instrument SQS send/receive/delete messages, all S3 methods |https://www.npmjs.com/package/cassandra-driver[cassandra-driver] |>=3.0.0 |Will instrument all queries |https://www.npmjs.com/package/elasticsearch[elasticsearch] |>=8.0.0 |Will instrument all queries |https://www.npmjs.com/package/@elastic/elasticsearch[@elastic/elasticsearch] |>=7.0.0 <8.0.0 |Will instrument all queries diff --git a/lib/agent.js b/lib/agent.js index 1b7b73507b..0877320d5c 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -321,6 +321,7 @@ const EMPTY_OPTS = {} // // Usage: // captureError(err, opts, cb) +// captureError(err, opts) // captureError(err, cb) // // where: diff --git a/lib/instrumentation/modules/aws-sdk.js b/lib/instrumentation/modules/aws-sdk.js index 67c133c1fe..f5af25b22d 100644 --- a/lib/instrumentation/modules/aws-sdk.js +++ b/lib/instrumentation/modules/aws-sdk.js @@ -1,16 +1,23 @@ 'use strict' const semver = require('semver') const shimmer = require('../shimmer') +const { instrumentationS3 } = require('./aws-sdk/s3') const { instrumentationSqs } = require('./aws-sdk/sqs') +const instrumentorFromSvcId = { + s3: instrumentationS3, + sqs: instrumentationSqs +} + // Called in place of AWS.Request.send and AWS.Request.promise // // Determines which amazon service an API request is for // and then passes call on to an appropriate instrumentation // function. function instrumentOperation (orig, origArguments, request, AWS, agent, { version, enabled }) { - if (request.service.serviceIdentifier === 'sqs') { - return instrumentationSqs(orig, origArguments, request, AWS, agent, { version, enabled }) + const instrumentor = instrumentorFromSvcId[request.service.serviceIdentifier] + if (instrumentor) { + return instrumentor(orig, origArguments, request, AWS, agent, { version, enabled }) } // if we're still here, then we still need to call the original method diff --git a/lib/instrumentation/modules/aws-sdk/s3.js b/lib/instrumentation/modules/aws-sdk/s3.js new file mode 100644 index 0000000000..b305adcb16 --- /dev/null +++ b/lib/instrumentation/modules/aws-sdk/s3.js @@ -0,0 +1,129 @@ +'use strict' + +// Instrument AWS S3 operations via the 'aws-sdk' package. + +const constants = require('../../../constants') + +const TYPE = 'storage' +const SUBTYPE = 's3' + +// Return the PascalCase operation name from `request.operation` by undoing to +// `lowerFirst()` from +// https://github.com/aws/aws-sdk-js/blob/c0c44b8a4e607aae521686898f39a3e359f727e4/lib/model/api.js#L63-L65 +// +// For example: 'headBucket' -> 'HeadBucket' +function opNameFromOperation (operation) { + return operation[0].toUpperCase() + operation.slice(1) +} + +// Return an APM "resource" string for the bucket, Access Point ARN, or Outpost +// ARN. ARNs are normalized to a shorter resource name. +// +// Known ARN patterns: +// - arn:aws:s3:::accesspoint/ +// - arn:aws:s3-outposts:::outpost//bucket/ +// - arn:aws:s3-outposts:::outpost//accesspoint/ +// +// In general that is: +// arn:$partition:$service:$region:$accountId:$resource +// +// This parses using the same "split on colon" used by the JavaScript AWS SDK v3. +// https://github.com/aws/aws-sdk-js-v3/blob/v3.18.0/packages/util-arn-parser/src/index.ts#L14-L37 +function resourceFromBucket (bucket) { + let resource = null + if (bucket) { + resource = bucket + if (resource.startsWith('arn:')) { + resource = bucket.split(':').slice(5).join(':') + } + } + return resource +} + +// Instrument an awk-sdk@2.x operation (i.e. a AWS.Request.send or +// AWS.Request.promise). +// +// @param {AWS.Request} request https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Request.html +function instrumentationS3 (orig, origArguments, request, AWS, agent, { version, enabled }) { + const opName = opNameFromOperation(request.operation) + let name = 'S3 ' + opName + const resource = resourceFromBucket(request.params && request.params.Bucket) + if (resource) { + name += ' ' + resource + } + + const span = agent.startSpan(name, TYPE, SUBTYPE, opName) + if (span) { + request.on('complete', function onComplete (response) { + // `response` is an AWS.Response + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Response.html + + // Determining the bucket's region. + // `request.httpRequest.region` isn't documented, but the aws-sdk@2 + // lib/services/s3.js will set it to the bucket's determined region. + // This can be asynchronously determined -- e.g. if it differs from the + // configured service endpoint region -- so this won't be set until + // 'complete'. + const region = request.httpRequest && request.httpRequest.region + + // Destination context. + // '.httpRequest.endpoint' might differ from '.service.endpoint' if + // the bucket is in a different region. + const endpoint = request.httpRequest && request.httpRequest.endpoint + const destContext = { + service: { + name: SUBTYPE, + type: TYPE + } + } + if (endpoint) { + destContext.address = endpoint.hostname + destContext.port = endpoint.port + } + if (resource) { + destContext.service.resource = resource + } + if (region) { + destContext.cloud = { region } + } + span.setDestinationContext(destContext) + + if (response) { + // Follow the spec for HTTP client span outcome. + // https://github.com/elastic/apm/blob/master/specs/agents/tracing-instrumentation-http.md#outcome + // + // For example, a S3 GetObject conditional request (e.g. using the + // IfNoneMatch param) will respond with response.error=NotModifed and + // statusCode=304. This is a *successful* outcome. + const statusCode = response.httpResponse && response.httpResponse.statusCode + if (statusCode) { + span._setOutcomeFromHttpStatusCode(statusCode) + } else { + // `statusCode` will be undefined for errors before sending a request, e.g.: + // InvalidConfiguration: Custom endpoint is not compatible with access point ARN + span._setOutcomeFromErrorCapture(constants.OUTCOME_FAILURE) + } + + if (response.error && (!statusCode || statusCode >= 400)) { + agent.captureError(response.error, { skipOutcome: true }) + } + } + + // Workaround a bug in the agent's handling of `span.sync`. + // + // The bug: Currently this span.sync is not set `false` because there is + // an HTTP span created (for this S3 request) in the same async op. That + // HTTP span becomes the "active span" for this async op, and *it* gets + // marked as sync=false in `before()` in async-hooks.js. + span.sync = false + + span.end() + }) + } + + return orig.apply(request, origArguments) +} + +module.exports = { + instrumentationS3 +} diff --git a/package.json b/package.json index 5f55c55a7f..e9eaa6efa9 100644 --- a/package.json +++ b/package.json @@ -183,6 +183,7 @@ "thunky": "^1.1.0", "typescript": "^3.7.5", "untildify": "^4.0.0", + "vasync": "^2.2.0", "wait-on": "^3.3.0", "ws": "^7.2.1" }, diff --git a/test/stacktraces/_mock_apm_server.js b/test/_mock_apm_server.js similarity index 100% rename from test/stacktraces/_mock_apm_server.js rename to test/_mock_apm_server.js diff --git a/test/docker-compose.ci.yml b/test/docker-compose.ci.yml index 4fb6f3cb16..dfe0748a34 100644 --- a/test/docker-compose.ci.yml +++ b/test/docker-compose.ci.yml @@ -3,7 +3,7 @@ version: '2.1' services: node_tests: image: node:${NODE_VERSION} - environment: + environment: MONGODB_HOST: 'mongodb' REDIS_HOST: 'redis' ES_HOST: 'elasticsearch' @@ -11,6 +11,7 @@ services: MYSQL_HOST: 'mysql' CASSANDRA_HOST: 'cassandra' MEMCACHED_HOST: 'memcached' + LOCALSTACK_HOST: 'localstack' PGHOST: 'postgres' PGUSER: 'postgres' depends_on: @@ -30,3 +31,5 @@ services: condition: service_started memcached: condition: service_started + localstack: + condition: service_started diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 00f031628d..f007cd8583 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -4,7 +4,7 @@ services: postgres: user: postgres image: postgres:9.6 - ports: + ports: - "5432:5432" volumes: - nodepgdata:/var/lib/postgresql/data @@ -20,7 +20,7 @@ services: mongodb: image: mongo - ports: + ports: - "27017:27017" volumes: - nodemongodata:/data/db @@ -50,7 +50,7 @@ services: image: mysql:5.7 environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 - ports: + ports: - "3306:3306" volumes: - nodemysqldata:/var/lib/mysql @@ -114,6 +114,22 @@ services: timeout: 10s retries: 5 + localstack: + # https://hub.docker.com/r/localstack/localstack/tags + image: localstack/localstack:0.12.12 + environment: + - "LOCALSTACK_SERVICES=s3" + - "DATA_DIR=/var/lib/localstack" + ports: + - "4566:4566" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4566/health"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - nodelocalstackdata:/var/lib/localstack + volumes: nodepgdata: driver: local @@ -127,3 +143,5 @@ volumes: driver: local nodecassandradata: driver: local + nodelocalstackdata: + driver: local diff --git a/test/instrumentation/modules/aws-sdk/fixtures/use-s3.js b/test/instrumentation/modules/aws-sdk/fixtures/use-s3.js new file mode 100644 index 0000000000..6a5778fcb8 --- /dev/null +++ b/test/instrumentation/modules/aws-sdk/fixtures/use-s3.js @@ -0,0 +1,278 @@ +'use strict' + +// Run a single scenario of using the S3 client (callback style) with APM +// enabled. This is used to test that the expected APM events are generated. +// It writes log.info (in ecs-logging format, see +// https://github.com/trentm/go-ecslog#install) for each S3 client API call. +// +// This script can also be used for manual testing of APM instrumentation of S3 +// against a real S3 account. This can be useful because tests are done against +// https://github.com/localstack/localstack that *simulates* S3 with imperfect +// fidelity. +// +// Auth note: By default this uses the AWS profile/configuration from the +// environment. If you do not have that configured (i.e. do not have +// "~/.aws/...") files, then you can still use localstack via setting: +// unset AWS_PROFILE +// export AWS_ACCESS_KEY_ID=fake +// export AWS_SECRET_ACCESS_KEY=fake +// See also: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html +// +// Usage: +// # Run against the default configured AWS profile, creating a new bucket +// # and deleting it afterwards. +// node use-s3.js | ecslog +// +// # Testing against localstack. +// docker run --rm -it -e SERVICES=s3 -p 4566:4566 localstack/localstack +// TEST_ENDPOINT=http://localhost:4566 node use-s3.js | ecslog +// +// # Use TEST_BUCKET_NAME to re-use an existing bucket (and not delete it). +// # For safety the bucket name must start with "elasticapmtest-bucket-". +// TEST_BUCKET_NAME=elasticapmtest-bucket-1 node use-s3.js | ecslog +// +// Output from a sample run is here: +// https://gist.github.com/trentm/c402bcab8c0571f26d879ec0bcf5759c + +const apm = require('../../../../..').start({ + serviceName: 'use-s3', + captureExceptions: false, + centralConfig: false, + metricsInterval: 0, + cloudProvider: 'none', + captureSpanStackTraces: false, + stackTraceLimit: 4, // get it smaller for reviewing output + logLevel: 'info' +}) + +const crypto = require('crypto') +const vasync = require('vasync') +const AWS = require('aws-sdk') + +const TEST_BUCKET_NAME_PREFIX = 'elasticapmtest-bucket-' + +// ---- support functions + +// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html +function useS3 (s3Client, bucketName, cb) { + const region = s3Client.config.region + const log = apm.logger.child({ + 'event.module': 'app', + endpoint: s3Client.config.endpoint, + bucketName, + region + }) + const key = 'aDir/aFile.txt' + const content = 'hi there' + + vasync.pipeline({ + arg: {}, + funcs: [ + // Limitation: this doesn't handle paging. + function listAllBuckets (arg, next) { + s3Client.listBuckets({}, function (err, data) { + log.info({ err, data }, 'listBuckets') + if (err) { + next(err) + } else { + arg.bucketIsPreexisting = data.Buckets.some(b => b.Name === bucketName) + next() + } + }) + }, + + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#createBucket-property + function createTheBucketIfNecessary (arg, next) { + if (arg.bucketIsPreexisting) { + next() + return + } + + s3Client.createBucket({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: region + } + }, function (err, data) { + // E.g. data: {"Location": "http://trentm-play-s3-bukkit2.s3.amazonaws.com/"} + log.info({ err, data }, 'createBucket') + next(err) + }) + }, + + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#bucketExists-waiter + function waitForBucketToExist (_, next) { + s3Client.waitFor('bucketExists', { Bucket: bucketName }, function (err, data) { + log.info({ err, data }, 'waitFor bucketExists') + next(err) + }) + }, + + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property + function createObj (_, next) { + var md5 = crypto.createHash('md5').update(content).digest('base64') + s3Client.putObject({ + Bucket: bucketName, + Key: key, + ContentType: 'text/plain', + Body: content, + ContentMD5: md5 + }, function (err, data) { + // data.ETag should match a hexdigest md5 of body. + log.info({ err, data }, 'putObject') + next(err) + }) + }, + + function waitForObjectToExist (_, next) { + s3Client.waitFor('objectExists', { + Bucket: bucketName, + Key: key + }, function (err, data) { + log.info({ err, data }, 'waitFor objectExists') + next(err) + }) + }, + + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property + function getObj (_, next) { + s3Client.getObject({ + Bucket: bucketName, + Key: key + }, function (err, data) { + log.info({ err, data }, 'getObject') + next(err) + }) + }, + + function getObjConditionalGet (_, next) { + const md5hex = crypto.createHash('md5').update(content).digest('hex') + const etag = `"${md5hex}"` + s3Client.getObject({ + IfNoneMatch: etag, + Bucket: bucketName, + Key: key + }, function (err, data) { + log.info({ err, data }, 'getObject conditional get') + // Expect a 'NotModified' error, statusCode=304. + if (err && err.code === 'NotModified') { + next() + } else if (err) { + next(err) + } else { + next(new Error('expected NotModified error for conditional request')) + } + }) + }, + + function getObjUsingPromise (_, next) { + const req = s3Client.getObject({ + Bucket: bucketName, + Key: key + }).promise() + + req.then( + function onResolve (data) { + log.info({ data }, 'getObject using Promise, resolve') + next() + }, + function onReject (err) { + log.info({ err }, 'getObject using Promise, reject') + next(err) + } + ) + }, + + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#deleteObject-property + function deleteTheObj (_, next) { + s3Client.deleteObject({ + Bucket: bucketName, + Key: key + }, function (err, data) { + log.info({ err, data }, 'deleteObject') + next(err) + }) + }, + + function deleteTheBucketIfCreatedIt (arg, next) { + if (arg.bucketIsPreexisting) { + next() + return + } + + s3Client.deleteBucket({ + Bucket: bucketName + }, function (err, data) { + log.info({ err, data }, 'deleteBucket') + next(err) + }) + } + ] + }, function (err) { + if (err) { + log.error({ err }, 'unexpected error using S3') + } + cb(err) + }) +} + +// Return a timestamp of the form YYYYMMDDHHMMSS, which can be used in an S3 +// bucket name: +// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html +function getTimestamp () { + return (new Date()).toISOString().split('.')[0].replace(/[^0-9]/g, '') +} + +// ---- mainline + +function main () { + // Config vars. + const region = process.env.TEST_REGION || 'us-east-2' + const endpoint = process.env.TEST_ENDPOINT || null + const bucketName = process.env.TEST_BUCKET_NAME || TEST_BUCKET_NAME_PREFIX + getTimestamp() + + // Guard against any bucket name being used because we will be creating and + // deleting objects in it, and potentially *deleting* the bucket. + if (!bucketName.startsWith(TEST_BUCKET_NAME_PREFIX)) { + throw new Error(`cannot use bucket name "${bucketName}", it must start with ${TEST_BUCKET_NAME_PREFIX}`) + } + + const s3Client = new AWS.S3({ + apiVersion: '2006-03-01', + region, + endpoint, + // In Jenkins CI the endpoint is "http://localstack:4566", which points to + // a "localstack" docker container on the same network as the container + // running tests. The aws-sdk S3 client defaults to "bucket style" URLs, + // i.e. "http://$bucketName.localstack:4566/$key". This breaks with: + // UnknownEndpoint: Inaccessible host: `mahbukkit.localstack'. This service may not be available in the `us-east-2' region. + // at Request.ENOTFOUND_ERROR (/app/node_modules/aws-sdk/lib/event_listeners.js:530:46) + // ... + // originalError: Error: getaddrinfo ENOTFOUND mahbukkit.localstack + // at GetAddrInfoReqWrap.onlookup [as oncomplete] (dns.js:66:26) { + // errno: 'ENOTFOUND', + // code: 'NetworkingError', + // syscall: 'getaddrinfo', + // hostname: 'mahbukkit.localstack', + // + // It *works* with common localstack usage where the endpoint uses + // *localhost*, because "$subdomain.localhost" DNS resolution still resolves + // to 127.0.0.1. + // + // The work around is to force the client to use "path-style" URLs, e.g.: + // http://localstack:4566/$bucketName/$key + s3ForcePathStyle: true + }) + + // Ensure an APM transaction so spans can happen. + const tx = apm.startTransaction('manual') + useS3(s3Client, bucketName, function (err) { + if (err) { + tx.setOutcome('failure') + } + tx.end() + process.exitCode = err ? 1 : 0 + }) +} + +main() diff --git a/test/instrumentation/modules/aws-sdk/s3.test.js b/test/instrumentation/modules/aws-sdk/s3.test.js new file mode 100644 index 0000000000..f97da6441f --- /dev/null +++ b/test/instrumentation/modules/aws-sdk/s3.test.js @@ -0,0 +1,289 @@ +'use strict' + +// Test S3 instrumentation of the 'aws-sdk' module. +// +// Note that this uses localstack for testing, which mimicks the S3 API but +// isn't identical. Some known limitations: +// - It basically does nothing with regions, so testing bucket region discovery +// isn't possible. +// - AFAIK localstack does not support Access Points, so access point ARNs +// cannot be tested. + +const { execFile } = require('child_process') + +const tape = require('tape') + +const { MockAPMServer } = require('../../../_mock_apm_server') + +const LOCALSTACK_HOST = process.env.LOCALSTACK_HOST || 'localhost' +const endpoint = 'http://' + LOCALSTACK_HOST + ':4566' + +// Execute 'node fixtures/use-s3js' and assert APM server gets the expected +// spans. +tape.test('simple S3 usage scenario', function (t) { + const server = new MockAPMServer() + server.start(function (serverUrl) { + const additionalEnv = { + ELASTIC_APM_SERVER_URL: serverUrl, + AWS_ACCESS_KEY_ID: 'fake', + AWS_SECRET_ACCESS_KEY: 'fake', + TEST_BUCKET_NAME: 'elasticapmtest-bucket-1', + TEST_ENDPOINT: endpoint, + TEST_REGION: 'us-east-2' + } + t.comment('executing test script with this env: ' + JSON.stringify(additionalEnv)) + execFile( + process.execPath, + ['fixtures/use-s3.js'], + { + cwd: __dirname, + timeout: 10000, // sanity guard on the test hanging + env: Object.assign({}, process.env, additionalEnv) + }, + function done (err, stdout, stderr) { + t.error(err, 'use-s3.js errored out') + if (err) { + t.comment(`use-s3.js stdout:\n${stdout}\n`) + t.comment(`use-s3.js stderr:\n${stderr}\n`) + } + t.ok(server.events[0].metadata, 'APM server got event metadata object') + + // Sort the events by timestamp, then work through each expected span. + const events = server.events.slice(1) + events.sort((a, b) => { + const aTimestamp = (a.transaction || a.span || {}).timestamp + const bTimestamp = (b.transaction || b.span || {}).timestamp + return aTimestamp < bTimestamp ? -1 : 1 + }) + + // First the transaction. + t.ok(events[0].transaction, 'got the transaction') + const tx = events.shift().transaction + t.equal(events.filter(e => e.span).length, events.length, + 'all remaining events are spans') + + // Currently HTTP spans under each S3 span are included. Eventually + // those will be excluded. Filter those out for now. + // https://github.com/elastic/apm-agent-nodejs/issues/2125 + const spans = events.map(e => e.span).filter(e => e.subtype !== 'http') + + // Compare some common fields across all spans. + t.equal(spans.filter(s => s.trace_id === tx.trace_id).length, + spans.length, 'all spans have the same trace_id') + t.equal(spans.filter(s => s.transaction_id === tx.id).length, + spans.length, 'all spans have the same transaction_id') + t.equal(spans.filter(s => s.outcome === 'success').length, + spans.length, 'all spans have outcome="success"') + t.equal(spans.filter(s => s.sync === false).length, + spans.length, 'all spans have sync=false') + t.equal(spans.filter(s => s.sample_rate === 1).length, + spans.length, 'all spans have sample_rate=1') + spans.forEach(s => { + // Remove variable and common fields to facilitate t.deepEqual below. + delete s.id + delete s.transaction_id + delete s.parent_id + delete s.trace_id + delete s.timestamp + delete s.duration + delete s.outcome + delete s.sync + delete s.sample_rate + }) + + // Work through each of the pipeline functions (listAppBuckets, + // createTheBucketIfNecessary, ...) in the script: + t.deepEqual(spans.shift(), { + name: 'S3 ListBuckets', + type: 'storage', + subtype: 's3', + action: 'ListBuckets', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { name: 's3', type: 'storage' }, + cloud: { region: 'us-east-2' } + } + } + }, 'listAllBuckets produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 CreateBucket elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'CreateBucket', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'createTheBucketIfNecessary produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 HeadBucket elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'HeadBucket', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'waitForBucketToExist produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 PutObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'PutObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'createObj produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 HeadObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'HeadObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'waitForObjectToExist produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 GetObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'GetObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'getObj produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 GetObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'GetObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'getObjConditionalGet produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 GetObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'GetObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'getObjUsingPromise produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 DeleteObject elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'DeleteObject', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'deleteTheObj produced expected span') + + t.deepEqual(spans.shift(), { + name: 'S3 DeleteBucket elasticapmtest-bucket-1', + type: 'storage', + subtype: 's3', + action: 'DeleteBucket', + context: { + destination: { + address: LOCALSTACK_HOST, + port: 4566, + service: { + name: 's3', + type: 'storage', + resource: 'elasticapmtest-bucket-1' + }, + cloud: { region: 'us-east-2' } + } + } + }, 'deleteTheBucketIfCreatedIt produced expected span') + + t.equal(spans.length, 0, 'all spans accounted for') + + server.close() + t.end() + } + ) + }) +}) diff --git a/test/instrumentation/modules/aws-sdk/sqs.js b/test/instrumentation/modules/aws-sdk/sqs.js index e0a45d1263..e9b7575d98 100644 --- a/test/instrumentation/modules/aws-sdk/sqs.js +++ b/test/instrumentation/modules/aws-sdk/sqs.js @@ -2,6 +2,7 @@ const agent = require('../../../..').start({ serviceName: 'test', secretToken: 'test', + cloudProvider: 'none', captureExceptions: false, metricsInterval: 0, centralConfig: false diff --git a/test/script/local-deps-start.sh b/test/script/local-deps-start.sh index 216a1c66a9..8cdeb1d18e 100755 --- a/test/script/local-deps-start.sh +++ b/test/script/local-deps-start.sh @@ -7,3 +7,8 @@ cassandra -p /tmp/cassandra.pid &> /tmp/cassandra.log memcached -d -P /tmp/memcached.pid redis-server /usr/local/etc/redis.conf --daemonize yes mysql.server start + +# Note: Running a "local" (i.e. outside of Docker) localstack is deprecated/not +# supported. So we run it in Docker. +docker run --name dev-localstack -d --rm -e SERVICES=s3 -p 4566:4566 localstack/localstack + diff --git a/test/script/local-deps-stop.sh b/test/script/local-deps-stop.sh index cf5cfb14f2..aed2755783 100755 --- a/test/script/local-deps-stop.sh +++ b/test/script/local-deps-stop.sh @@ -7,3 +7,4 @@ kill `cat /tmp/cassandra.pid` kill `cat /tmp/memcached.pid` redis-cli shutdown mysql.server stop +docker stop dev-localstack diff --git a/test/stacktraces/stacktraces.test.js b/test/stacktraces/stacktraces.test.js index 7e24a5e6c4..0120d89962 100644 --- a/test/stacktraces/stacktraces.test.js +++ b/test/stacktraces/stacktraces.test.js @@ -9,7 +9,7 @@ const path = require('path') const tape = require('tape') const logging = require('../../lib/logging') -const { MockAPMServer } = require('./_mock_apm_server') +const { MockAPMServer } = require('../_mock_apm_server') const { gatherStackTrace, stackTraceFromErrStackString } = require('../../lib/stacktraces') const log = logging.createLogger('off')