Merge branch 'master' into remeh/corechecks-custom-tags

.github/CODEOWNERS

@@ -36,7 +36,7 @@
 /pkg/util/kubernetes/                   @DataDog/container-integrations @DataDog/burrito
 /pkg/util/retry/                        @DataDog/container-integrations
 
-/pkg/logs/                              @DataDog/ramen-intake
+/pkg/logs/                              @DataDog/logs-intake
 
 /pkg/metadata/ecs/                      @DataDog/burrito
 /pkg/metadata/kubernetes/               @DataDog/burrito

.gitignore

@@ -1,5 +1,5 @@
-# dogstatsd image temporary file
-Dockerfiles/dogstatsd/alpine/dogstatsd
+# dogstatsd image temporary binaries
+Dockerfiles/dogstatsd/alpine/static/
 
 # folders
 vendor/
@@ -29,6 +29,7 @@ datadog.yaml
 network-tracer.yaml
 dogstatsd.yaml
 Dockerfiles/cluster-agent/datadog-cluster.yaml
+Dockerfiles/cluster-agent/dist
 
 # jetbrains IDE directory
 /.idea/

.gitlab-ci.yml

@@ -31,11 +31,13 @@ variables:
   DEB_S3_BUCKET: apt.datad0g.com
   RPM_S3_BUCKET: yum.datad0g.com
   WIN_S3_BUCKET: dd-agent-mstesting
+  ANDROID_S3_BUCKET: dd-agent-androidtesting
   DEB_RPM_BUCKET_BRANCH: nightly  # branch of the DEB_S3_BUCKET and RPM_S3_BUCKET repos to release to, 'nightly' or 'beta'
   DEB_TESTING_S3_BUCKET: apttesting.datad0g.com
   RPM_TESTING_S3_BUCKET: yumtesting.datad0g.com
   WINDOWS_TESTING_S3_BUCKET: $WIN_S3_BUCKET/pipelines/$CI_PIPELINE_ID
   WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds
+  ANDROID_BUILDS_S3_BUCKET: $ANDROID_S3_BUCKET/builds
   DEB_RPM_TESTING_BUCKET_BRANCH: testing  # branch of the DEB_TESTING_S3_BUCKET and RPM_TESTING_S3_BUCKET repos to release to, 'testing'
   DD_REPO_BRANCH_NAME: $CI_COMMIT_REF_NAME
   S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
@@ -225,6 +227,7 @@ cluster_agent-build:
     - inv -e cluster-agent.build
     - $S3_CP_CMD $SRC_PATH/$CLUSTER_AGENT_BINARIES_DIR/datadog-cluster-agent $S3_ARTEFACTS_URI/datadog-cluster-agent
     - $S3_CP_CMD $SRC_PATH/Dockerfiles/cluster-agent/datadog-cluster.yaml $S3_ARTEFACTS_URI/datadog-cluster.yaml
+    - $S3_CP_CMD --recursive $SRC_PATH/$CLUSTER_AGENT_BINARIES_DIR/dist/templates $S3_ARTEFACTS_URI/dist/templates
 
 #
 # integration_test
@@ -357,7 +360,6 @@ agent_rpm-x64:
 
 # build Agent package for rpm-x64
 agent_suse-x64:
-  allow_failure: true
   stage: package_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/suse_x64:latest
   tags: [ "runner:main", "size:2xlarge" ]
@@ -444,12 +446,12 @@ agent_android_apk:
     # for now do the steps manually.  Should eventually move this to an invoke
     # task
     - inv -e android.build
-    - mkdir -p $CI_PROJECT_DIR/apk
-    - cp ./bin/agent/ddagent-release-unsigned.apk $CI_PROJECT_DIR/apk
+    - mkdir -p $OMNIBUS_PACKAGE_DIR
+    - cp ./bin/agent/ddagent-*-unsigned.apk $OMNIBUS_PACKAGE_DIR
   artifacts:
     expire_in: 2 weeks
     paths:
-      - $CI_PROJECT_DIR/apk
+      - $OMNIBUS_PACKAGE_DIR
 
 # build Dogstastd package for deb-x64
 dogstatsd_deb-x64:
@@ -512,7 +514,6 @@ dogstatsd_rpm-x64:
 
 # build Dogstastd package for rpm-x64
 dogstatsd_suse-x64:
-  allow_failure: true
   stage: package_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/suse_x64:latest
   tags: [ "runner:main", "size:large" ]
@@ -588,7 +589,6 @@ deploy_rpm_testing:
 # deploy rpm packages to yum staging repo
 deploy_suse_rpm_testing:
   <<: *run_when_testkitchen_triggered
-  allow_failure: true
   stage: testkitchen_deploy
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
   before_script:
@@ -606,7 +606,6 @@ deploy_suse_rpm_testing:
 # deploy windows packages to our testing bucket
 deploy_windows_testing:
   <<: *run_when_testkitchen_triggered
-  allow_failure: true
   stage: testkitchen_deploy
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
   before_script:
@@ -629,9 +628,10 @@ kitchen_windows:
     - cd $DD_AGENT_TESTING_DIR
     - mkdir $CI_PROJECT_DIR/kitchen_logs
     - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
-    - export TEST_PLATFORMS="win2012,MicrosoftWindowsServer:WindowsServer:2012-Datacenter:3.127.20171115"
-    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2012r2,MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:4.127.20171115"
-    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2016,MicrosoftWindowsServer:WindowsServer:2016-Datacenter:2016.127.20171116"
+    - export TEST_PLATFORMS="win2012,MicrosoftWindowsServer:WindowsServer:2012-Datacenter:3.127.20181122"
+    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2012r2,MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:4.127.20181125"
+    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2016,MicrosoftWindowsServer:WindowsServer:2016-Datacenter:2016.127.20181122"
+    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2019,MicrosoftWindowsServer:WindowsServer:2019-Datacenter:2019.0.20181122"
     - bash -l tasks/run-test-kitchen.sh
   artifacts:
     expire_in: 2 weeks
@@ -652,7 +652,7 @@ kitchen_windows_installer:
     - cd $DD_AGENT_TESTING_DIR
     - mkdir $CI_PROJECT_DIR/kitchen_logs
     - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
-    - export TEST_PLATFORMS="win2012,MicrosoftWindowsServer:WindowsServer:2012-Datacenter:3.127.20171115"
+    - export TEST_PLATFORMS="win2012,MicrosoftWindowsServer:WindowsServer:2012-Datacenter:3.127.20181122"
     - bash -l tasks/run-test-kitchen.sh windows-install-test
   artifacts:
     expire_in: 2 weeks
@@ -954,6 +954,17 @@ check_already_deployed_version:
   script:
     - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh
 
+# If we trigger a build only pipeline we stop here.
+check_if_build_only:
+  <<: *run_when_triggered
+  stage: check_deploy
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
+  before_script:
+    - ls $OMNIBUS_PACKAGE_DIR
+  tags: [ "runner:main", "size:large" ]
+  script:
+    - if [ "$DEB_RPM_BUCKET_BRANCH" == "none" ]; then echo "Stopping pipeline"; exit 1; fi
+
 #
 # deploy
 #
@@ -1017,6 +1028,17 @@ deploy_windows_tags:
     # (when building a custom beta for example).
     - if [ "WINDOWS_DO_NOT_UPDATE_LATEST" != "true" ]; then $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-agent-*-x86_64.msi s3://$WINDOWS_BUILDS_S3_BUCKET/tagged/datadog-agent-6-latest.amd64.msi --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732; fi
 
+# deploy android packages to a public s3 bucket when tagged
+deploy_android_tags:
+  stage: deploy
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
+  before_script:
+    - ls $OMNIBUS_PACKAGE_DIR
+  <<: *run_when_triggered_on_tag
+  tags: [ "runner:main", "size:large" ]
+  script:
+    - $S3_CP_CMD --recursive --exclude "*" --include "*.apk" $OMNIBUS_PACKAGE_DIR s3://$ANDROID_BUILDS_S3_BUCKET/tagged/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
+
 # deploy rpm packages to yum staging repo
 deploy_rpm:
   <<: *run_when_triggered
@@ -1041,7 +1063,6 @@ deploy_rpm:
 # deploy suse rpm packages to yum staging repo
 deploy_suse_rpm:
   <<: *run_when_triggered
-  allow_failure: true
   stage: deploy
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
   before_script:
@@ -1142,6 +1163,7 @@ deploy_cloudfront_invalidate:
   before_script:
     - ls $OMNIBUS_PACKAGE_DIR
   tags: [ "runner:main", "size:large" ]
+  when: always
   script:
     - cd /deploy_scripts/cloudfront-invalidation
     - "REPO=apt PATTERN_SUBSTRING=/$DEB_RPM_BUCKET_BRANCH/ ./invalidate.sh"

CHANGELOG.rst

@@ -2,6 +2,217 @@
 Release Notes
 =============
 
+.. _Release Notes_6.8.3:
+
+6.8.3
+=====
+
+.. _Release Notes_6.8.3_Prelude:
+
+Prelude
+-------
+
+Release on: 2018-12-27
+
+.. _Release Notes_6.8.3_Bug Fixes:
+
+Bug Fixes
+---------
+
+- Fix a bug that could send the last log multiple times when a container was not writing
+  new logs
+
+- Fix a panic that could occur if a container doesn't send logs for more than 30 sec and
+  the timestamp of the last received log is corrupted
+
+.. _Release Notes_6.8.2:
+
+6.8.2
+=====
+
+.. _Release Notes_6.8.2_Prelude:
+
+Prelude
+-------
+
+Release on: 2018-12-19
+
+.. _Release Notes_6.8.2_Bug Fixes:
+
+Bug Fixes
+---------
+
+- Fix a panic that could occur when a container stopped while the agent was reading logs from it.
+
+.. _Release Notes_6.8.1:
+
+6.8.1
+=====
+
+.. _Release Notes_6.8.1_Prelude:
+
+Prelude
+-------
+
+This is a container only release that fixes a bug introduced in ``6.8.0`` that was impacting the kubernetes integration.
+
+Release on: 2018-12-17
+
+- Please refer to the `6.8.1 tag on integrations-core <https://github.com/DataDog/integrations-core/blob/master/AGENT_CHANGELOG.md#datadog-agent-version-681>`_ for the list of changes on the Core Checks.
+
+.. _Release Notes_6.8.1_Bug Fixes:
+
+Bug Fixes
+---------
+
+- Fixes the default ``kubelet`` check configuration that was preventing the kubernetes integration from working properly
+
+.. _Release Notes_6.8.0:
+
+6.8.0
+=====
+
+.. _Release Notes_6.8.0_Prelude:
+
+Prelude
+-------
+
+Please note that a critical bug has been identified in this release that would prevent the kubernetes integration from collecting kubelet metrics on containerized agents.
+The severity of the issue has led us to remove the ``6.8.0`` images on dockerhub and to make the ``latest`` tag point to the ``6.7.0`` release.
+If you have upgraded to this version of the containerized agent we recommend you downgrade to ``6.7.0``. Linux packages are not affected.
+
+Release on: 2018-12-13
+
+- Please refer to the `6.8.0 tag on integrations-core <https://github.com/DataDog/integrations-core/blob/master/AGENT_CHANGELOG.md#datadog-agent-version-680>`_ for the list of changes on the Core Checks.
+
+- Please refer to the `6.8.0 tag on trace-agent <https://github.com/DataDog/datadog-trace-agent/releases/tag/6.8.0>`_ for the list of changes on the Trace Agent.
+
+- Please refer to the `6.8.0 tag on process-agent <https://github.com/DataDog/datadog-process-agent/releases/tag/6.8.0>`_ for the list of changes on the Process Agent.
+
+The Datadog Agent now automatically look for the container short image name to set the default value for the log source and service.
+The source is especially important as it triggers the **automatic configuration of your platform with integration pipeline and facets**.
+The Datadog Agent autodiscovery can still be used to override the default source and service with pod annotations or container labels.
+
+Upgrade Notes
+-------------
+
+- The agent now requires a cluster agent version 1.0+ to establish
+  a valid connection
+- JMX garbage collection metrics ``jvm.gc.cms.count`` and ``jvm.gc.parnew.time`` were renamed to ``jvm.gc.minor_collection_count``, ``jvm.gc.major_collection_count``, ``jvm.gc.minor_collection_time``, ``jvm.gc.major_collection_time`` in 6.6 to be more meaningful. To ensure backward compatibility the change was reverted in this release and the new names put behind a config option. If you started relying on these new names please enable the ``new_gc_metrics`` option in your jmx configurations. An example can be found `here <https://github.com/DataDog/datadog-agent/blob/1aee233a18dedbb8af86da0ce1f2e305206aacf8/cmd/agent/dist/conf.d/jmx.d/conf.yaml.example#L8-L13>`_. This flag will be enabled by default in a future major release.
+
+New Features
+------------
+
+- Enable docker config provider if docker.sock exists
+
+- The new command ``datadog-agent config`` prints the runtime config of the
+  agent.
+
+- Adds eBPF-based network collection component called network-tracer.
+
+- Add diagnosis to the agent for connectivity to the cluster agent
+
+- ``datadog-agent integration install`` command prevents a user from downgrading an integration
+  to a version older than the one shipped by default in the agent.
+
+- Adding kerberos support with libkrb5.
+
+- ``datadog-agent integration install`` command moves configuration files present in
+  the ``data`` directory of the wheel upon successful installation
+
+
+Enhancement Notes
+-----------------
+
+- Adding a default location on Windows for the file storing pointers to make sure we never lose nor duplicate any logs
+
+- Add an option to the `agent check` command to run the check n times
+
+- Set service and source to the docker short image name when container_collect_all flag
+  is enabled and no label or annotation is defined
+
+- Docker: the datadog/dogstatsd image now ships a healthcheck
+
+- Improved consistency of the ECS and Fargate tagging
+
+- Improve logging when python checks use invalid types for tags
+
+- Added a ``region`` tag to Fargate containers, indicating the AWS region
+  they run in
+
+- Adds system.cpu.interrupt, and system.mem.committed, system.mem.paged,
+  system.mem.nonpaged, system.mem.cached metrics on Windows
+
+- Add ``permissions.log`` file to the flare archive.
+
+- Add an agent go-routine dump to the flare as reported
+  by the built-in pprof runtime profiling interface.
+
+- The agent can now expose its healthcheck on a dedicated http port.
+  The Kubernetes daemonset uses this by defaut, on port 5555.
+
+- It's possible now to have different poll intervals for
+  each autodiscovery configuration providers
+
+- Improve Windows Event parsing. Event.EventData.Data fields are parsed as one JSON object. Event.EventData.Binary field
+  is parsed to its string value
+
+- Rename the Windows Event "#text" field to "value". This fixes the facet
+  creation of those fields
+
+- Add a ``status.log`` and a ``config-check.log`` with a basic message in the flare
+  if the agent is not running or is unreachable.
+
+- Added support for wildcards to `DD_KUBERNETES_POD_LABELS_AS_TAGS`. For example,
+  `DD_KUBERNETES_POD_LABELS_AS_TAGS='{"*":"kube_%%label%%"}'` will all pod labels as
+  tags to your metrics with tags names prefixed by `kube_`.
+
+Deprecation Notes
+-----------------
+
+- Removed support for logs_config.tcp_forward_port as it's no longer needed for other integrations.
+
+
+Bug Fixes
+---------
+
+- Configure error log when failing to run docker inspect to read as debug instead, as this log is duplicated by the tagger.
+
+- Fix a bug where `datadog-agent integration` users could not test the
+  `--in-toto` flag due to a filesystem permission issue.
+
+- The cluster agent client init now fails as expected if the
+  cluster agent URL is not valid
+
+- Print correct error when the ``datadog-agent integration`` command fails after installing an integration
+
+- Fix build failure on 32bit armv7
+
+- Fix a bug with Docker logging driver where logs would not be tailed after a log
+  rotation when the option `--log-opt max-file=1` was set.
+
+- Display the correct timezone name in the status page.
+
+- On Windows, the agent now properly computes the location of ProgramData for
+  configuration files instead of using hardcoded values
+
+
+Other Notes
+-----------
+
+- JMXFetch upgraded to 0.23.0. See https://github.com/DataDog/jmxfetch/releases/tag/0.23.0
+
+- On linux, use the cgo dns resolver instead of the golang one. The will make
+  the agent use glibc to resolve hostnames and should give more predictable
+  results.
+
+- Starting with this Agent release, all the Datadog integrations that are installed
+  with the ``datadog-agent integration install`` command are reset to their
+  default versions when the Agent is upgraded.
+  This guarantees the integrity of the embedded python environment after the upgrade.
+
+- The ``datadog-agent integration`` command is now in Beta.
+
 .. _Release Notes_6.7.0:
 
 6.7.0

Dockerfiles/agent/Dockerfile

@@ -73,6 +73,8 @@ RUN if [ -n "$WITH_JMX" ]; then apt-get update \
 RUN apt-get update \
   # CVE-fixing time!
   && apt-get install -y util-linux ncurses-bin ncurses-base libncursesw5:amd64 \
+  # https://security-tracker.debian.org/tracker/CVE-2018-15686
+  && apt-get install -y libudev1 libsystemd0 \
   # https://security-tracker.debian.org/tracker/CVE-2016-2779
   && rm -f /usr/sbin/runuser \
   # https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-6954