BS-147 Adding build metadata to omnibus version manifest (#1103)

* BS-147 Adding build metadata to omnibus version manifest

Signed-off-by: Justin Gruber <[email protected]>

lib/omnibus.rb

@@ -34,34 +34,36 @@ module Omnibus
   #
   DEFAULT_CONFIG = "omnibus.rb".freeze
 
-  autoload :Builder,          "omnibus/builder"
-  autoload :BuildVersion,     "omnibus/build_version"
-  autoload :BuildVersionDSL,  "omnibus/build_version_dsl"
-  autoload :Cleaner,          "omnibus/cleaner"
-  autoload :Compressor,       "omnibus/compressor"
-  autoload :Config,           "omnibus/config"
-  autoload :Error,            "omnibus/exceptions"
-  autoload :FileSyncer,       "omnibus/file_syncer"
-  autoload :Generator,        "omnibus/generator"
-  autoload :GitCache,         "omnibus/git_cache"
-  autoload :HealthCheck,      "omnibus/health_check"
-  autoload :Instrumentation,  "omnibus/instrumentation"
-  autoload :Library,          "omnibus/library"
-  autoload :Logger,           "omnibus/logger"
-  autoload :Logging,          "omnibus/logging"
-  autoload :Metadata,         "omnibus/metadata"
-  autoload :NullArgumentable, "omnibus/null_argumentable"
-  autoload :Ohai,             "omnibus/ohai"
-  autoload :Package,          "omnibus/package"
-  autoload :Packager,         "omnibus/packager"
-  autoload :Project,          "omnibus/project"
-  autoload :Publisher,        "omnibus/publisher"
-  autoload :Reports,          "omnibus/reports"
-  autoload :S3Cache,          "omnibus/s3_cache"
-  autoload :Software,         "omnibus/software"
-  autoload :Templating,       "omnibus/templating"
-  autoload :ThreadPool,       "omnibus/thread_pool"
-  autoload :Licensing,        "omnibus/licensing"
+  autoload :Builder,              "omnibus/builder"
+  autoload :BuildVersion,         "omnibus/build_version"
+  autoload :BuildVersionDSL,      "omnibus/build_version_dsl"
+  autoload :BuildSystemMetadata,  "omnibus/build_system_metadata"
+  autoload :Buildkite,            "omnibus/build_system_metadata/buildkite"
+  autoload :Cleaner,              "omnibus/cleaner"
+  autoload :Compressor,           "omnibus/compressor"
+  autoload :Config,               "omnibus/config"
+  autoload :Error,                "omnibus/exceptions"
+  autoload :FileSyncer,           "omnibus/file_syncer"
+  autoload :Generator,            "omnibus/generator"
+  autoload :GitCache,             "omnibus/git_cache"
+  autoload :HealthCheck,          "omnibus/health_check"
+  autoload :Instrumentation,      "omnibus/instrumentation"
+  autoload :Library,              "omnibus/library"
+  autoload :Logger,               "omnibus/logger"
+  autoload :Logging,              "omnibus/logging"
+  autoload :Metadata,             "omnibus/metadata"
+  autoload :NullArgumentable,     "omnibus/null_argumentable"
+  autoload :Ohai,                 "omnibus/ohai"
+  autoload :Package,              "omnibus/package"
+  autoload :Packager,             "omnibus/packager"
+  autoload :Project,              "omnibus/project"
+  autoload :Publisher,            "omnibus/publisher"
+  autoload :Reports,              "omnibus/reports"
+  autoload :S3Cache,              "omnibus/s3_cache"
+  autoload :Software,             "omnibus/software"
+  autoload :Templating,           "omnibus/templating"
+  autoload :ThreadPool,           "omnibus/thread_pool"
+  autoload :Licensing,            "omnibus/licensing"
 
   autoload :GitFetcher,  "omnibus/fetchers/git_fetcher"
   autoload :NetFetcher,  "omnibus/fetchers/net_fetcher"
@@ -77,16 +79,16 @@ module Omnibus
   autoload :ManifestEntry, "omnibus/manifest_entry"
   autoload :ManifestDiff,  "omnibus/manifest_diff"
 
-  autoload :ChangeLog, "omnibus/changelog"
-  autoload :GitRepository, "omnibus/git_repository"
+  autoload :ChangeLog,      "omnibus/changelog"
+  autoload :GitRepository,  "omnibus/git_repository"
 
   autoload :SemanticVersion, "omnibus/semantic_version"
 
   module Command
-    autoload :Base,    "omnibus/cli/base"
-    autoload :Cache,   "omnibus/cli/cache"
-    autoload :Publish, "omnibus/cli/publish"
-    autoload :ChangeLog, "omnibus/cli/changelog"
+    autoload :Base,       "omnibus/cli/base"
+    autoload :Cache,      "omnibus/cli/cache"
+    autoload :Publish,    "omnibus/cli/publish"
+    autoload :ChangeLog,  "omnibus/cli/changelog"
   end
 
   class << self

lib/omnibus/build_system_metadata.rb

@@ -0,0 +1,36 @@
+#
+# Copyright 2012-2018 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "omnibus/build_system_metadata/buildkite"
+
+module Omnibus
+  # Provides methods for fetching Omnibus project build metadata from Buildkite
+  #
+  # @note Requires environment variables provided whichever platform metdata is being pulled from
+  #
+  class BuildSystemMetadata
+
+    class << self
+
+      def to_hash
+        if !ENV["BUILDKITE"].nil? && !ENV["BUILDKITE"].empty?
+          Omnibus::Buildkite.to_hash
+        end
+      end
+
+    end
+  end
+end

lib/omnibus/build_system_metadata/buildkite.rb

@@ -0,0 +1,129 @@
+#
+# Copyright 2012-2018 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  # Provides methods for fetching Omnibus project build metadata from Buildkite
+  #
+  # @note Requires environment variables provided by Buildkite
+  #
+  class Buildkite
+
+    class << self
+
+      #
+      # Constants for the buildkite environment variables
+      #
+      AMI_ID_ENV_KEY = "BUILDKITE_AGENT_META_DATA_AWS_AMI_ID".freeze
+      HOSTNAME_ENV_KEY = "BUILDKITE_AGENT_META_DATA_HOSTNAME".freeze
+      DOCKER_VERSION_ENV_KEY = "BUILDKITE_AGENT_META_DATA_DOCKER".freeze
+      BUILDKITE_COMMAND_ENV_KEY = "BUILDKITE_COMMAND".freeze
+
+      #
+      # The AMI ID of the instance the build is happening on.
+      #
+      # @note This is only present when the instance is a Windows or Linux instance.
+      #
+      # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_AWS_AMI_ID"] in Buildkite build job.
+      #
+      # @return [String]
+      #   either the AMI ID, or 'unknown'
+      #
+      def ami_id
+        ami_id = "unknown"
+        if !ENV[AMI_ID_ENV_KEY].nil? && !ENV[AMI_ID_ENV_KEY].empty?
+          ami_id = ENV[AMI_ID_ENV_KEY]
+        end
+        ami_id
+      end
+
+      #
+      # The hostname of the instance the build is happening on.
+      #
+      # @note This is only present when the instance is a MacOS instance.
+      #
+      # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_HOSTNAME"] in Buildkite build job.
+      #
+      # @return [String]
+      #   either the hostname, or 'unknown'
+      #
+      def hostname
+        hostname = "unknown"
+        if !ENV[HOSTNAME_ENV_KEY].nil? && !ENV[HOSTNAME_ENV_KEY].empty? && ami_id == "unknown"
+          hostname = ENV[HOSTNAME_ENV_KEY]
+        end
+        hostname
+      end
+
+      #
+      # A boolean representing if the build is using docker or not.
+      #
+      # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_DOCKER"] in Buildkite build job.
+      #
+      # @return [Boolean]
+      #
+      def is_docker_build
+        !ENV[DOCKER_VERSION_ENV_KEY].nil? && !ENV[DOCKER_VERSION_ENV_KEY].empty? ? true : false
+      end
+
+      #
+      # The version of docker that was used in the build.
+      #
+      # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_DOCKER"] in Buildkite build job.
+      #
+      # @return [String]
+      #
+      def docker_version
+        ENV[DOCKER_VERSION_ENV_KEY] if is_docker_build
+      end
+
+      #
+      # The OS Image that is being used in the Docker build
+      #
+      # @note Relies on presence of ENV["BUILDKITE_COMMAND"] in Buildkite build job.
+      #
+      # @return [String]
+      #   String with the parameter that was provided in the `docker build` command
+      #
+      def docker_image
+        buildkite_command = ENV[BUILDKITE_COMMAND_ENV_KEY]
+        if is_docker_build && buildkite_command && buildkite_command.include?("OS_IMAGE")
+          os_image = buildkite_command.match(/OS_IMAGE=(?<image_id>[\S]*)/)
+          os_image[:image_id]
+        end
+      end
+
+      #
+      # The version of Omnibus that is in `version.rb`
+      #
+      # @return [String]
+      #
+      def omnibus_version
+        Omnibus::VERSION
+      end
+
+      def to_hash
+        ret = {}
+        ret[:ami_id] = ami_id if ami_id != "unknown"
+        ret[:hostname] = hostname if hostname != "unknown"
+        ret[:is_docker_build] = is_docker_build if is_docker_build
+        ret[:docker_version] = docker_version if docker_version
+        ret[:docker_image] = docker_image if docker_image
+        ret[:omnibus_version] = omnibus_version
+        ret
+      end
+    end
+  end
+end

lib/omnibus/manifest.rb

@@ -75,10 +75,14 @@ def to_hash
         memo[k] = h
         memo
       end
+
+      build_system_metadata = Omnibus::BuildSystemMetadata.to_hash
+
       ret = {
         manifest_format: LATEST_MANIFEST_FORMAT,
         software: software_hash,
       }
+      ret[:build_system_metadata] = build_system_metadata if build_system_metadata
       ret[:build_version] = build_version if build_version
       ret[:build_git_revision] = build_git_revision if build_git_revision
       ret[:license] = license
@@ -118,7 +122,9 @@ def self.from_hash_v1(manifest_data)
     end
 
     def self.from_hash_v2(manifest_data)
-      m = Omnibus::Manifest.new(manifest_data[:build_version], manifest_data[:build_git_revision], manifest_data[:license])
+      m = Omnibus::Manifest.new(manifest_data[:build_version],
+                                 manifest_data[:build_git_revision],
+                                 manifest_data[:license])
       manifest_data[:software].each do |name, entry_data|
         m.add(name, Omnibus::ManifestEntry.new(name, keys_to_syms(entry_data)))
       end

spec/unit/build_system_metadata_spec.rb

@@ -0,0 +1,82 @@
+require "spec_helper"
+
+module Omnibus
+  describe BuildSystemMetadata do
+
+    let(:ami_id) { "ami-1234ab5678cd9ef01" }
+    let(:hostname) { "CHF-V-ABC-DE000.local" }
+    let(:docker_version) { "20.0.0" }
+    let(:docker_image) { "artifactory-url.com/release-type/base-platform" }
+    let(:docker_command) { "docker build . -f images/omnibus_toolchain_containers/platform/Dockerfile -t artifactory-url.com/release-type/omnibus-toolchain-platform -t chefes/omnibus-toolchain-platform --build-arg OS_IMAGE=#{docker_image}" }
+
+    subject(:buildkite_metadata) { Omnibus::Buildkite }
+
+    describe "#to_hash" do
+      context "builds occur on buildkite" do
+
+        before(:each) do
+          clear_defaults
+        end
+
+        it "returns an ami_id if one is found" do
+          with_ami_id
+          expect(buildkite_metadata.to_hash[:ami_id]).to eq(ami_id)
+        end
+
+        it "returns an hostname if one is found" do
+          with_hostname
+          expect(buildkite_metadata.to_hash[:hostname]).to eq(hostname)
+        end
+
+        it "returns is_docker_build if one is found" do
+          with_docker
+          expect(buildkite_metadata.to_hash[:is_docker_build]).to eq(true)
+        end
+
+        it "returns a docker_version if one is found" do
+          with_docker
+          expect(buildkite_metadata.to_hash[:docker_version]).to eq(docker_version)
+        end
+
+        it "returns a docker_image if one is found" do
+          with_docker
+          expect(buildkite_metadata.to_hash[:docker_image]).to eq(docker_image)
+        end
+
+        it "returns an omnibus_version if one is found" do
+          expect(buildkite_metadata.to_hash[:omnibus_version]).to eq(Omnibus::VERSION)
+        end
+
+      end
+    end
+
+    def with_ami_id
+      stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", ami_id)
+    end
+
+    def with_hostname
+      stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", hostname)
+    end
+
+    def with_docker
+      stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", docker_version)
+      stub_env("BUILDKITE_COMMAND", docker_command)
+    end
+
+    def clear_defaults
+      without_ami_id_and_hostname
+      without_docker
+    end
+
+    def without_ami_id_and_hostname
+      stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", nil)
+      stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", nil)
+    end
+
+    def without_docker
+      stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", nil)
+      stub_env("BUILDKITE_COMMAND", nil)
+    end
+
+  end
+end