From df393163e6941028c61afe9d38245d6ef517ab32 Mon Sep 17 00:00:00 2001 From: Justin Gruber Date: Fri, 3 Feb 2023 09:31:15 -0800 Subject: [PATCH] BS-147 Adding build metadata to omnibus version manifest (#1103) * BS-147 Adding build metadata to omnibus version manifest Signed-off-by: Justin Gruber --- lib/omnibus.rb | 70 +++--- lib/omnibus/build_system_metadata.rb | 36 +++ .../build_system_metadata/buildkite.rb | 129 +++++++++++ lib/omnibus/manifest.rb | 8 +- spec/unit/build_system_metadata_spec.rb | 82 +++++++ spec/unit/buildkite_spec.rb | 207 ++++++++++++++++++ 6 files changed, 497 insertions(+), 35 deletions(-) create mode 100644 lib/omnibus/build_system_metadata.rb create mode 100644 lib/omnibus/build_system_metadata/buildkite.rb create mode 100644 spec/unit/build_system_metadata_spec.rb create mode 100644 spec/unit/buildkite_spec.rb diff --git a/lib/omnibus.rb b/lib/omnibus.rb index ed958f252..eaf063904 100644 --- a/lib/omnibus.rb +++ b/lib/omnibus.rb @@ -34,34 +34,36 @@ module Omnibus # DEFAULT_CONFIG = "omnibus.rb".freeze - autoload :Builder, "omnibus/builder" - autoload :BuildVersion, "omnibus/build_version" - autoload :BuildVersionDSL, "omnibus/build_version_dsl" - autoload :Cleaner, "omnibus/cleaner" - autoload :Compressor, "omnibus/compressor" - autoload :Config, "omnibus/config" - autoload :Error, "omnibus/exceptions" - autoload :FileSyncer, "omnibus/file_syncer" - autoload :Generator, "omnibus/generator" - autoload :GitCache, "omnibus/git_cache" - autoload :HealthCheck, "omnibus/health_check" - autoload :Instrumentation, "omnibus/instrumentation" - autoload :Library, "omnibus/library" - autoload :Logger, "omnibus/logger" - autoload :Logging, "omnibus/logging" - autoload :Metadata, "omnibus/metadata" - autoload :NullArgumentable, "omnibus/null_argumentable" - autoload :Ohai, "omnibus/ohai" - autoload :Package, "omnibus/package" - autoload :Packager, "omnibus/packager" - autoload :Project, "omnibus/project" - autoload :Publisher, "omnibus/publisher" - autoload :Reports, "omnibus/reports" - autoload :S3Cache, "omnibus/s3_cache" - autoload :Software, "omnibus/software" - autoload :Templating, "omnibus/templating" - autoload :ThreadPool, "omnibus/thread_pool" - autoload :Licensing, "omnibus/licensing" + autoload :Builder, "omnibus/builder" + autoload :BuildVersion, "omnibus/build_version" + autoload :BuildVersionDSL, "omnibus/build_version_dsl" + autoload :BuildSystemMetadata, "omnibus/build_system_metadata" + autoload :Buildkite, "omnibus/build_system_metadata/buildkite" + autoload :Cleaner, "omnibus/cleaner" + autoload :Compressor, "omnibus/compressor" + autoload :Config, "omnibus/config" + autoload :Error, "omnibus/exceptions" + autoload :FileSyncer, "omnibus/file_syncer" + autoload :Generator, "omnibus/generator" + autoload :GitCache, "omnibus/git_cache" + autoload :HealthCheck, "omnibus/health_check" + autoload :Instrumentation, "omnibus/instrumentation" + autoload :Library, "omnibus/library" + autoload :Logger, "omnibus/logger" + autoload :Logging, "omnibus/logging" + autoload :Metadata, "omnibus/metadata" + autoload :NullArgumentable, "omnibus/null_argumentable" + autoload :Ohai, "omnibus/ohai" + autoload :Package, "omnibus/package" + autoload :Packager, "omnibus/packager" + autoload :Project, "omnibus/project" + autoload :Publisher, "omnibus/publisher" + autoload :Reports, "omnibus/reports" + autoload :S3Cache, "omnibus/s3_cache" + autoload :Software, "omnibus/software" + autoload :Templating, "omnibus/templating" + autoload :ThreadPool, "omnibus/thread_pool" + autoload :Licensing, "omnibus/licensing" autoload :GitFetcher, "omnibus/fetchers/git_fetcher" autoload :NetFetcher, "omnibus/fetchers/net_fetcher" @@ -77,16 +79,16 @@ module Omnibus autoload :ManifestEntry, "omnibus/manifest_entry" autoload :ManifestDiff, "omnibus/manifest_diff" - autoload :ChangeLog, "omnibus/changelog" - autoload :GitRepository, "omnibus/git_repository" + autoload :ChangeLog, "omnibus/changelog" + autoload :GitRepository, "omnibus/git_repository" autoload :SemanticVersion, "omnibus/semantic_version" module Command - autoload :Base, "omnibus/cli/base" - autoload :Cache, "omnibus/cli/cache" - autoload :Publish, "omnibus/cli/publish" - autoload :ChangeLog, "omnibus/cli/changelog" + autoload :Base, "omnibus/cli/base" + autoload :Cache, "omnibus/cli/cache" + autoload :Publish, "omnibus/cli/publish" + autoload :ChangeLog, "omnibus/cli/changelog" end class << self diff --git a/lib/omnibus/build_system_metadata.rb b/lib/omnibus/build_system_metadata.rb new file mode 100644 index 000000000..fa3e1c772 --- /dev/null +++ b/lib/omnibus/build_system_metadata.rb @@ -0,0 +1,36 @@ +# +# Copyright 2012-2018 Chef Software, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "omnibus/build_system_metadata/buildkite" + +module Omnibus + # Provides methods for fetching Omnibus project build metadata from Buildkite + # + # @note Requires environment variables provided whichever platform metdata is being pulled from + # + class BuildSystemMetadata + + class << self + + def to_hash + if !ENV["BUILDKITE"].nil? && !ENV["BUILDKITE"].empty? + Omnibus::Buildkite.to_hash + end + end + + end + end +end \ No newline at end of file diff --git a/lib/omnibus/build_system_metadata/buildkite.rb b/lib/omnibus/build_system_metadata/buildkite.rb new file mode 100644 index 000000000..9de6147fb --- /dev/null +++ b/lib/omnibus/build_system_metadata/buildkite.rb @@ -0,0 +1,129 @@ +# +# Copyright 2012-2018 Chef Software, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Omnibus + # Provides methods for fetching Omnibus project build metadata from Buildkite + # + # @note Requires environment variables provided by Buildkite + # + class Buildkite + + class << self + + # + # Constants for the buildkite environment variables + # + AMI_ID_ENV_KEY = "BUILDKITE_AGENT_META_DATA_AWS_AMI_ID".freeze + HOSTNAME_ENV_KEY = "BUILDKITE_AGENT_META_DATA_HOSTNAME".freeze + DOCKER_VERSION_ENV_KEY = "BUILDKITE_AGENT_META_DATA_DOCKER".freeze + BUILDKITE_COMMAND_ENV_KEY = "BUILDKITE_COMMAND".freeze + + # + # The AMI ID of the instance the build is happening on. + # + # @note This is only present when the instance is a Windows or Linux instance. + # + # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_AWS_AMI_ID"] in Buildkite build job. + # + # @return [String] + # either the AMI ID, or 'unknown' + # + def ami_id + ami_id = "unknown" + if !ENV[AMI_ID_ENV_KEY].nil? && !ENV[AMI_ID_ENV_KEY].empty? + ami_id = ENV[AMI_ID_ENV_KEY] + end + ami_id + end + + # + # The hostname of the instance the build is happening on. + # + # @note This is only present when the instance is a MacOS instance. + # + # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_HOSTNAME"] in Buildkite build job. + # + # @return [String] + # either the hostname, or 'unknown' + # + def hostname + hostname = "unknown" + if !ENV[HOSTNAME_ENV_KEY].nil? && !ENV[HOSTNAME_ENV_KEY].empty? && ami_id == "unknown" + hostname = ENV[HOSTNAME_ENV_KEY] + end + hostname + end + + # + # A boolean representing if the build is using docker or not. + # + # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_DOCKER"] in Buildkite build job. + # + # @return [Boolean] + # + def is_docker_build + !ENV[DOCKER_VERSION_ENV_KEY].nil? && !ENV[DOCKER_VERSION_ENV_KEY].empty? ? true : false + end + + # + # The version of docker that was used in the build. + # + # @note Relies on presence of ENV["BUILDKITE_AGENT_META_DATA_DOCKER"] in Buildkite build job. + # + # @return [String] + # + def docker_version + ENV[DOCKER_VERSION_ENV_KEY] if is_docker_build + end + + # + # The OS Image that is being used in the Docker build + # + # @note Relies on presence of ENV["BUILDKITE_COMMAND"] in Buildkite build job. + # + # @return [String] + # String with the parameter that was provided in the `docker build` command + # + def docker_image + buildkite_command = ENV[BUILDKITE_COMMAND_ENV_KEY] + if is_docker_build && buildkite_command && buildkite_command.include?("OS_IMAGE") + os_image = buildkite_command.match(/OS_IMAGE=(?[\S]*)/) + os_image[:image_id] + end + end + + # + # The version of Omnibus that is in `version.rb` + # + # @return [String] + # + def omnibus_version + Omnibus::VERSION + end + + def to_hash + ret = {} + ret[:ami_id] = ami_id if ami_id != "unknown" + ret[:hostname] = hostname if hostname != "unknown" + ret[:is_docker_build] = is_docker_build if is_docker_build + ret[:docker_version] = docker_version if docker_version + ret[:docker_image] = docker_image if docker_image + ret[:omnibus_version] = omnibus_version + ret + end + end + end +end \ No newline at end of file diff --git a/lib/omnibus/manifest.rb b/lib/omnibus/manifest.rb index 2042d2204..373f1e239 100644 --- a/lib/omnibus/manifest.rb +++ b/lib/omnibus/manifest.rb @@ -75,10 +75,14 @@ def to_hash memo[k] = h memo end + + build_system_metadata = Omnibus::BuildSystemMetadata.to_hash + ret = { manifest_format: LATEST_MANIFEST_FORMAT, software: software_hash, } + ret[:build_system_metadata] = build_system_metadata if build_system_metadata ret[:build_version] = build_version if build_version ret[:build_git_revision] = build_git_revision if build_git_revision ret[:license] = license @@ -118,7 +122,9 @@ def self.from_hash_v1(manifest_data) end def self.from_hash_v2(manifest_data) - m = Omnibus::Manifest.new(manifest_data[:build_version], manifest_data[:build_git_revision], manifest_data[:license]) + m = Omnibus::Manifest.new(manifest_data[:build_version], + manifest_data[:build_git_revision], + manifest_data[:license]) manifest_data[:software].each do |name, entry_data| m.add(name, Omnibus::ManifestEntry.new(name, keys_to_syms(entry_data))) end diff --git a/spec/unit/build_system_metadata_spec.rb b/spec/unit/build_system_metadata_spec.rb new file mode 100644 index 000000000..65b75d931 --- /dev/null +++ b/spec/unit/build_system_metadata_spec.rb @@ -0,0 +1,82 @@ +require "spec_helper" + +module Omnibus + describe BuildSystemMetadata do + + let(:ami_id) { "ami-1234ab5678cd9ef01" } + let(:hostname) { "CHF-V-ABC-DE000.local" } + let(:docker_version) { "20.0.0" } + let(:docker_image) { "artifactory-url.com/release-type/base-platform" } + let(:docker_command) { "docker build . -f images/omnibus_toolchain_containers/platform/Dockerfile -t artifactory-url.com/release-type/omnibus-toolchain-platform -t chefes/omnibus-toolchain-platform --build-arg OS_IMAGE=#{docker_image}" } + + subject(:buildkite_metadata) { Omnibus::Buildkite } + + describe "#to_hash" do + context "builds occur on buildkite" do + + before(:each) do + clear_defaults + end + + it "returns an ami_id if one is found" do + with_ami_id + expect(buildkite_metadata.to_hash[:ami_id]).to eq(ami_id) + end + + it "returns an hostname if one is found" do + with_hostname + expect(buildkite_metadata.to_hash[:hostname]).to eq(hostname) + end + + it "returns is_docker_build if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:is_docker_build]).to eq(true) + end + + it "returns a docker_version if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:docker_version]).to eq(docker_version) + end + + it "returns a docker_image if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:docker_image]).to eq(docker_image) + end + + it "returns an omnibus_version if one is found" do + expect(buildkite_metadata.to_hash[:omnibus_version]).to eq(Omnibus::VERSION) + end + + end + end + + def with_ami_id + stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", ami_id) + end + + def with_hostname + stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", hostname) + end + + def with_docker + stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", docker_version) + stub_env("BUILDKITE_COMMAND", docker_command) + end + + def clear_defaults + without_ami_id_and_hostname + without_docker + end + + def without_ami_id_and_hostname + stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", nil) + stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", nil) + end + + def without_docker + stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", nil) + stub_env("BUILDKITE_COMMAND", nil) + end + + end +end \ No newline at end of file diff --git a/spec/unit/buildkite_spec.rb b/spec/unit/buildkite_spec.rb new file mode 100644 index 000000000..a31c51a93 --- /dev/null +++ b/spec/unit/buildkite_spec.rb @@ -0,0 +1,207 @@ +require "spec_helper" + +module Omnibus + describe Buildkite do + + let(:ami_id) { "ami-1234ab5678cd9ef01" } + let(:hostname) { "CHF-V-ABC-DE000.local" } + let(:docker_version) { "20.0.0" } + let(:docker_image) { "artifactory-url.com/release-type/base-platform" } + let(:docker_command) { "docker build . -f images/omnibus_toolchain_containers/platform/Dockerfile -t artifactory-url.com/release-type/omnibus-toolchain-platform -t chefes/omnibus-toolchain-platform --build-arg OS_IMAGE=#{docker_image}" } + + subject(:buildkite_metadata) { described_class } + + before(:each) do + clear_defaults + end + + describe "#ami_id" do + it "returns the ami_id if one is found" do + with_ami_id + expect(buildkite_metadata.ami_id).to eq(ami_id) + end + + it "returns the unknown if nothing present" do + expect(buildkite_metadata.ami_id).to eq("unknown") + end + end + + describe "#hostname" do + it "returns the hostname if one is found" do + with_hostname + expect(buildkite_metadata.hostname).to eq(hostname) + end + + it "returns the unknown if nothing present" do + expect(buildkite_metadata.hostname).to eq("unknown") + end + end + + describe "#is_docker_build" do + it "returns true if docker metadata is present" do + with_docker + expect(buildkite_metadata.is_docker_build).to eq(true) + end + + it "returns false if docker metadata is missing" do + expect(buildkite_metadata.is_docker_build).to eq(false) + end + end + + describe "#docker_version" do + it "returns the docker version if one is found" do + with_docker + expect(buildkite_metadata.docker_version).to eq(docker_version) + end + + it "returns nothing if docker version is missing" do + expect(buildkite_metadata.docker_version).to be_nil + end + end + + describe "#docker_image" do + it "returns the docker image id if one is found" do + with_docker + expect(buildkite_metadata.docker_image).to eq(docker_image) + end + + it "returns nothing if docker image id is missing" do + expect(buildkite_metadata.docker_image).to be_nil + end + end + + describe "#omnibus_version" do + it "returns the omnibus_version if one is found" do + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + end + + describe "#to_hash" do + it "returns an ami_id if one is found" do + with_ami_id + expect(buildkite_metadata.to_hash[:ami_id]).to eq(ami_id) + end + + it "returns an hostname if one is found" do + with_hostname + expect(buildkite_metadata.to_hash[:hostname]).to eq(hostname) + end + + it "returns is_docker_build if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:is_docker_build]).to eq(true) + end + + it "returns a docker_version if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:docker_version]).to eq(docker_version) + end + + it "returns a docker_image if one is found" do + with_docker + expect(buildkite_metadata.to_hash[:docker_image]).to eq(docker_image) + end + + it "returns an omnibus_version if one is found" do + expect(buildkite_metadata.to_hash[:omnibus_version]).to eq(Omnibus::VERSION) + end + end + + context "platform builds on linux" do + it "uses docker" do + with_ami_id + with_docker + + expect(buildkite_metadata.ami_id).to eq(ami_id) + expect(buildkite_metadata.is_docker_build).to eq(true) + expect(buildkite_metadata.docker_version).to eq(docker_version) + expect(buildkite_metadata.docker_image).to eq(docker_image) + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + + it "does not use docker" do + with_ami_id + + expect(buildkite_metadata.ami_id).to eq(ami_id) + expect(buildkite_metadata.is_docker_build).to eq(false) + expect(buildkite_metadata.docker_version).to be_nil + expect(buildkite_metadata.docker_image).to be_nil + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + end + + context "platform builds on windows" do + it "uses docker" do + with_ami_id + with_docker + + expect(buildkite_metadata.ami_id).to eq(ami_id) + expect(buildkite_metadata.is_docker_build).to eq(true) + expect(buildkite_metadata.docker_version).to eq(docker_version) + expect(buildkite_metadata.docker_image).to eq(docker_image) + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + + it "does not use docker" do + with_ami_id + + expect(buildkite_metadata.ami_id).to eq(ami_id) + expect(buildkite_metadata.is_docker_build).to eq(false) + expect(buildkite_metadata.docker_version).to be_nil + expect(buildkite_metadata.docker_image).to be_nil + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + end + + describe "platform builds on macOS" do + it "uses docker" do + with_hostname + with_docker + + expect(buildkite_metadata.hostname).to eq(hostname) + expect(buildkite_metadata.is_docker_build).to eq(true) + expect(buildkite_metadata.docker_version).to eq(docker_version) + expect(buildkite_metadata.docker_image).to eq(docker_image) + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + + it "does not use docker" do + with_hostname + + expect(buildkite_metadata.hostname).to eq(hostname) + expect(buildkite_metadata.is_docker_build).to eq(false) + expect(buildkite_metadata.docker_version).to be_nil + expect(buildkite_metadata.docker_image).to be_nil + expect(buildkite_metadata.omnibus_version).to eq(Omnibus::VERSION) + end + end + + def with_ami_id + stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", ami_id) + end + + def with_hostname + stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", hostname) + end + + def with_docker + stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", docker_version) + stub_env("BUILDKITE_COMMAND", docker_command) + end + + def clear_defaults + without_ami_id_and_hostname + without_docker + end + + def without_ami_id_and_hostname + stub_env("BUILDKITE_AGENT_META_DATA_AWS_AMI_ID", nil) + stub_env("BUILDKITE_AGENT_META_DATA_HOSTNAME", nil) + end + + def without_docker + stub_env("BUILDKITE_AGENT_META_DATA_DOCKER", nil) + stub_env("BUILDKITE_COMMAND", nil) + end + end +end \ No newline at end of file