diff --git a/lib/pact_broker/config/runtime_configuration.rb b/lib/pact_broker/config/runtime_configuration.rb new file mode 100644 index 000000000..37ecfd3c5 --- /dev/null +++ b/lib/pact_broker/config/runtime_configuration.rb @@ -0,0 +1,186 @@ +require "anyway_config" +require "pact_broker/config/runtime_configuration_logging_methods" +require "pact_broker/config/runtime_configuration_database_methods" +require "pact_broker/config/runtime_configuration_coercion_methods" +require "pact_broker/version" + +module PactBroker + module Config + class RuntimeConfiguration < Anyway::Config + include RuntimeConfigurationLoggingMethods + include RuntimeConfigurationDatabaseMethods + include RuntimeConfigurationCoercionMethods + + class << self + def sensitive_values(*values) + @sensitive_values ||= [] + if values + @sensitive_values.concat([*values]) + else + @sensitive_values + end + end + + def sensitive_value?(value) + sensitive_values.any? { |key| key == value || key == value.to_sym || key.kind_of?(Regexp) && key =~ value } + end + end + + DATABASE_ATTRIBUTES = { + database_adapter: "postgres", + database_username: nil, + database_password: nil, + database_name: nil, + database_host: nil, + database_port: nil, + database_url: nil, + database_sslmode: nil, + sql_log_level: :debug, + sql_log_warn_duration: 5, + database_max_connections: nil, + database_pool_timeout: 5, + database_connect_max_retries: 0, + database_timezone: :utc, + auto_migrate_db: true, + auto_migrate_db_data: true, + allow_missing_migration_files: true, + validate_database_connection_config: true, + use_case_sensitive_resource_names: true, + database_statement_timeout: 15, + metrics_sql_statement_timeout: 30, + } + + LOGGING_ATTRIBUTES = { + log_dir: File.expand_path("./log"), + warning_error_class_names: ["Sequel::ForeignKeyConstraintViolation", "PG::QueryCanceled"], + } + + WEBHOOK_ATTRIBUTES = { + webhook_retry_schedule: [10, 60, 120, 300, 600, 1200], #10 sec, 1 min, 2 min, 5 min, 10 min, 20 min => 38 minutes + webhook_http_method_whitelist: ["POST"], + webhook_http_code_success: [200, 201, 202, 203, 204, 205, 206], + webhook_scheme_whitelist: ["https"], + webhook_host_whitelist: [], + disable_ssl_verification: false, + user_agent: "Pact Broker v#{PactBroker::VERSION}", + } + + RESOURCE_ATTRIBUTES = { + port: 9292, + base_url: nil, + base_urls: [], + use_hal_browser: true, + enable_diagnostic_endpoints: true, + use_rack_protection: true, + badge_provider_mode: :proxy, + enable_public_badge_access: false, + shields_io_base_url: "https://img.shields.io", + } + + DOMAIN_ATTRIBUTES = { + order_versions_by_date: true, + base_equality_only_on_content_that_affects_verification_results: true, + check_for_potential_duplicate_pacticipant_names: true, + create_deployed_versions_for_tags: true, + semver_formats: ["%M.%m.%p%s%d", "%M.%m", "%M"] + } + + ALL_ATTRIBUTES = [DATABASE_ATTRIBUTES, LOGGING_ATTRIBUTES, WEBHOOK_ATTRIBUTES, RESOURCE_ATTRIBUTES, DOMAIN_ATTRIBUTES].inject(&:merge) + + def self.getter_and_setter_method_names + ALL_ATTRIBUTES.keys + ALL_ATTRIBUTES.keys.collect{ |k| "#{k}=".to_sym } + [:warning_error_classes, :database_configuration] - [:base_url] + end + + config_name :pact_broker + + attr_config(ALL_ATTRIBUTES) + + sensitive_values(:database_url, :database_password) + + def database_port= database_port + super(database_port&.to_i) + end + + def database_connect_max_retries= database_connect_max_retries + super(database_connect_max_retries&.to_i) + end + + def database_timezone= database_timezone + super(database_timezone&.to_sym) + end + + def sql_log_level= sql_log_level + super(sql_log_level&.downcase&.to_sym) + end + + def sql_log_warn_duration= sql_log_warn_duration + super(sql_log_warn_duration&.to_f) + end + + def badge_provider_mode= badge_provider_mode + super(badge_provider_mode&.to_sym) + end + + def metrics_sql_statement_timeout= metrics_sql_statement_timeout + super(metrics_sql_statement_timeout&.to_i) + end + + def warning_error_class_names= warning_error_class_names + super(value_to_string_array(warning_error_class_names, "warning_error_class_names")) + end + + def semver_formats= semver_formats + super(value_to_string_array(semver_formats, "semver_formats")) + end + + def webhook_retry_schedule= webhook_retry_schedule + super(value_to_integer_array(webhook_retry_schedule, "webhook_retry_schedule")) + end + + def webhook_http_method_whitelist= webhook_http_method_whitelist + super(value_to_string_array(webhook_http_method_whitelist, "webhook_http_method_whitelist")) + end + + def webhook_http_code_success= webhook_http_code_success + super(value_to_integer_array(webhook_http_code_success, "webhook_http_code_success")) + end + + def webhook_scheme_whitelist= webhook_scheme_whitelist + super(value_to_string_array(webhook_scheme_whitelist, "webhook_scheme_whitelist")) + end + + def webhook_host_whitelist= webhook_host_whitelist + super(value_to_string_array(webhook_host_whitelist, "webhook_host_whitelist")) + end + + def base_url= base_url + super(value_to_string_array(base_url, "base_url")) + end + + alias_method :original_base_url, :base_url + + def base_url + raise NotImplementedError + end + + def base_urls= base_urls + super(value_to_string_array(base_urls, "base_urls")) + end + + def base_urls + (super + [*original_base_url]).uniq + end + + def warning_error_classes + warning_error_class_names.collect do | class_name | + begin + Object.const_get(class_name) + rescue NameError => e + puts("Class #{class_name} couldn't be loaded as a warning error class (#{e.class} - #{e.message}). Ignoring.") + nil + end + end.compact + end + end + end +end diff --git a/lib/pact_broker/config/runtime_configuration_coercion_methods.rb b/lib/pact_broker/config/runtime_configuration_coercion_methods.rb new file mode 100644 index 000000000..371f3abef --- /dev/null +++ b/lib/pact_broker/config/runtime_configuration_coercion_methods.rb @@ -0,0 +1,41 @@ +require "pact_broker/config/space_delimited_string_list" +require "pact_broker/config/space_delimited_integer_list" + +module PactBroker + module Config + module RuntimeConfigurationCoercionMethods + def value_to_string_array value, property_name + if value.is_a?(String) + PactBroker::Config::SpaceDelimitedStringList.parse(value) + elsif value.is_a?(Array) + # parse structured values to possible regexp + [*value].flat_map do | val | + if val.is_a?(String) + PactBroker::Config::SpaceDelimitedStringList.parse(val) + else + [val] + end + end + elsif value + raise ConfigurationError.new("Pact Broker configuration property `#{property_name}` must be a space delimited String or an Array. Got: #{value.inspect}") + end + end + + private :value_to_string_array + + def value_to_integer_array value, property_name + if value.is_a?(String) + PactBroker::Config::SpaceDelimitedIntegerList.parse(value) + elsif value.is_a?(Array) + value.collect { |v| v.to_i } + elsif value.is_a?(Integer) + [value] + elsif value + raise ConfigurationError.new("Pact Broker configuration property `#{property_name}` must be a space delimited String or an Array of Integers. Got: #{value.inspect}") + end + end + + private :value_to_integer_array + end + end +end diff --git a/lib/pact_broker/config/runtime_configuration_database_methods.rb b/lib/pact_broker/config/runtime_configuration_database_methods.rb new file mode 100644 index 000000000..cc2f86d18 --- /dev/null +++ b/lib/pact_broker/config/runtime_configuration_database_methods.rb @@ -0,0 +1,65 @@ +module PactBroker + module Config + module RuntimeConfigurationDatabaseMethods + def database_configuration + database_credentials + .merge( + encoding: "utf8", + sslmode: database_sslmode, + sql_log_level: sql_log_level, + log_warn_duration: sql_log_warn_duration, + max_connections: database_max_connections, + pool_timeout: database_pool_timeout, + driver_options: driver_options + ).compact + end + + + def postgres? + database_credentials[:adapter] == "postgres" + end + private :postgres? + + def driver_options + if postgres? + { options: "-c statement_timeout=#{database_statement_timeout}s" } + end + end + private :driver_options + + def database_credentials + if database_url + database_configuration_from_url + else + database_configuration_from_parts + end + end + private :database_credentials + + def database_configuration_from_parts + { + adapter: database_adapter, + user: database_username, + password: database_password, + host: database_host, + database: database_name, + database_port: database_port + }.compact + end + private :database_credentials + + def database_configuration_from_url + uri = URI(database_url) + { + adapter: uri.scheme, + user: uri.user, + password: uri.password, + host: uri.host, + database: uri.path.sub(/^\//, ""), + port: uri.port&.to_i, + }.compact + end + private :database_configuration_from_url + end + end +end diff --git a/lib/pact_broker/config/runtime_configuration_logging_methods.rb b/lib/pact_broker/config/runtime_configuration_logging_methods.rb new file mode 100644 index 000000000..9dc5c6167 --- /dev/null +++ b/lib/pact_broker/config/runtime_configuration_logging_methods.rb @@ -0,0 +1,42 @@ +require "uri" + +module PactBroker + module Config + module RuntimeConfigurationLoggingMethods + def log_configuration(logger) + logger.info "------------------------------------------------------------------------" + logger.info "PACT BROKER CONFIGURATION:" + to_source_trace.sort_by { |key, _| key }.each { |key, value| log_config_inner(key, value, logger) } + logger.info "------------------------------------------------------------------------" + end + + def log_config_inner(key, value, logger) + if !value.has_key? :value + value.sort_by { |inner_key, _| inner_key }.each { |inner_key, inner_value| log_config_inner("#{key}:#{inner_key}", inner_value) } + elsif self.class.sensitive_value?(key) + logger.info "#{key}=#{redact(key, value[:value])} source=[#{value[:source]}]" + else + logger.info "#{key}=#{value[:value]} source=[#{value[:source]}]" + end + end + private :log_config_inner + + def redact name, value + if value && name.to_s.end_with?("_url") + begin + uri = URI(value) + uri.password = "*****" + uri.to_s + rescue StandardError + "*****" + end + elsif !value.nil? + "*****" + else + nil + end + end + private :redact + end + end +end diff --git a/lib/pact_broker/configuration.rb b/lib/pact_broker/configuration.rb index 6341cc283..47a73834a 100644 --- a/lib/pact_broker/configuration.rb +++ b/lib/pact_broker/configuration.rb @@ -1,11 +1,10 @@ require "pact_broker/version" require "pact_broker/error" -require "pact_broker/config/space_delimited_string_list" -require "pact_broker/config/space_delimited_integer_list" require "semantic_logger" +require "forwardable" +require "pact_broker/config/runtime_configuration" module PactBroker - class ConfigurationError < PactBroker::Error; end def self.configuration @@ -18,6 +17,9 @@ def self.reset_configuration end class Configuration + extend Forwardable + + delegate PactBroker::Config::RuntimeConfiguration.getter_and_setter_method_names => :runtime_configuration SAVABLE_SETTING_NAMES = [ :order_versions_by_date, @@ -36,38 +38,29 @@ class Configuration :seed_example_data, :badge_provider_mode, :warning_error_class_names, - :base_url, + :base_urls, :log_dir, :allow_missing_migration_files, :auto_migrate_db_data, :use_rack_protection, - :metrics_sql_statement_timeout, - :create_deployed_versions_for_tags + :create_deployed_versions_for_tags, + :metrics_sql_statement_timeout ] - attr_accessor :base_url, :log_dir, :database_connection, :auto_migrate_db, :auto_migrate_db_data, :allow_missing_migration_files, :example_data_seeder, :seed_example_data, :use_hal_browser, :html_pact_renderer, :use_rack_protection - attr_accessor :validate_database_connection_config, :enable_diagnostic_endpoints, :version_parser, :sha_generator - attr_accessor :use_case_sensitive_resource_names, :order_versions_by_date - attr_accessor :warning_error_class_names - attr_accessor :check_for_potential_duplicate_pacticipant_names - attr_accessor :webhook_retry_schedule - attr_accessor :user_agent - attr_reader :webhook_http_method_whitelist, :webhook_scheme_whitelist, :webhook_host_whitelist, :webhook_http_code_success - attr_accessor :semver_formats - attr_accessor :enable_public_badge_access, :shields_io_base_url, :badge_provider_mode - attr_accessor :disable_ssl_verification + attr_accessor :database_connection + attr_accessor :example_data_seeder, :seed_example_data + attr_accessor :html_pact_renderer, :version_parser, :sha_generator attr_accessor :content_security_policy, :hal_browser_content_security_policy_overrides - attr_accessor :base_equality_only_on_content_that_affects_verification_results - attr_reader :api_error_reporters + attr_accessor :api_error_reporters attr_reader :custom_logger attr_accessor :policy_builder, :policy_scope_builder, :base_resource_class_factory - attr_accessor :metrics_sql_statement_timeout - attr_accessor :create_deployed_versions_for_tags - alias_method :policy_finder=, :policy_builder= alias_method :policy_scope_finder=, :policy_scope_builder= + attr_accessor :runtime_configuration + def initialize + @runtime_configuration = PactBroker::Config::RuntimeConfiguration.new @before_resource_hook = ->(resource){} @after_resource_hook = ->(resource){} @authenticate_with_basic_auth = nil @@ -82,18 +75,6 @@ def self.default_configuration require "pact_broker/pacts/generate_sha" config = Configuration.new - config.log_dir = File.expand_path("./log") - config.auto_migrate_db = true - config.auto_migrate_db_data = true - config.allow_missing_migration_files = false - config.use_rack_protection = true - config.use_hal_browser = true - config.validate_database_connection_config = true - config.enable_diagnostic_endpoints = true - config.enable_public_badge_access = false # For security - config.shields_io_base_url = "https://img.shields.io".freeze - config.badge_provider_mode = :proxy # other option is :redirect - config.use_case_sensitive_resource_names = true config.html_pact_renderer = default_html_pact_render config.version_parser = PactBroker::Versions::ParseSemanticVersion config.sha_generator = PactBroker::Pacts::GenerateSha @@ -102,17 +83,7 @@ def self.default_configuration require "pact_broker/db/seed_example_data" PactBroker::DB::SeedExampleData.call end - config.user_agent = "Pact Broker v#{PactBroker::VERSION}" - config.base_equality_only_on_content_that_affects_verification_results = true - config.order_versions_by_date = true - config.semver_formats = ["%M.%m.%p%s%d", "%M.%m", "%M"] - config.webhook_retry_schedule = [10, 60, 120, 300, 600, 1200] #10 sec, 1 min, 2 min, 5 min, 10 min, 20 min => 38 minutes - config.check_for_potential_duplicate_pacticipant_names = true - config.disable_ssl_verification = false - config.webhook_http_method_whitelist = ["POST"] - config.webhook_http_code_success = [200, 201, 202, 203, 204, 205, 206] - config.webhook_scheme_whitelist = ["https"] - config.webhook_host_whitelist = [] + # TODO get rid of unsafe-inline config.content_security_policy = { script_src: "'self' 'unsafe-inline'", @@ -133,9 +104,6 @@ def self.default_configuration require "pact_broker/api/resources/default_base_resource" PactBroker::Api::Resources::DefaultBaseResource } - config.warning_error_class_names = ["Sequel::ForeignKeyConstraintViolation", "PG::QueryCanceled"] - config.metrics_sql_statement_timeout = 30 - config.create_deployed_versions_for_tags = true config end # rubocop: enable Metrics/MethodLength @@ -149,9 +117,7 @@ def logger= logger end def log_configuration - SAVABLE_SETTING_NAMES.sort.each do | setting | - logger.info "PactBroker.configuration.#{setting}=#{PactBroker.configuration.send(setting).inspect}" - end + runtime_configuration.log_configuration(logger) end def self.default_html_pact_render @@ -243,57 +209,5 @@ def load_from_database! require "pact_broker/config/load" PactBroker::Config::Load.call(self) end - - def webhook_http_method_whitelist= webhook_http_method_whitelist - @webhook_http_method_whitelist = parse_space_delimited_string_list_property("webhook_http_method_whitelist", webhook_http_method_whitelist) - end - - def webhook_http_code_success= webhook_http_code_success - @webhook_http_code_success = parse_space_delimited_integer_list_property("webhook_http_code_success", webhook_http_code_success) - end - - def webhook_scheme_whitelist= webhook_scheme_whitelist - @webhook_scheme_whitelist = parse_space_delimited_string_list_property("webhook_scheme_whitelist", webhook_scheme_whitelist) - end - - def webhook_host_whitelist= webhook_host_whitelist - @webhook_host_whitelist = parse_space_delimited_string_list_property("webhook_host_whitelist", webhook_host_whitelist) - end - - def base_urls - base_url ? base_url.split(" ") : [] - end - - def warning_error_classes - warning_error_class_names.collect do | class_name | - begin - Object.const_get(class_name) - rescue NameError => e - logger.warn("Class #{class_name} couldn't be loaded as a warning error class (#{e.class} - #{e.message}). Ignoring.") - nil - end - end.compact - end - - private - - def parse_space_delimited_string_list_property property_name, property_value - case property_value - when String then Config::SpaceDelimitedStringList.parse(property_value) - when Array then Config::SpaceDelimitedStringList.new(property_value) - else - raise ConfigurationError.new("Pact Broker configuration property `#{property_name}` must be a space delimited String or an Array") - end - end - - def parse_space_delimited_integer_list_property property_name, property_value - case property_value - when String then Config::SpaceDelimitedIntegerList.parse(property_value) - when Array then Config::SpaceDelimitedIntegerList.new(property_value) - else - raise ConfigurationError.new("Pact Broker configuration property `#{property_name}` must be a space delimited String or an Array with Integer values") - end - end - end end diff --git a/lib/pact_broker/db/validate_encoding.rb b/lib/pact_broker/db/validate_encoding.rb index ebd00e84c..f03a0785d 100644 --- a/lib/pact_broker/db/validate_encoding.rb +++ b/lib/pact_broker/db/validate_encoding.rb @@ -3,11 +3,8 @@ module PactBroker module DB - class ConnectionConfigurationError < StandardError; end - class ValidateEncoding - extend PactBroker::Messages def self.call connection @@ -16,7 +13,6 @@ def self.call connection raise ConnectionConfigurationError.new(message("errors.validation.connection_encoding_not_utf8", encoding: encoding.inspect)) end end - end end end diff --git a/lib/pact_broker/initializers/database_connection.rb b/lib/pact_broker/initializers/database_connection.rb new file mode 100644 index 000000000..943cf00e9 --- /dev/null +++ b/lib/pact_broker/initializers/database_connection.rb @@ -0,0 +1,53 @@ +require "sequel" +require "pact_broker/db/log_quietener" + +## +# Sequel by default does not test connections in its connection pool before +# handing them to a client. To enable connection testing you need to load the +# "connection_validator" extension like below. The connection validator +# extension is configurable, by default it only checks connections once per +# hour: +# +# http://sequel.rubyforge.org/rdoc-plugins/files/lib/sequel/extensions/connection_validator_rb.html +# +# +# A gotcha here is that it is not enough to enable the "connection_validator" +# extension, we also need to specify that we want to use the threaded connection +# pool, as noted in the documentation for the extension. +# +# -1 means that connections will be validated every time, which avoids errors +# when databases are restarted and connections are killed. This has a performance +# penalty, so consider increasing this timeout if building a frequently accessed service. + +module PactBroker + def self.create_database_connection(logger, config, max_retries = 0) + logger&.info "Connecting to database with config: #{config.merge(password: "*****")}" + config_with_logger = if config[:sql_log_level] == :none + config.reject { |k, _| k == :sql_log_level } + else + config.merge(logger: PactBroker::DB::LogQuietener.new(logger)) + end + + tries = 0 + max_tries = max_retries + 1 + connection = nil + wait = 3 + + begin + connection = Sequel.connect(config_with_logger) + rescue StandardError => e + if (tries += 1) < max_tries + logger&.info "Error connecting to database (#{e.class}). Waiting #{wait} seconds and trying again. #{max_tries-tries} tries to go." + sleep wait + retry + else + raise e + end + end + logger&.info "Connected to database #{config[:database]}" + connection.extension(:connection_validator) + connection.pool.connection_validation_timeout = -1 + connection.timezone = config[:database_timezone] + connection + end +end diff --git a/pact_broker.gemspec b/pact_broker.gemspec index 78408c6d6..6c9248f5d 100644 --- a/pact_broker.gemspec +++ b/pact_broker.gemspec @@ -65,4 +65,5 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "semantic_logger", "~> 4.3" gem.add_runtime_dependency "sanitize", ">= 5.2.1", "~> 5.2" gem.add_runtime_dependency "wisper", "~> 2.0" + gem.add_runtime_dependency "anyway_config", "~> 2.1" end diff --git a/spec/lib/pact_broker/config/runtime_configuration_spec.rb b/spec/lib/pact_broker/config/runtime_configuration_spec.rb new file mode 100644 index 000000000..aa920866d --- /dev/null +++ b/spec/lib/pact_broker/config/runtime_configuration_spec.rb @@ -0,0 +1,71 @@ +require "pact_broker/config/runtime_configuration" + +module PactBroker + module Config + describe RuntimeConfiguration do + describe "base_url" do + it "does not expose base_url for delegation" do + expect(RuntimeConfiguration.getter_and_setter_method_names).to_not include :base_url + end + + it "does not support the method base_url as base_urls should be used instead" do + expect { RuntimeConfiguration.new.base_url }.to raise_error NotImplementedError + end + end + + context "with a base_url and base_urls as strings" do + subject do + runtime_configuration = RuntimeConfiguration.new + runtime_configuration.base_url = "foo blah" + runtime_configuration.base_urls = "bar wiffle" + runtime_configuration + end + + its(:base_urls) { is_expected.to eq %w[bar wiffle foo blah] } + end + + context "with a base_url and base_urls as the same strings" do + subject do + runtime_configuration = RuntimeConfiguration.new + runtime_configuration.base_url = "foo blah" + runtime_configuration.base_urls = "foo meep" + runtime_configuration + end + + its(:base_urls) { is_expected.to eq %w[foo meep blah] } + end + + context "with just base_url as a string" do + subject do + runtime_configuration = RuntimeConfiguration.new + runtime_configuration.base_url = "foo blah" + runtime_configuration + end + + its(:base_urls) { is_expected.to eq %w[foo blah] } + end + + context "with just base_urls as a string" do + subject do + runtime_configuration = RuntimeConfiguration.new + runtime_configuration.base_url = nil + runtime_configuration.base_urls = "bar wiffle" + runtime_configuration + end + + its(:base_urls) { is_expected.to eq %w[bar wiffle] } + end + + context "with base_url and base_urls as arrays" do + subject do + runtime_configuration = RuntimeConfiguration.new + runtime_configuration.base_url = %w[foo blah] + runtime_configuration.base_urls = %w[bar wiffle] + runtime_configuration + end + + its(:base_urls) { is_expected.to eq %w[bar wiffle foo blah] } + end + end + end +end diff --git a/spec/lib/pact_broker/configuration_spec.rb b/spec/lib/pact_broker/configuration_spec.rb index 2ba34845a..8b6242d70 100644 --- a/spec/lib/pact_broker/configuration_spec.rb +++ b/spec/lib/pact_broker/configuration_spec.rb @@ -55,8 +55,8 @@ module PactBroker end it "allows setting the whitelist by an array" do - PactBroker.configuration.webhook_http_method_whitelist = ["foo"] - expect(PactBroker.configuration.webhook_http_method_whitelist).to be_a Config::SpaceDelimitedStringList + PactBroker.configuration.webhook_http_method_whitelist = ["foo", "/.*/"] + expect(PactBroker.configuration.webhook_http_method_whitelist).to eq ["foo", /.*/] end end @@ -68,7 +68,7 @@ module PactBroker it "allows setting the 'webhook_http_code_success' by an array" do PactBroker.configuration.webhook_http_code_success = [200, 201, 202] - expect(PactBroker.configuration.webhook_http_code_success).to be_a Config::SpaceDelimitedIntegerList + expect(PactBroker.configuration.webhook_http_code_success).to eq [200, 201, 202] end end