From 60d8074ebdcdf433de6fea227000f8316c27d06d Mon Sep 17 00:00:00 2001 From: Matt Muller <53055821+mullermp@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:05:28 -0400 Subject: [PATCH] Better document notifications feature (#153) --- .rubocop.yml | 5 +- CHANGELOG.md | 2 + README.md | 24 ++++----- lib/aws/rails/notifications.rb | 8 +-- lib/aws/rails/railtie.rb | 51 +++++++++--------- sample-app/README.md | 27 +++++++++- .../app/controllers/users_controller.rb | 3 ++ .../config/initializers/instrument_aws_sdk.rb | 1 + sample_app_old/README.md | 9 ---- .../aws/rails/railtie_test.rb | 53 ++++++++++--------- test/dummy/Gemfile | 4 ++ test/dummy/bin/rails | 4 ++ test/dummy/config/application.rb | 5 +- test/dummy/config/boot.rb | 1 + test/dummy/config/credentials.yml.enc | 1 + test/dummy/config/master.key | 1 + test/test_helper.rb | 2 + 17 files changed, 116 insertions(+), 85 deletions(-) create mode 100644 sample-app/config/initializers/instrument_aws_sdk.rb rename spec/aws/rails/railtie_spec.rb => test/aws/rails/railtie_test.rb (71%) create mode 100644 test/dummy/Gemfile create mode 100644 test/dummy/bin/rails create mode 100644 test/dummy/config/boot.rb create mode 100644 test/dummy/config/credentials.yml.enc create mode 100644 test/dummy/config/master.key diff --git a/.rubocop.yml b/.rubocop.yml index b42a58d1..eda14ef3 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -9,6 +9,7 @@ AllCops: - 'sample-app/**/*' - 'sample_app_old/**/*' - 'spec/dummy/**/*' + - 'test/dummy/**/*' - 'spec/fixtures/**/*' - 'spec/fixtures/**/*' - 'db/**/*' @@ -32,11 +33,13 @@ Style/GlobalVars: Metrics/BlockLength: Exclude: - 'spec/**/*.rb' + - 'test/**/*.rb' - aws-sdk-rails.gemspec Metrics/ModuleLength: Exclude: - 'spec/**/*.rb' + - 'test/**/*.rb' Style/HashSyntax: EnforcedShorthandSyntax: never @@ -45,5 +48,5 @@ Style/Documentation: Exclude: - 'lib/generators/**/*.rb' - 'lib/aws/rails/notifications.rb' - - 'test/**/*.rb' - 'spec/**/*.rb' + - 'test/**/*.rb' diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f411002..7dee150a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Unreleased Changes * Issue - `ActionDispatch::Session::DynamoDbStore` now inherits `ActionDispatch::Session::AbstractStore` by wrapping `Aws::SessionStore::DynamoDB::RackMiddleware`. +* Issue - Do not skip autoload modules for `Aws::Rails.instrument_sdk_operations`. + 4.1.0 (2024-09-27) ------------------ diff --git a/README.md b/README.md index 3f9b9ca0..ff41b826 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,7 @@ class. ### Usage -To use the session store, add or edit your -`config/initializers/session_store.rb` file: +To use the session store, add or edit your `config/initializers/session_store.rb` file: ```ruby options = { table_name: '_your_app_session' } # overrides from YAML or ENV @@ -348,25 +347,22 @@ message['X-SES-FROM-ARN'] = 'arn:aws:ses:us-west-2:012345678910:identity/bigchun message.deliver ``` -## Active Support Notification Instrumentation for AWS SDK calls -To add `ActiveSupport::Notifications` Instrumentation to all AWS SDK client -operations call `Aws::Rails.instrument_sdk_operations` before you construct any -SDK clients. +## Active Support Notifications for AWS SDK calls + +To add `ActiveSupport::Notifications` instrumentation to all AWS SDK client operations, +add or edit your `config/initializers/instrument_aws_sdk.rb` file: -Example usage in `config/initializers/instrument_aws_sdk.rb` ```ruby Aws::Rails.instrument_sdk_operations ``` -Events are published for each client operation call with the following event -name: ..aws. For example, S3's put_object has an event -name of: `put_object.S3.aws`. The service name will always match the -namespace of the service client (eg Aws::S3::Client => 'S3'). -The payload of the event is the +Events are published for each client operation call with the following event name: +`..aws`. For example, S3's `:put_object` has an event name +of: `put_object.S3.aws`. The service name will always match the namespace of the +service client (e.g. Aws::S3::Client => 'S3'). The payload of the event is the [request context](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Seahorse/Client/RequestContext.html). -You can subscribe to these events as you would other - `ActiveSupport::Notifications`: +You can subscribe to these events as you would for other `ActiveSupport::Notifications`: ```ruby ActiveSupport::Notifications.subscribe('put_object.S3.aws') do |name, start, finish, id, payload| diff --git a/lib/aws/rails/notifications.rb b/lib/aws/rails/notifications.rb index ef9566bf..68fabaf4 100644 --- a/lib/aws/rails/notifications.rb +++ b/lib/aws/rails/notifications.rb @@ -5,15 +5,11 @@ module Aws module Rails - # Instruments client operation calls for ActiveSupport::Notifications - # Each client operation will produce an event with name: - # ..aws # @api private class Notifications < Seahorse::Client::Plugin + # This plugin needs to be first, which means it is called first in the stack, + # to start recording time, and returns last def add_handlers(handlers, _config) - # This plugin needs to be first - # which means it is called first in the stack, to start recording time, - # and returns last handlers.add(Handler, step: :initialize, priority: 99) end diff --git a/lib/aws/rails/railtie.rb b/lib/aws/rails/railtie.rb index ee1dd4d4..cc7a0422 100644 --- a/lib/aws/rails/railtie.rb +++ b/lib/aws/rails/railtie.rb @@ -8,21 +8,17 @@ class Railtie < ::Rails::Railtie initializer 'aws-sdk-rails.initialize', before: :load_config_initializers do # Initialization Actions + Aws::Rails.log_to_rails_logger Aws::Rails.use_rails_encrypted_credentials Aws::Rails.add_action_mailer_delivery_method Aws::Rails.add_action_mailer_delivery_method(:sesv2) - Aws::Rails.log_to_rails_logger end initializer 'aws-sdk-rails.insert_middleware' do |app| Aws::Rails.add_sqsd_middleware(app) end - initializer 'aws-sdk-rails.sdk_eager_load' do - config.before_eager_load do - config.eager_load_namespaces << Aws - end - + initializer 'aws-sdk-rails.eager_load' do Aws.define_singleton_method(:eager_load!) do Aws.constants.each do |c| m = Aws.const_get(c) @@ -33,6 +29,10 @@ class Railtie < ::Rails::Railtie end end end + + config.before_eager_load do + config.eager_load_namespaces << Aws + end end rake_tasks do @@ -41,6 +41,20 @@ class Railtie < ::Rails::Railtie end end + # Configures the AWS SDK for Ruby's logger to use the Rails logger. + def self.log_to_rails_logger + Aws.config[:logger] = ::Rails.logger + nil + end + + # Configures the AWS SDK with credentials from Rails encrypted credentials. + def self.use_rails_encrypted_credentials + # limit the config keys we merge to credentials only + aws_credential_keys = %i[access_key_id secret_access_key session_token account_id] + creds = ::Rails.application.credentials[:aws].to_h.slice(*aws_credential_keys) + Aws.config.merge!(creds) + end + # This is called automatically from the SDK's Railtie, but can be manually # called if you want to specify options for building the Aws::SES::Client or # Aws::SESV2::Client. @@ -61,31 +75,14 @@ def self.add_action_mailer_delivery_method(name = :ses, client_options = {}) end end - # Configures the AWS SDK for Ruby's logger to use the Rails logger. - def self.log_to_rails_logger - Aws.config[:logger] = ::Rails.logger - nil - end - - # Configures the AWS SDK with credentials from Rails encrypted credentials. - def self.use_rails_encrypted_credentials - # limit the config keys we merge to credentials only - aws_credential_keys = %i[access_key_id secret_access_key session_token account_id] - creds = ::Rails.application.credentials[:aws].to_h.slice(*aws_credential_keys) - Aws.config.merge!(creds) - end - - # Adds ActiveSupport Notifications instrumentation to AWS SDK - # client operations. Each operation will produce an event with a name: - # ..aws. For example, S3's put_object has an event - # name of: put_object.S3.aws + # Add ActiveSupport Notifications instrumentation to AWS SDK client operations. + # Each operation will produce an event with a name `..aws`. + # For example, S3's put_object has an event name of: put_object.S3.aws def self.instrument_sdk_operations Aws.constants.each do |c| - next if Aws.autoload?(c) - m = Aws.const_get(c) if m.is_a?(Module) && m.const_defined?(:Client) && - m.const_get(:Client).superclass == Seahorse::Client::Base + (client = m.const_get(:Client)) && client.superclass == Seahorse::Client::Base m.const_get(:Client).add_plugin(Aws::Rails::Notifications) end end diff --git a/sample-app/README.md b/sample-app/README.md index 02ec3222..fcd66614 100644 --- a/sample-app/README.md +++ b/sample-app/README.md @@ -30,7 +30,7 @@ Run `EDITOR=nano bundle exec rails credentials:edit` to edit credentials. Commented credentials are defined under the `:aws` key. Uncomment the credentials, which should look like: -``` +```yaml aws: access_key_id: secret secret_access_key: akid @@ -44,6 +44,31 @@ Run `bundle exec rails console` to start the console. Inspect the output of `Aws.config` and ensure the credentials are set. +## ActiveSupport Notifications + +### Setup + +This is configured in `config/initializers/instrument_aws_sdk.rb`. See the `aws-sdk-rails` README. + +`UsersController#index` captures any AWS SDK notification with: + +```ruby +ActiveSupport::Notifications.subscribe(/[.]aws/) do |name, start, finish, id, _payload| + Rails.logger.info "Got notification: #{name} #{start} #{finish} #{id}" +end +``` + +### Testing + +Start the service with `bundle exec rails server` and visit `http://127.0.0.1:3000/users`. + +In the logs, you should at least see a notification for DynamoDB `update_item` from the session store. +It should look like: + +``` +Got notification: update_item.DynamoDB.aws ... +``` + ## DynamoDB Session Store ### Setup diff --git a/sample-app/app/controllers/users_controller.rb b/sample-app/app/controllers/users_controller.rb index 87ca2f98..f2b492be 100644 --- a/sample-app/app/controllers/users_controller.rb +++ b/sample-app/app/controllers/users_controller.rb @@ -3,6 +3,9 @@ class UsersController < ApplicationController # GET /users def index + ActiveSupport::Notifications.subscribe(/[.]aws/) do |name, start, finish, id, _payload| + Rails.logger.info "Got notification: #{name} #{start} #{finish} #{id}" + end @users = User.all end diff --git a/sample-app/config/initializers/instrument_aws_sdk.rb b/sample-app/config/initializers/instrument_aws_sdk.rb new file mode 100644 index 00000000..86167af9 --- /dev/null +++ b/sample-app/config/initializers/instrument_aws_sdk.rb @@ -0,0 +1 @@ +Aws::Rails.instrument_sdk_operations diff --git a/sample_app_old/README.md b/sample_app_old/README.md index 5fa45b06..896062e7 100644 --- a/sample_app_old/README.md +++ b/sample_app_old/README.md @@ -28,15 +28,6 @@ Make sure your email address is verified in SES. Fixture based testing of SES is possible via RSpec request helpers that this gem offers. How to use them is documented within the main README. How to setup inbound emails with SES is also covered there. -## ActiveSupport Notifications - -ActiveSupport notifications for AWS clients are configured in -`config/initializers/instrument_aws_sdk/rb` to log an event -whenever an AWS client makes any service calls. To demo, follow -any one of the ActiveStorage, SES or SQS ActiveJob and the -AWS calls should be logged with: -`Recieved an ActiveSupport::Notification for: send_message.SQS.aws event` - ## SQS ActiveJob * Start rails with `AWS_ACTIVE_JOB_QUEUE_URL=https://my_sqs_queue_url rails server` diff --git a/spec/aws/rails/railtie_spec.rb b/test/aws/rails/railtie_test.rb similarity index 71% rename from spec/aws/rails/railtie_spec.rb rename to test/aws/rails/railtie_test.rb index ea4901f5..435fed22 100644 --- a/spec/aws/rails/railtie_spec.rb +++ b/test/aws/rails/railtie_test.rb @@ -2,24 +2,41 @@ require 'test_helper' +require 'aws-sdk-core' + module Aws - # Test services namespaces - module Service1 - Client = Aws::SES::Client.dup + # Test service for Notifications + # rubocop:disable Lint/EmptyClass + module Service + class Client < Seahorse::Client::Base; end end - module Service2 - Client = Aws::SES::Client.dup + module NotService + class Client; end end + class Client; end + # rubocop:enable Lint/EmptyClass + module Rails describe 'Railtie' do - it 'adds action mailer delivery method' do + it 'uses aws credentials from rails encrypted credentials' do + rails_creds = ::Rails.application.credentials.aws + expect(Aws.config[:access_key_id]).to eq rails_creds[:access_key_id] + expect(Aws.config[:secret_access_key]).to eq rails_creds[:secret_access_key] + expect(Aws.config[:session_token]).to eq rails_creds[:session_token] + expect(Aws.config[:account_id]).to eq rails_creds[:account_id] + + expect(rails_creds[:something]).not_to be_nil + expect(Aws.config[:something]).to be_nil + end + + it 'adds action mailer delivery methods' do expect(ActionMailer::Base.delivery_methods[:ses]).to eq Aws::Rails::SesMailer expect(ActionMailer::Base.delivery_methods[:sesv2]).to eq Aws::Rails::Sesv2Mailer end - it 'sets the Aws logger' do + it 'sets the Rails logger to Aws global config' do expect(Aws.config[:logger]).to eq ::Rails.logger end @@ -28,27 +45,11 @@ module Rails expect(::Rails.application.config.eager_load_namespaces).to include(Aws) end - describe '.use_rails_encrypted_credentials' do - let(:rails_creds) { ::Rails.application.credentials.aws } - - it 'sets aws credentials' do - expect(Aws.config[:access_key_id]).to eq rails_creds[:access_key_id] - expect(Aws.config[:secret_access_key]).to eq rails_creds[:secret_access_key] - end - - it 'does not load non credential keys into aws config' do - expect(rails_creds[:non_credential_key]).not_to be_nil - expect(Aws.config[:non_credential_key]).to be_nil - end - end - describe '.instrument_sdk_operations' do it 'adds the Notifications plugin to sdk clients' do - expect(Aws::Service1::Client).to receive(:add_plugin).with(Aws::Rails::Notifications) - expect(Aws::Service2::Client).to receive(:add_plugin).with(Aws::Rails::Notifications) - - # Ensure other Clients don't get plugin added - allow_any_instance_of(Class).to receive(:add_plugin) + expect(Aws::Service::Client).to receive(:add_plugin).with(Aws::Rails::Notifications) + expect(Aws::NotService::Client).not_to receive(:add_plugin) + expect(Aws::Client).not_to receive(:add_plugin) Aws::Rails.instrument_sdk_operations end diff --git a/test/dummy/Gemfile b/test/dummy/Gemfile new file mode 100644 index 00000000..55d364f6 --- /dev/null +++ b/test/dummy/Gemfile @@ -0,0 +1,4 @@ +source "https://rubygems.org" + +gem "aws-sdk-rails", path: "../../" +gem "rails" diff --git a/test/dummy/bin/rails b/test/dummy/bin/rails new file mode 100644 index 00000000..efc03774 --- /dev/null +++ b/test/dummy/bin/rails @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path("../config/application", __dir__) +require_relative "../config/boot" +require "rails/commands" diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb index f9888ae3..faa8a1a1 100644 --- a/test/dummy/config/application.rb +++ b/test/dummy/config/application.rb @@ -1,13 +1,16 @@ # frozen_string_literal: true require 'rails' +require 'action_controller/railtie' +require 'action_mailer/railtie' + require 'aws-sdk-rails' require 'aws-sessionstore-dynamodb' module Dummy class Application < Rails::Application config.load_defaults Rails::VERSION::STRING.to_f - config.eager_load = false + config.eager_load = true config.secret_key_base = 'secret' end end diff --git a/test/dummy/config/boot.rb b/test/dummy/config/boot.rb new file mode 100644 index 00000000..fab86f17 --- /dev/null +++ b/test/dummy/config/boot.rb @@ -0,0 +1 @@ +require 'bundler/setup' diff --git a/test/dummy/config/credentials.yml.enc b/test/dummy/config/credentials.yml.enc new file mode 100644 index 00000000..27f0ad58 --- /dev/null +++ b/test/dummy/config/credentials.yml.enc @@ -0,0 +1 @@ +6bBVcp2SyvttZcv4ATnSjW73mFUklcvQvoN1atdK0NdmKyNZwas37cSuFSaKFoco/Thf7xrUsiN2cxWYUO9qui0wV5bdTJ5MdpGU05Z9f0jcXp6sPGANc7P/j7GeVFoVVxnNgtIcE4ksx0dQwjlN1/zE7ziihPJPG5OO0g/IEgxs2vMJS1zDLScBmc+P+6D6KjxekjRyeonZ5a+pK9twiHhz0kps7ZS5Jql+7e6sB1SIBWMKiBHc/y5T/BNa0oK0Tj9Y/lMVrNk4O1jXXU1q2YCc7jXDaY8StKXDYUf4GGwkmKM9Ri91s/6+S0FqByV08ZET/u8YEaLYrsj++uSL9sgJ3Kc9AdAoK/oxT9XrdsP0ibm2zx/Jv8GojGpJWuOdMRNshcfCOl2EeQNGMlZhh6gJ+HJZXQcWo+oe4idLtVcn9zd4Lu53uxVZg5PycvKsBfGNbpExnXmG6kBMxY6svdaXvhdmtXHPAIMByu4WnIVb29kWFQ==--Ijdq6LGvh2aVS+yb--Ck2PDvGXuw9Ok9IUfv1N2A== \ No newline at end of file diff --git a/test/dummy/config/master.key b/test/dummy/config/master.key new file mode 100644 index 00000000..f7e59911 --- /dev/null +++ b/test/dummy/config/master.key @@ -0,0 +1 @@ +65a3ce664efa1804645dc56e1759903f \ No newline at end of file diff --git a/test/test_helper.rb b/test/test_helper.rb index 94124a51..53af6d6f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,6 +2,8 @@ require 'minitest/autorun' require 'minitest/unit' +require 'rspec/expectations/minitest_integration' +require 'rspec/mocks/minitest_integration' require 'minitest-spec-rails' ENV['RAILS_ENV'] = 'test'