From f1feea34fae469dc1701afeff5b2a05988497c7f Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 7 Sep 2022 17:28:50 +0200 Subject: [PATCH 01/35] MONGOID-5445 Add load_async to criteria --- lib/mongoid.rb | 10 +++ lib/mongoid/config.rb | 10 +++ lib/mongoid/contextual.rb | 5 ++ lib/mongoid/contextual/mongo.rb | 22 +++++- .../contextual/mongo/async_load_task.rb | 74 +++++++++++++++++++ 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 lib/mongoid/contextual/mongo/async_load_task.rb diff --git a/lib/mongoid.rb b/lib/mongoid.rb index 22816bd9e2..4700639598 100644 --- a/lib/mongoid.rb +++ b/lib/mongoid.rb @@ -132,4 +132,14 @@ def discriminator_key=(value) class << self prepend GlobalDiscriminatorKeyAssignment end + + def global_thread_pool_async_query_executor + concurrency = Mongoid.global_executor_concurrency || 4 + @@global_thread_pool_async_query_executor ||= Concurrent::ThreadPoolExecutor.new( + min_threads: 0, + max_threads: concurrency, + max_queue: concurrency * 4, + fallback_policy: :caller_runs + ) + end end diff --git a/lib/mongoid/config.rb b/lib/mongoid/config.rb index 0443bfa2b8..fe349e72a0 100644 --- a/lib/mongoid/config.rb +++ b/lib/mongoid/config.rb @@ -127,6 +127,16 @@ module Config # always return a Hash. option :legacy_attributes, default: false + # Sets the async_query_executor for an application. By default the thread pool executor + # set to `:immediate. Options are: + # + # - :immediate - Initializes a single +Concurrent::ImmediateExecutor+ + # - :global_thread_pool - Initializes a single +Concurrent::ThreadPoolExecutor+ + # that uses the +async_query_concurrency+ for the +max_threads+ value. + option :async_query_executor, default: :immediate + + option :global_executor_concurrency, default: nil + # Returns the Config singleton, for use in the configure DSL. # # @return [ self ] The Config singleton. diff --git a/lib/mongoid/contextual.rb b/lib/mongoid/contextual.rb index 6d5829f791..7c0146dd49 100644 --- a/lib/mongoid/contextual.rb +++ b/lib/mongoid/contextual.rb @@ -35,6 +35,11 @@ def context @context ||= create_context end + def load_async + context.load_async if context.respond_to?(:load_async) + self + end + private # Create the context for the queries to execute. Will be memory for diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 9cc9b1ad9e..8acd6a5ffa 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "mongoid/contextual/mongo/async_load_task" require "mongoid/contextual/atomic" require "mongoid/contextual/aggregable/mongo" require "mongoid/contextual/command" @@ -773,6 +774,10 @@ def third_to_last! third_to_last || raise_document_not_found_error end + def load_async + @preload_task = AsyncLoadTask.new(view, klass, criteria) unless @load_task + end + private # Update the documents for the provided method. @@ -855,9 +860,20 @@ def inverse_sorting # # @return [ Array | Mongo::Collection::View ] The docs to iterate. def documents_for_iteration - return view unless eager_loadable? - docs = view.map{ |doc| Factory.from_db(klass, doc, criteria) } - eager_load(docs) + if @preload_task + if @preload_task.started? + @preload_task.future.value! + else + @preload_task.cancel + @preload_task.execute + end + else + return view unless eager_loadable? + docs = view.map do |doc| + Factory.from_db(klass, doc, criteria) + end + eager_load(docs) + end end # Yield to the document. diff --git a/lib/mongoid/contextual/mongo/async_load_task.rb b/lib/mongoid/contextual/mongo/async_load_task.rb new file mode 100644 index 0000000000..00a8487ee0 --- /dev/null +++ b/lib/mongoid/contextual/mongo/async_load_task.rb @@ -0,0 +1,74 @@ +require "mongoid/association/eager_loadable" + +module Mongoid + module Contextual + class Mongo + class AsyncLoadTask + include Association::EagerLoadable + + IMMEDIATE_EXECUTOR = Concurrent::ImmediateExecutor.new + + attr_accessor :future, :criteria + + def initialize(view, klass, criteria) + @view = view + @klass = klass + @criteria = criteria + + @mutex = Mutex.new + @started = false + @cancelled = false + @future = Concurrent::Promises.future_on(executor, self) do |task| + if !task.started? && !task.cancelled? + task.execute + end + end.touch + end + + def executor + case Mongoid.async_query_executor + when :immediate + IMMEDIATE_EXECUTOR + when :global_thread_pool + Mongoid.global_thread_pool_async_query_executor + end + end + + def started? + @mutex.synchronize do + @started + end + end + + def cancel + @mutex.synchronize do + @cancelled = true + end + end + + def cancelled? + @mutex.synchronize do + @cancelled + end + end + + def execute + started! + documents = @view.map do |doc| + Factory.from_db(@klass, doc, @criteria) + end + eager_load(documents) if eager_loadable? + documents + end + + private + + def started! + @mutex.synchronize do + @started = true + end + end + end + end + end +end From b08665e73b42f4f074301ddbdc4e14f5f822a959 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 8 Sep 2022 11:48:29 +0200 Subject: [PATCH 02/35] Add specs --- lib/mongoid/contextual/mongo.rb | 10 +-- .../{async_load_task.rb => preload_task.rb} | 34 +++++----- spec/mongoid/contextual/mongo_spec.rb | 66 +++++++++++++++++++ 3 files changed, 90 insertions(+), 20 deletions(-) rename lib/mongoid/contextual/mongo/{async_load_task.rb => preload_task.rb} (73%) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 8acd6a5ffa..52f085b4dd 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "mongoid/contextual/mongo/async_load_task" +require "mongoid/contextual/mongo/preload_task" require "mongoid/contextual/atomic" require "mongoid/contextual/aggregable/mongo" require "mongoid/contextual/command" @@ -38,6 +38,8 @@ class Mongo # @attribute [r] view The Mongo collection view. attr_reader :view + attr_reader :preload_task + # Get the number of documents matching the query. # # @example Get the number of matching documents. @@ -775,7 +777,7 @@ def third_to_last! end def load_async - @preload_task = AsyncLoadTask.new(view, klass, criteria) unless @load_task + @preload_task = PreloadTask.new(view, klass, criteria) unless @preload_task end private @@ -862,9 +864,9 @@ def inverse_sorting def documents_for_iteration if @preload_task if @preload_task.started? - @preload_task.future.value! + @preload_task.value! else - @preload_task.cancel + @preload_task.unschedule @preload_task.execute end else diff --git a/lib/mongoid/contextual/mongo/async_load_task.rb b/lib/mongoid/contextual/mongo/preload_task.rb similarity index 73% rename from lib/mongoid/contextual/mongo/async_load_task.rb rename to lib/mongoid/contextual/mongo/preload_task.rb index 00a8487ee0..f7856007f8 100644 --- a/lib/mongoid/contextual/mongo/async_load_task.rb +++ b/lib/mongoid/contextual/mongo/preload_task.rb @@ -3,26 +3,28 @@ module Mongoid module Contextual class Mongo - class AsyncLoadTask + # @api private + class PreloadTask + extend Forwardable include Association::EagerLoadable + def_delegators :@future, :value!, :value, :wait!, :wait + IMMEDIATE_EXECUTOR = Concurrent::ImmediateExecutor.new - attr_accessor :future, :criteria + attr_accessor :criteria def initialize(view, klass, criteria) @view = view @klass = klass @criteria = criteria - @mutex = Mutex.new - @started = false - @cancelled = false + @state = :pending @future = Concurrent::Promises.future_on(executor, self) do |task| - if !task.started? && !task.cancelled? + if task.pending? task.execute end - end.touch + end end def executor @@ -34,26 +36,26 @@ def executor end end - def started? + def pending? @mutex.synchronize do - @started + @state == :pending end end - def cancel + def started? @mutex.synchronize do - @cancelled = true + @state == :started end end - def cancelled? + def unschedule @mutex.synchronize do - @cancelled + @state = :cancelled end end def execute - started! + start documents = @view.map do |doc| Factory.from_db(@klass, doc, @criteria) end @@ -63,9 +65,9 @@ def execute private - def started! + def start @mutex.synchronize do - @started = true + @state = :started end end end diff --git a/spec/mongoid/contextual/mongo_spec.rb b/spec/mongoid/contextual/mongo_spec.rb index de8c570382..a0bdb2c0e7 100644 --- a/spec/mongoid/contextual/mongo_spec.rb +++ b/spec/mongoid/contextual/mongo_spec.rb @@ -4445,4 +4445,70 @@ end end end + + describe '#load_async' do + let!(:band) do + Band.create!(name: "Depeche Mode") + end + + let(:criteria) do + Band.where(name: "Depeche Mode") + end + + let(:context) do + described_class.new(criteria) + end + + context 'with global thread pool async query executor' do + config_override :async_query_executor, :global_thread_pool + + it 'preloads the documents' do + context.load_async + context.preload_task.wait + + expect(context.view).not_to receive(:map) + expect(context.to_a).to eq([band]) + end + + it 're-raises exception during preload' do + expect_any_instance_of(Mongoid::Contextual::Mongo::PreloadTask) + .to receive(:execute) + .at_least(:once) + .and_raise(Mongo::Error::OperationFailure) + + context.load_async + context.preload_task.wait + + expect do + context.to_a + end.to raise_error(Mongo::Error::OperationFailure) + end + end + + context 'with immediate thread pool async query executor' do + config_override :async_query_executor, :immediate + + it 'preloads the documents' do + context.load_async + context.preload_task.wait + + expect(context.view).not_to receive(:map) + expect(context.to_a).to eq([band]) + end + + it 're-raises exception during preload' do + expect_any_instance_of(Mongoid::Contextual::Mongo::PreloadTask) + .to receive(:execute) + .at_least(:once) + .and_raise(Mongo::Error::OperationFailure) + + context.load_async + context.preload_task.wait + + expect do + context.to_a + end.to raise_error(Mongo::Error::OperationFailure) + end + end + end end From b82701ca832c5001f66188040d05591fad99ab87 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Fri, 9 Sep 2022 10:17:58 +0200 Subject: [PATCH 03/35] Docs --- docs/reference/queries.txt | 48 ++++++- docs/release-notes/mongoid-8.1.txt | 6 + lib/mongoid/contextual.rb | 7 + lib/mongoid/contextual/mongo.rb | 39 ++--- .../contextual/mongo/documents_loader.rb | 135 ++++++++++++++++++ lib/mongoid/contextual/mongo/preload_task.rb | 76 ---------- mongoid.gemspec | 1 + .../contextual/mongo/documents_loader_spec.rb | 117 +++++++++++++++ spec/mongoid/contextual/mongo_spec.rb | 12 +- 9 files changed, 339 insertions(+), 102 deletions(-) create mode 100644 lib/mongoid/contextual/mongo/documents_loader.rb delete mode 100644 lib/mongoid/contextual/mongo/preload_task.rb create mode 100644 spec/mongoid/contextual/mongo/documents_loader_spec.rb diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index cdab68baa7..5ffd91c2c4 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2388,5 +2388,51 @@ will query the database and separately cache their results. To use the cached results, call ``all.to_a.first`` on the model class. -.. _legacy-query-cache-limitations: +.. _load-async: + +Asynchronous Queries +==================== + +Mongoid allows running database queries asynchronously in the background. +This can be beneficial when there is a need to get documents from different +collections. + +In order to schedule an asynchronous query call ``load_async`` method on a +``Criteria``: + +.. code-block:: ruby + + class PagesController < ApplicationController + def index + @active_bands = Band.where(active: true).load_async + @best_events = Event.best.load_async + @public_articles = Article.where(public: true).load_async + end + end + +In the above example three queries will be scheduled for asynchronous execution. +Queries will be executed as soon as there are available threads in the thread pool. +Results of the queries can be later accessed as usual: + +.. code-block:: html + +
    + <%- @active_bands.each do -%> +
  • <%= band.name %>
    • + <%- end -%> +
+ +Asynchronous queries are disabled by default. In order to enable them, the +following config options should be set: + +.. code-block:: yaml + + development: + ... + options: + # Execute asynchronous queries using a global thread pool. + async_query_executor: :global_thread_pool + # Number of threads in the pool. This option is defaulted to 4. + global_executor_concurrency: 5 + diff --git a/docs/release-notes/mongoid-8.1.txt b/docs/release-notes/mongoid-8.1.txt index 2cab9deaa6..84a91add7e 100644 --- a/docs/release-notes/mongoid-8.1.txt +++ b/docs/release-notes/mongoid-8.1.txt @@ -17,6 +17,12 @@ The complete list of releases is available `on GitHub please consult GitHub releases for detailed release notes and JIRA for the complete list of issues fixed in each release, including bug fixes. +Added ``load_async`` method on ``Criteria`` to asynchronously load documents +---------------------------------------------------------------------------- + +The new ``load_async`` method on ``Criteria`` allows :ref:`running database queries asynchronously ` + + Added ``attribute_before_last_save``, ``saved_change_to_attribute``, ``saved_change_to_attribute?``, and ``will_save_change_to_attribute?`` methods ---------------------------------------------------------------------------------------------------------------------------------------------------- diff --git a/lib/mongoid/contextual.rb b/lib/mongoid/contextual.rb index 7c0146dd49..648ccae1f6 100644 --- a/lib/mongoid/contextual.rb +++ b/lib/mongoid/contextual.rb @@ -35,6 +35,13 @@ def context @context ||= create_context end + # Instructs the context to schedule an asynchronous loading of documents + # specified by the criteria. + # + # Note that depending on the context and on the Mongoid configuration, + # documents can be loaded synchronously on the caller's thread. + # + # @return [ Criteria ] Returns self. def load_async context.load_async if context.respond_to?(:load_async) self diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 52f085b4dd..9d9918293b 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "mongoid/contextual/mongo/preload_task" +require "mongoid/contextual/mongo/documents_loader" require "mongoid/contextual/atomic" require "mongoid/contextual/aggregable/mongo" require "mongoid/contextual/command" @@ -38,7 +38,7 @@ class Mongo # @attribute [r] view The Mongo collection view. attr_reader :view - attr_reader :preload_task + attr_reader :documents_loader # Get the number of documents matching the query. # @@ -776,8 +776,15 @@ def third_to_last! third_to_last || raise_document_not_found_error end + # Schedule a task to load documents for the context. + # + # Depending on the Mongoid configuration, the scheduled task can be executed + # immediately on the caller's thread, or can be scheduled for an + # asynchronous execution. + # + # @api private def load_async - @preload_task = PreloadTask.new(view, klass, criteria) unless @preload_task + @documents_loader = DocumentsLoader.new(view, klass, criteria) unless @documents_loader end private @@ -847,27 +854,21 @@ def inverse_sorting Hash[sort.map{|k, v| [k, -1*v]}] end - # Get the documents the context should iterate. This follows 3 rules: - # - # 1. If the query is cached, and we already have documents loaded, use - # them. - # 2. If we are eager loading, then eager load the documents and use - # those. - # 3. Use the query. + # Get the documents the context should iterate. # - # @api private - # - # @example Get the documents for iteration. - # context.documents_for_iteration + # If the documents have been already preloaded by `Document::Loader` + # instance, they will be used. # # @return [ Array | Mongo::Collection::View ] The docs to iterate. + # + # @api private def documents_for_iteration - if @preload_task - if @preload_task.started? - @preload_task.value! + if @documents_loader + if @documents_loader.started? + @documents_loader.value! else - @preload_task.unschedule - @preload_task.execute + @documents_loader.unschedule + @documents_loader.execute end else return view unless eager_loadable? diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb new file mode 100644 index 0000000000..56ae54a537 --- /dev/null +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -0,0 +1,135 @@ +require "mongoid/association/eager_loadable" + +module Mongoid + module Contextual + class Mongo + # Loads documents for the provided criteria. + # + # @api private + class DocumentsLoader + extend Forwardable + include Association::EagerLoadable + + def_delegators :@future, :value!, :value, :wait!, :wait + + # Synchronous executor to be used when async_query_executor config option + # is set to :immediate. This excutor runs all operations on the current + # thread, blocking as necessary. + IMMEDIATE_EXECUTOR = Concurrent::ImmediateExecutor.new + + # Returns suitable executor according to Mongoid config options. + # + # @return [ Concurrent::ImmediateExecutor | Concurrent::ThreadPoolExecutor ] The executor + # to be used to execute document loading tasks. + def self.executor + case Mongoid.async_query_executor + when :immediate + IMMEDIATE_EXECUTOR + when :global_thread_pool + Mongoid.global_thread_pool_async_query_executor + end + end + + # @return [ Mongoid::Criteria ] Criteria that specifies which documents should + # be loaded. Exposed here because `eager_loadable?` method from + # `Association::EagerLoadable` expects is to be available. + attr_accessor :criteria + + # Instantiates the document loader instance an immediately schedules + # its execution using provided executor. + # + # @param [ Mongo::Collection::View ] view The collection view to get + # records from the database. + # @param [ Class ] klass Mongoid model class to instantiate documents. + # All records obtained from the database will be converted to an + # instance of this class, if possible. + # @param [ Mongoid::Criteria ] criteria. Criteria that specifies which + # documents should be loaded. + # @param [ Concurrent::AbstractExecutorService ] executor. Executor that + # is capable of running `Concurrent::Promises::Future` instances. + def initialize(view, klass, criteria, executor: self.class.executor) + @view = view + @klass = klass + @criteria = criteria + @mutex = Mutex.new + @state = :pending + @future = Concurrent::Promises.future_on(executor, self) do |task| + if task.pending? + task.send(:start) + task.execute + end + end + end + + # Returns false or true whether the loader is in pending state. + # + # Pending state means that the loader execution has been scheduled, + # but has not been started yet. + # + # @return [ true | false ] true if the loader is in pending state, + # otherwise false. + def pending? + @mutex.synchronize do + @state == :pending + end + end + + # Returns false or true whether the loader is in started state. + # + # Started state means that the loader execution has been started. + # Note that the loader stays in this state even after the execution + # completed (successfully or failed). + # + # @return [ true | false ] true if the loader is in started state, + # otherwise false. + def started? + @mutex.synchronize do + @state == :started + end + end + + # Mark the loader as unscheduled. + # + # If the loader is marked unscheduled, it will not be executed. The only + # option to load the documents is to call `execute` method directly. + # + # Please note that if execution of a task has been already started, + # unscheduling does not have any effect. + def unschedule + @mutex.synchronize do + @state = :cancelled unless state == :started + end + end + + # Loads records specified by `@criteria` from the database, and convert + # them to Mongoid documents of `@klass` type. + # + # This method is called by the task (possibly asynchronous) scheduled + # when creating an instance of the loader. However, this method can be + # called directly, if it is desired to execute loading on the caller + # thread immediately. + # + # Calling this method does not change the state of the loader. + # + # @return [ Array ] Array of document loaded from + # the database. + def execute + documents = @view.map do |doc| + Factory.from_db(@klass, doc, @criteria) + end + eager_load(documents) if eager_loadable? + documents + end + + private + + # Mark the loader as started. + def start + @mutex.synchronize do + @state = :started + end + end + end + end + end +end diff --git a/lib/mongoid/contextual/mongo/preload_task.rb b/lib/mongoid/contextual/mongo/preload_task.rb deleted file mode 100644 index f7856007f8..0000000000 --- a/lib/mongoid/contextual/mongo/preload_task.rb +++ /dev/null @@ -1,76 +0,0 @@ -require "mongoid/association/eager_loadable" - -module Mongoid - module Contextual - class Mongo - # @api private - class PreloadTask - extend Forwardable - include Association::EagerLoadable - - def_delegators :@future, :value!, :value, :wait!, :wait - - IMMEDIATE_EXECUTOR = Concurrent::ImmediateExecutor.new - - attr_accessor :criteria - - def initialize(view, klass, criteria) - @view = view - @klass = klass - @criteria = criteria - @mutex = Mutex.new - @state = :pending - @future = Concurrent::Promises.future_on(executor, self) do |task| - if task.pending? - task.execute - end - end - end - - def executor - case Mongoid.async_query_executor - when :immediate - IMMEDIATE_EXECUTOR - when :global_thread_pool - Mongoid.global_thread_pool_async_query_executor - end - end - - def pending? - @mutex.synchronize do - @state == :pending - end - end - - def started? - @mutex.synchronize do - @state == :started - end - end - - def unschedule - @mutex.synchronize do - @state = :cancelled - end - end - - def execute - start - documents = @view.map do |doc| - Factory.from_db(@klass, doc, @criteria) - end - eager_load(documents) if eager_loadable? - documents - end - - private - - def start - @mutex.synchronize do - @state = :started - end - end - end - end - end -end diff --git a/mongoid.gemspec b/mongoid.gemspec index ffbb650483..23f605d2f9 100644 --- a/mongoid.gemspec +++ b/mongoid.gemspec @@ -38,6 +38,7 @@ Gem::Specification.new do |s| # See: https://github.com/rails/rails/pull/43951 s.add_dependency("activemodel", ['>=5.1', '<7.1', '!= 7.0.0']) s.add_dependency("mongo", ['>=2.18.0', '<3.0.0']) + s.add_dependency("concurrent-ruby", ['>= 1.1.10', '<1.2']) # The ruby2_keywords gem is recommended for handling argument delegation issues, # especially if support for 2.6 or prior is required. diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb new file mode 100644 index 0000000000..09d66a1dcb --- /dev/null +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Mongoid::Contextual::Mongo::DocumentsLoader do + let(:view) do + double('view').tap do |view| + allow(view).to receive(:map) + end + end + + let(:klass) do + double + end + + let(:criteria) do + double('criteria').tap do |criteria| + allow(criteria).to receive(:inclusions).and_return([]) + end + end + + let(:executor) do + described_class.executor + end + + let(:subject) do + described_class.new(view, klass, criteria, executor: executor) + end + + context 'state management' do + let(:executor) do + # Such executor will never execute a task, so it guarantees that + # our task will stay in its initial state. + Concurrent::ThreadPoolExecutor.new( + min_threads: 0, + max_threads: 0, + ) + end + + describe '#initialize' do + it 'initializes in pending state' do + expect(subject.pending?).to be_truthy + expect(subject.started?).to be_falsey + end + end + + describe '#unschedule' do + it 'changes state' do + subject.unschedule + expect(subject.pending?).to be_falsey + expect(subject.started?).to be_falsey + end + end + + describe '#execute' do + it 'does not change state' do + prev_started = subject.started? + prev_pending = subject.pending? + subject.execute + expect(subject.started?).to eq(prev_started) + expect(subject.pending?).to eq(prev_pending) + end + end + + context 'when the task is completed' do + let(:executor) do + Concurrent::ImmediateExecutor.new + end + + it 'changes the state to started' do + subject.wait! + expect(subject.started?).to be_truthy + expect(subject.pending?).to be_falsey + end + end + end + + context 'loading documents' do + let(:view) do + klass.collection.find(criteria.selector, session: criteria.send(:_session)) + end + + let(:criteria) do + Band.where(name: "Depeche Mode") + end + + let(:klass) do + Band + end + + let!(:band) do + Band.create!(name: 'Depeche Mode') + end + + context 'asynchronously' do + it 'loads documents ' do + subject.wait! + expect(subject.value).to eq([band]) + end + end + + context 'synchronously' do + let(:executor) do + # Such executor will never execute a task, so it guarantees that + # our task will stay in its initial state. + Concurrent::ThreadPoolExecutor.new( + min_threads: 0, + max_threads: 0, + ) + end + + it 'loads documents' do + expect(subject.execute).to eq([band]) + end + end + end +end diff --git a/spec/mongoid/contextual/mongo_spec.rb b/spec/mongoid/contextual/mongo_spec.rb index a0bdb2c0e7..eb95b039f4 100644 --- a/spec/mongoid/contextual/mongo_spec.rb +++ b/spec/mongoid/contextual/mongo_spec.rb @@ -4464,20 +4464,20 @@ it 'preloads the documents' do context.load_async - context.preload_task.wait + context.documents_loader.wait expect(context.view).not_to receive(:map) expect(context.to_a).to eq([band]) end it 're-raises exception during preload' do - expect_any_instance_of(Mongoid::Contextual::Mongo::PreloadTask) + expect_any_instance_of(Mongoid::Contextual::Mongo::DocumentsLoader) .to receive(:execute) .at_least(:once) .and_raise(Mongo::Error::OperationFailure) context.load_async - context.preload_task.wait + context.documents_loader.wait expect do context.to_a @@ -4490,20 +4490,20 @@ it 'preloads the documents' do context.load_async - context.preload_task.wait + context.documents_loader.wait expect(context.view).not_to receive(:map) expect(context.to_a).to eq([band]) end it 're-raises exception during preload' do - expect_any_instance_of(Mongoid::Contextual::Mongo::PreloadTask) + expect_any_instance_of(Mongoid::Contextual::Mongo::DocumentsLoader) .to receive(:execute) .at_least(:once) .and_raise(Mongo::Error::OperationFailure) context.load_async - context.preload_task.wait + context.documents_loader.wait expect do context.to_a From 686b7724ae2db65da9484b000586dabda1a36c7d Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 10:39:53 +0200 Subject: [PATCH 04/35] Specs and docs --- lib/mongoid/config.rb | 3 + .../contextual/mongo/documents_loader.rb | 2 +- spec/mongoid/config_spec.rb | 73 +++++++++++++++++++ .../contextual/mongo/documents_loader_spec.rb | 18 +++++ spec/mongoid_spec.rb | 24 ++++++ 5 files changed, 119 insertions(+), 1 deletion(-) diff --git a/lib/mongoid/config.rb b/lib/mongoid/config.rb index 094171c7bf..3275a82375 100644 --- a/lib/mongoid/config.rb +++ b/lib/mongoid/config.rb @@ -135,6 +135,9 @@ module Config # that uses the +async_query_concurrency+ for the +max_threads+ value. option :async_query_executor, default: :immediate + # Defines how many asynchronous queries can be executed concurrently. + # This option should be set only if `async_query_executor` is set + # to `:global_thread_pool`, it will be ignored otherwise. option :global_executor_concurrency, default: nil # When this flag is false, a document will become read-only only once the diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index 56ae54a537..fe0300797b 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -97,7 +97,7 @@ def started? # unscheduling does not have any effect. def unschedule @mutex.synchronize do - @state = :cancelled unless state == :started + @state = :cancelled unless @state == :started end end diff --git a/spec/mongoid/config_spec.rb b/spec/mongoid/config_spec.rb index 4ac5d4bca0..e815822dee 100644 --- a/spec/mongoid/config_spec.rb +++ b/spec/mongoid/config_spec.rb @@ -224,6 +224,79 @@ end end + context 'async_query_executor option' do + let(:option) { :async_query_executor } + + before do + Mongoid::Config.reset + Mongoid.configure do |config| + config.load_configuration(conf) + end + end + + context "when it is not set in the config" do + + let(:conf) { CONFIG } + + it "it is set to its default" do + expect(Mongoid.send(option)).to eq(:immediate) + end + end + + context 'when the value is :immediate' do + + let(:conf) do + CONFIG.merge(options: { option => :immediate }) + end + + it "is set to false" do + expect(Mongoid.send(option)).to be(:immediate) + end + end + + context 'when the value is :global_thread_pool' do + + let(:conf) do + CONFIG.merge(options: { option => :global_thread_pool }) + end + + it "is set to false" do + expect(Mongoid.send(option)).to be(:global_thread_pool) + end + end + end + + context 'global_executor_concurrency option' do + let(:option) { :global_executor_concurrency } + + before do + Mongoid::Config.reset + Mongoid.configure do |config| + config.load_configuration(conf) + end + end + + context "when it is not set in the config" do + + let(:conf) { CONFIG } + + it "it is set to its default" do + expect(Mongoid.send(option)).to eq(nil) + end + end + + context 'when the value is set to a number' do + + let(:conf) do + CONFIG.merge(options: { option => 5 }) + end + + it "is set to the number" do + expect(Mongoid.send(option)).to be(5) + end + end + end + shared_examples "a config option" do before do diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb index 09d66a1dcb..b5104ec045 100644 --- a/spec/mongoid/contextual/mongo/documents_loader_spec.rb +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -114,4 +114,22 @@ end end end + + describe '.executor' do + context 'when immediate executor configured' do + config_override :async_query_executor, :immediate + + it 'returns immediate executor' do + expect(described_class.executor).to eq(described_class::IMMEDIATE_EXECUTOR) + end + end + + context 'when global thread pool executor configured' do + config_override :async_query_executor, :global_thread_pool + + it 'returns global thread pool executor' do + expect(described_class.executor).to eq(Mongoid.global_thread_pool_async_query_executor) + end + end + end end diff --git a/spec/mongoid_spec.rb b/spec/mongoid_spec.rb index 5f25635594..e6562fabb6 100644 --- a/spec/mongoid_spec.rb +++ b/spec/mongoid_spec.rb @@ -95,4 +95,28 @@ expect(Mongoid.models).to include(Band) end end + + describe ".global_thread_pool_async_query_executor" do + after(:each) do + Mongoid.class_variable_set(:@@global_thread_pool_async_query_executor, nil) + end + + context 'when global_executor_concurrency option is set' do + config_override :global_executor_concurrency, 50 + + it 'returns an executor' do + executor = Mongoid.global_thread_pool_async_query_executor + expect(executor).not_to be_nil + expect(executor.max_length).to eq( 50 ) + end + end + + context 'when global_executor_concurrency option is not set' do + it 'returns an executor' do + executor = Mongoid.global_thread_pool_async_query_executor + expect(executor).not_to be_nil + expect(executor.max_length).to eq( 4 ) + end + end + end end From bd23dc770905c83301d504b769413989eef63e8b Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 12:06:54 +0200 Subject: [PATCH 05/35] Fix spec --- spec/mongoid_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/mongoid_spec.rb b/spec/mongoid_spec.rb index e6562fabb6..ba1e5d2dab 100644 --- a/spec/mongoid_spec.rb +++ b/spec/mongoid_spec.rb @@ -97,6 +97,10 @@ end describe ".global_thread_pool_async_query_executor" do + before(:each) do + Mongoid.class_variable_set(:@@global_thread_pool_async_query_executor, nil) + end + after(:each) do Mongoid.class_variable_set(:@@global_thread_pool_async_query_executor, nil) end From ea282627ddd4beac074ab52c91da69ead64f9993 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 12:26:04 +0200 Subject: [PATCH 06/35] More docs --- docs/reference/queries.txt | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 5ffd91c2c4..b3ff9f5170 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2411,7 +2411,6 @@ In order to schedule an asynchronous query call ``load_async`` method on a end In the above example three queries will be scheduled for asynchronous execution. -Queries will be executed as soon as there are available threads in the thread pool. Results of the queries can be later accessed as usual: .. code-block:: html @@ -2422,6 +2421,25 @@ Results of the queries can be later accessed as usual: <%- end -%> +Even if a query is scheduled for asynchronous execution, it might be executed +synchronously on the caller's thread. There are the following options depending +on when the query results are being accessed: + +#. If the scheduled asynchronous task has been already executed, the results are returned. +#. If the task has been started, but not finished yet, the caller's thread blocks until the task is finished. +#. If the task has not been started yet, it it being removed from the execution queue, and the query is being executed synchronously on the caller's thread. + +.. note:: + + Even though ``load_async`` method returns ``Criteria`` object, you should not + do any operations of this object expect accessing query results. The query is + scheduled for an execution immediately after calling ``load_async``, therefore + later changes on the `Criteria`` object may not be applied. + + +Configuring asynchronous query execution +---------------------------------------- + Asynchronous queries are disabled by default. In order to enable them, the following config options should be set: @@ -2434,5 +2452,3 @@ following config options should be set: async_query_executor: :global_thread_pool # Number of threads in the pool. This option is defaulted to 4. global_executor_concurrency: 5 - - From 5330c3bd93fa8cb4be876a5803298bfe137b35b3 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:42:33 +0200 Subject: [PATCH 07/35] Update docs/reference/queries.txt Co-authored-by: Neil Shweky --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index b3ff9f5170..ce579ddb23 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2422,7 +2422,7 @@ Results of the queries can be later accessed as usual: Even if a query is scheduled for asynchronous execution, it might be executed -synchronously on the caller's thread. There are the following options depending +synchronously on the caller's thread. There are three possible scenarios depending on when the query results are being accessed: #. If the scheduled asynchronous task has been already executed, the results are returned. From ffe952c44c964522a62f75c4d35dd3547814a514 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:42:42 +0200 Subject: [PATCH 08/35] Update docs/reference/queries.txt Co-authored-by: Neil Shweky --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index ce579ddb23..0a9f495447 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2427,7 +2427,7 @@ on when the query results are being accessed: #. If the scheduled asynchronous task has been already executed, the results are returned. #. If the task has been started, but not finished yet, the caller's thread blocks until the task is finished. -#. If the task has not been started yet, it it being removed from the execution queue, and the query is being executed synchronously on the caller's thread. +#. If the task has not been started yet, it is removed from the execution queue, and the query is executed synchronously on the caller's thread. .. note:: From 0e8b843fc1e73acfdd5845489090ee30fc4db3eb Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:42:53 +0200 Subject: [PATCH 09/35] Update docs/reference/queries.txt Co-authored-by: Neil Shweky --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 0a9f495447..eef56897cc 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2432,7 +2432,7 @@ on when the query results are being accessed: .. note:: Even though ``load_async`` method returns ``Criteria`` object, you should not - do any operations of this object expect accessing query results. The query is + do any operations on this object except accessing query results. The query is scheduled for an execution immediately after calling ``load_async``, therefore later changes on the `Criteria`` object may not be applied. From acd1a9456d16d674640fd947746a871cad4d01e4 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:01 +0200 Subject: [PATCH 10/35] Update docs/reference/queries.txt Co-authored-by: Neil Shweky --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index eef56897cc..7465335b28 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2433,7 +2433,7 @@ on when the query results are being accessed: Even though ``load_async`` method returns ``Criteria`` object, you should not do any operations on this object except accessing query results. The query is - scheduled for an execution immediately after calling ``load_async``, therefore + scheduled for execution immediately after calling ``load_async``, therefore later changes on the `Criteria`` object may not be applied. From a166d8a06005b97696d13065e7ac8c4e9eea2b36 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:09 +0200 Subject: [PATCH 11/35] Update docs/reference/queries.txt Co-authored-by: Neil Shweky --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 7465335b28..7f5d04d989 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2434,7 +2434,7 @@ on when the query results are being accessed: Even though ``load_async`` method returns ``Criteria`` object, you should not do any operations on this object except accessing query results. The query is scheduled for execution immediately after calling ``load_async``, therefore - later changes on the `Criteria`` object may not be applied. + later changes to the `Criteria`` object may not be applied. Configuring asynchronous query execution From a7a975daa3a6163c9c8f5f113a748d8df4f4be92 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:18 +0200 Subject: [PATCH 12/35] Update docs/release-notes/mongoid-8.1.txt Co-authored-by: Neil Shweky --- docs/release-notes/mongoid-8.1.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/mongoid-8.1.txt b/docs/release-notes/mongoid-8.1.txt index f902139db1..a7195e8088 100644 --- a/docs/release-notes/mongoid-8.1.txt +++ b/docs/release-notes/mongoid-8.1.txt @@ -20,7 +20,7 @@ the complete list of issues fixed in each release, including bug fixes. Added ``load_async`` method on ``Criteria`` to asynchronously load documents ---------------------------------------------------------------------------- -The new ``load_async`` method on ``Criteria`` allows :ref:`running database queries asynchronously ` +The new ``load_async`` method on ``Criteria`` allows :ref:`running database queries asynchronously `. Added ``attribute_before_last_save``, ``saved_change_to_attribute``, ``saved_change_to_attribute?``, and ``will_save_change_to_attribute?`` methods From 6de107f7aaa5d9f6cccd21591afeb056c1f4b2ef Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:30 +0200 Subject: [PATCH 13/35] Update lib/mongoid/config.rb Co-authored-by: Neil Shweky --- lib/mongoid/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/config.rb b/lib/mongoid/config.rb index 3275a82375..115bf2bc92 100644 --- a/lib/mongoid/config.rb +++ b/lib/mongoid/config.rb @@ -128,7 +128,7 @@ module Config option :legacy_attributes, default: false # Sets the async_query_executor for an application. By default the thread pool executor - # set to `:immediate. Options are: + # is set to `:immediate. Options are: # # - :immediate - Initializes a single +Concurrent::ImmediateExecutor+ # - :global_thread_pool - Initializes a single +Concurrent::ThreadPoolExecutor+ From 171c8a9a4690fad2c7e0f85323c6cedc879d0322 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:37 +0200 Subject: [PATCH 14/35] Update lib/mongoid/contextual/mongo/documents_loader.rb Co-authored-by: Neil Shweky --- lib/mongoid/contextual/mongo/documents_loader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index fe0300797b..b1f59c0c5a 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -36,7 +36,7 @@ def self.executor attr_accessor :criteria # Instantiates the document loader instance an immediately schedules - # its execution using provided executor. + # its execution using the provided executor. # # @param [ Mongo::Collection::View ] view The collection view to get # records from the database. From 69f68bb2d003efd77756d400890989a9bad39b58 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:43:56 +0200 Subject: [PATCH 15/35] Update lib/mongoid/contextual/mongo/documents_loader.rb Co-authored-by: Neil Shweky --- lib/mongoid/contextual/mongo/documents_loader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index b1f59c0c5a..73560efddb 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -35,7 +35,7 @@ def self.executor # `Association::EagerLoadable` expects is to be available. attr_accessor :criteria - # Instantiates the document loader instance an immediately schedules + # Instantiates the document loader instance and immediately schedules # its execution using the provided executor. # # @param [ Mongo::Collection::View ] view The collection view to get From 4a3130ad4a6fb5264cc836669454787b5c68baa8 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 16:44:05 +0200 Subject: [PATCH 16/35] Update spec/mongoid/contextual/mongo/documents_loader_spec.rb Co-authored-by: Neil Shweky --- spec/mongoid/contextual/mongo/documents_loader_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb index b5104ec045..70841e22cb 100644 --- a/spec/mongoid/contextual/mongo/documents_loader_spec.rb +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -93,7 +93,7 @@ end context 'asynchronously' do - it 'loads documents ' do + it 'loads documents' do subject.wait! expect(subject.value).to eq([band]) end From 5a99a831f12c2e9c9863680acc877bfcbb56dbf8 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 17:20:09 +0200 Subject: [PATCH 17/35] Fix code review remarks --- lib/mongoid/contextual/mongo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 45dcaad885..6b0e373833 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -788,7 +788,7 @@ def third_to_last! # # @api private def load_async - @documents_loader = DocumentsLoader.new(view, klass, criteria) unless @documents_loader + @documents_loader ||= DocumentsLoader.new(view, klass, criteria) end private From 7424de1c99fd0d602ae3be3eded3075679ff4c31 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 20:05:53 +0200 Subject: [PATCH 18/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 7f5d04d989..81ed30f96c 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2397,7 +2397,7 @@ Mongoid allows running database queries asynchronously in the background. This can be beneficial when there is a need to get documents from different collections. -In order to schedule an asynchronous query call ``load_async`` method on a +In order to schedule an asynchronous query call the ``load_async`` method on a ``Criteria``: .. code-block:: ruby From 1bee49a4d8fe48293c660eed0f961f4bd16f03fa Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 14 Sep 2022 20:10:26 +0200 Subject: [PATCH 19/35] Update lib/mongoid/config.rb Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- lib/mongoid/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/config.rb b/lib/mongoid/config.rb index 115bf2bc92..f84de677ed 100644 --- a/lib/mongoid/config.rb +++ b/lib/mongoid/config.rb @@ -127,7 +127,7 @@ module Config # always return a Hash. option :legacy_attributes, default: false - # Sets the async_query_executor for an application. By default the thread pool executor + # Sets the async_query_executor for the application. By default the thread pool executor # is set to `:immediate. Options are: # # - :immediate - Initializes a single +Concurrent::ImmediateExecutor+ From 26ebe6036ef0cc4576b5146c9d2a74274484d464 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 15 Sep 2022 12:48:33 +0200 Subject: [PATCH 20/35] Fix code review remarks --- docs/reference/configuration.txt | 15 ++++++ docs/reference/queries.txt | 11 ++-- lib/config/locales/en.yml | 15 ++++++ lib/mongoid.rb | 10 ---- lib/mongoid/config.rb | 3 +- lib/mongoid/config/validators.rb | 1 + .../config/validators/async_query_executor.rb | 22 ++++++++ .../contextual/mongo/documents_loader.rb | 43 ++++++++++++--- lib/mongoid/errors.rb | 2 + .../errors/invalid_async_query_executor.rb | 25 +++++++++ .../invalid_global_executor_concurrency.rb | 22 ++++++++ spec/mongoid/config_spec.rb | 11 ++++ .../contextual/mongo/documents_loader_spec.rb | 52 ++++++++++++++++--- spec/mongoid_spec.rb | 28 ---------- 14 files changed, 203 insertions(+), 57 deletions(-) create mode 100644 lib/mongoid/config/validators/async_query_executor.rb create mode 100644 lib/mongoid/errors/invalid_async_query_executor.rb create mode 100644 lib/mongoid/errors/invalid_global_executor_concurrency.rb diff --git a/docs/reference/configuration.txt b/docs/reference/configuration.txt index a018f43086..35827b3d6f 100644 --- a/docs/reference/configuration.txt +++ b/docs/reference/configuration.txt @@ -265,6 +265,16 @@ for details on driver options. # if the database name is not explicitly defined. (default: nil) app_name: MyApplicationName + # Type of executor for queries scheduled using ``load_async`` method. + # + # There are two possible values for this option: + # + # - :immediate - Queries will be immediately executed on a current thread. + # This is the default option. + # - :global_thread_pool - Queries will be executed asynchronously in + # background using a thread pool. + #async_query_executor: :immediate + # Mark belongs_to associations as required by default, so that saving a # model with a missing belongs_to association will trigger a validation # error. (default: true) @@ -358,6 +368,11 @@ for details on driver options. # Raise an exception when a field is redefined. (default: false) duplicate_fields_exception: false + # Defines how many asynchronous queries can be executed concurrently. + # This option should be set only if `async_query_executor` option is set + # to `:global_thread_pool`. + #global_executor_concurrency: nil + # Include the root model name in json serialization. (default: false) include_root_in_json: false diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 81ed30f96c..af6a380d88 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2440,8 +2440,13 @@ on when the query results are being accessed: Configuring asynchronous query execution ---------------------------------------- -Asynchronous queries are disabled by default. In order to enable them, the -following config options should be set: +Asynchronous queries are disabled by default. When asynchronous queries are +disabled, ``load_async`` will execute the query immediately he current thread, +blocking as necessary. Therefore, calling ``load_async`` on criteria in this case +is roughly the equivalent of calling ``to_a``. + +In order to enable asynchronous query execution, the following config options +should be set: .. code-block:: yaml @@ -2451,4 +2456,4 @@ following config options should be set: # Execute asynchronous queries using a global thread pool. async_query_executor: :global_thread_pool # Number of threads in the pool. This option is defaulted to 4. - global_executor_concurrency: 5 + # global_executor_concurrency: 4 diff --git a/lib/config/locales/en.yml b/lib/config/locales/en.yml index 922109b312..70d24850fe 100644 --- a/lib/config/locales/en.yml +++ b/lib/config/locales/en.yml @@ -133,6 +133,14 @@ en: A collation option is only supported if the query is executed on a MongoDB server with version >= 3.4." resolution: "Remove the collation option from the query." + invalid_async_query_executor: + message: "Invalid async_query_executor option: %{executor}." + summary: "A invalid async query executor was specified. + The valid options are: %{options}." + resolution: "Pick an allowed option or fix the typo. If you were + expecting the option to be there, please consult the following page + with respect to Mongoid's configuration:\n\n + \_\_https://www.mongodb.com/docs/mongoid/current/reference/configuration/#mongoid-configuration-options" invalid_config_file: message: "Invalid configuration file: %{path}." summary: "Your mongoid.yml configuration file does not contain the @@ -238,6 +246,13 @@ en: resolution: "Please provide a valid type value for the field. Refer to: https://docs.mongodb.com/mongoid/current/reference/fields/#using-symbols-or-strings-instead-of-classes" + invalid_global_executor_concurrency: + message: "Invalid global_executor_concurrency option." + summary: "You set global_executor_concurrency while async_query_executor + option is not set to :global_thread_pool. The global_executor_concurrency is + allowed only for the global thread pool executor." + resolution: "Set global_executor_concurrency option to :global_thread_pool + or remove global_executor_concurrency option." invalid_includes: message: "Invalid includes directive: %{klass}.includes(%{args})" summary: "Eager loading in Mongoid only supports providing arguments diff --git a/lib/mongoid.rb b/lib/mongoid.rb index 4700639598..22816bd9e2 100644 --- a/lib/mongoid.rb +++ b/lib/mongoid.rb @@ -132,14 +132,4 @@ def discriminator_key=(value) class << self prepend GlobalDiscriminatorKeyAssignment end - - def global_thread_pool_async_query_executor - concurrency = Mongoid.global_executor_concurrency || 4 - @@global_thread_pool_async_query_executor ||= Concurrent::ThreadPoolExecutor.new( - min_threads: 0, - max_threads: concurrency, - max_queue: concurrency * 4, - fallback_policy: :caller_runs - ) - end end diff --git a/lib/mongoid/config.rb b/lib/mongoid/config.rb index f84de677ed..de0067351e 100644 --- a/lib/mongoid/config.rb +++ b/lib/mongoid/config.rb @@ -137,7 +137,7 @@ module Config # Defines how many asynchronous queries can be executed concurrently. # This option should be set only if `async_query_executor` is set - # to `:global_thread_pool`, it will be ignored otherwise. + # to `:global_thread_pool`. option :global_executor_concurrency, default: nil # When this flag is false, a document will become read-only only once the @@ -321,6 +321,7 @@ def truncate! # @param [ Hash ] options The configuration options. def options=(options) if options + Validators::AsyncQueryExecutor.validate(options) options.each_pair do |option, value| Validators::Option.validate(option) send("#{option}=", value) diff --git a/lib/mongoid/config/validators.rb b/lib/mongoid/config/validators.rb index 5fbe21403d..3675bea76f 100644 --- a/lib/mongoid/config/validators.rb +++ b/lib/mongoid/config/validators.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require "mongoid/config/validators/async_query_executor" require "mongoid/config/validators/option" require "mongoid/config/validators/client" diff --git a/lib/mongoid/config/validators/async_query_executor.rb b/lib/mongoid/config/validators/async_query_executor.rb new file mode 100644 index 0000000000..b98eb8d6fb --- /dev/null +++ b/lib/mongoid/config/validators/async_query_executor.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Mongoid + module Config + module Validators + + # Validator for async query executor configuration. + module AsyncQueryExecutor + extend self + + + def validate(options) + if options.key?(:async_query_executor) + if options[:async_query_executor].to_sym == :immediate && !options[:global_executor_concurrency].nil? + raise Errors::InvalidGlobalExecutorConcurrency + end + end + end + end + end + end +end diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index 73560efddb..c09731c7a0 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -12,21 +12,50 @@ class DocumentsLoader def_delegators :@future, :value!, :value, :wait!, :wait - # Synchronous executor to be used when async_query_executor config option - # is set to :immediate. This excutor runs all operations on the current + # Returns synchronous executor to be used when async_query_executor config option + # is set to :immediate. This executor runs all operations on the current # thread, blocking as necessary. - IMMEDIATE_EXECUTOR = Concurrent::ImmediateExecutor.new + # + # @return [ Concurrent::ImmediateExecutor ] The executor + # to be used to execute document loading tasks. + def self.immediate_executor + @@immediate_executor ||= Concurrent::ImmediateExecutor.new + end + + # Returns asynchronous executor to be used when async_query_executor config option + # is set to :global_thread_pool. This executor runs operations on background threads + # using a thread pool. + # + # @return [ Concurrent::ThreadPoolExecutor ] The executor + # to be used to execute document loading tasks. + def self.global_thread_pool_async_query_executor + concurrency = Mongoid.global_executor_concurrency || 4 + @@global_thread_pool_async_query_executor ||= Concurrent::ThreadPoolExecutor.new( + min_threads: 0, + max_threads: concurrency, + max_queue: concurrency * 4, + fallback_policy: :caller_runs + ) + end # Returns suitable executor according to Mongoid config options. # + # @param [ String | Symbol] name The query executor name, can be either + # :immediate or :global_thread_pool. Defaulted to `async_query_executor` + # config option. + # # @return [ Concurrent::ImmediateExecutor | Concurrent::ThreadPoolExecutor ] The executor # to be used to execute document loading tasks. - def self.executor - case Mongoid.async_query_executor + # + # @raise [ Errors::InvalidQueryExecutor ] If an unknown name is provided. + def self.executor(name = Mongoid.async_query_executor) + case name.to_sym when :immediate - IMMEDIATE_EXECUTOR + immediate_executor when :global_thread_pool - Mongoid.global_thread_pool_async_query_executor + global_thread_pool_async_query_executor + else + raise Errors::InvalidQueryExecutor.new(name) end end diff --git a/lib/mongoid/errors.rb b/lib/mongoid/errors.rb index 2dd534ce1a..6d3b32a938 100644 --- a/lib/mongoid/errors.rb +++ b/lib/mongoid/errors.rb @@ -10,6 +10,7 @@ require "mongoid/errors/document_not_found" require "mongoid/errors/empty_config_file" require "mongoid/errors/in_memory_collation_not_supported" +require "mongoid/errors/invalid_async_query_executor" require "mongoid/errors/invalid_collection" require "mongoid/errors/invalid_config_file" require "mongoid/errors/invalid_config_option" @@ -18,6 +19,7 @@ require "mongoid/errors/invalid_field_option" require "mongoid/errors/invalid_field_type" require "mongoid/errors/invalid_find" +require "mongoid/errors/invalid_global_executor_concurrency" require "mongoid/errors/invalid_includes" require "mongoid/errors/invalid_index" require "mongoid/errors/invalid_options" diff --git a/lib/mongoid/errors/invalid_async_query_executor.rb b/lib/mongoid/errors/invalid_async_query_executor.rb new file mode 100644 index 0000000000..180e7b7dc3 --- /dev/null +++ b/lib/mongoid/errors/invalid_async_query_executor.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Mongoid + module Errors + + # This error is raised when a bad async query executor option is attempted + # to be set. + class InvalidQueryExecutor < MongoidError + + # Create the new error. + # + # @param [ Symbol | String ] executor The attempted async query executor. + # + # @api private + def initialize(executor) + super( + compose_message( + "invalid_async_query_executor", + { executor: executor, options: [:immediate, :global_thread_pool] } + ) + ) + end + end + end +end diff --git a/lib/mongoid/errors/invalid_global_executor_concurrency.rb b/lib/mongoid/errors/invalid_global_executor_concurrency.rb new file mode 100644 index 0000000000..0cb842e493 --- /dev/null +++ b/lib/mongoid/errors/invalid_global_executor_concurrency.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Mongoid + module Errors + + # This error is raised when a bad global executor concurrency option is attempted + # to be set. + class InvalidGlobalExecutorConcurrency < MongoidError + + # Create the new error. + # + # @api private + def initialize + super( + compose_message( + "invalid_global_executor_concurrency" + ) + ) + end + end + end +end diff --git a/spec/mongoid/config_spec.rb b/spec/mongoid/config_spec.rb index e815822dee..d402d1775e 100644 --- a/spec/mongoid/config_spec.rb +++ b/spec/mongoid/config_spec.rb @@ -725,6 +725,17 @@ def self.logger }.to raise_error(Mongoid::Errors::InvalidConfigOption) end end + + context 'when invalid global_executor_concurrency option provided' do + it "raises an error" do + expect do + described_class.options = { + async_query_executor: :immediate, + global_executor_concurrency: 5 + } + end.to raise_error(Mongoid::Errors::InvalidGlobalExecutorConcurrency) + end + end end describe "#clients=" do diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb index 70841e22cb..0659525bb1 100644 --- a/spec/mongoid/contextual/mongo/documents_loader_spec.rb +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -116,19 +116,55 @@ end describe '.executor' do - context 'when immediate executor configured' do - config_override :async_query_executor, :immediate - + context 'when immediate executor requested' do it 'returns immediate executor' do - expect(described_class.executor).to eq(described_class::IMMEDIATE_EXECUTOR) + expect( + described_class.executor(:immediate) + ).to eq(described_class.immediate_executor) end end - context 'when global thread pool executor configured' do - config_override :async_query_executor, :global_thread_pool - + context 'when global thread pool executor requested' do it 'returns global thread pool executor' do - expect(described_class.executor).to eq(Mongoid.global_thread_pool_async_query_executor) + expect( + described_class.executor(:global_thread_pool) + ).to eq(described_class.global_thread_pool_async_query_executor) + end + end + + context 'when an unknown executor requested' do + it 'raises an error' do + expect do + described_class.executor(:i_am_an_invalid_option) + end.to raise_error(Mongoid::Errors::InvalidQueryExecutor) + end + end + end + + describe ".global_thread_pool_async_query_executor" do + before(:each) do + described_class.class_variable_set(:@@global_thread_pool_async_query_executor, nil) + end + + after(:each) do + described_class.class_variable_set(:@@global_thread_pool_async_query_executor, nil) + end + + context 'when global_executor_concurrency option is set' do + config_override :global_executor_concurrency, 50 + + it 'returns an executor' do + executor = described_class.global_thread_pool_async_query_executor + expect(executor).not_to be_nil + expect(executor.max_length).to eq( 50 ) + end + end + + context 'when global_executor_concurrency option is not set' do + it 'returns an executor' do + executor = described_class.global_thread_pool_async_query_executor + expect(executor).not_to be_nil + expect(executor.max_length).to eq( 4 ) end end end diff --git a/spec/mongoid_spec.rb b/spec/mongoid_spec.rb index ba1e5d2dab..5f25635594 100644 --- a/spec/mongoid_spec.rb +++ b/spec/mongoid_spec.rb @@ -95,32 +95,4 @@ expect(Mongoid.models).to include(Band) end end - - describe ".global_thread_pool_async_query_executor" do - before(:each) do - Mongoid.class_variable_set(:@@global_thread_pool_async_query_executor, nil) - end - - after(:each) do - Mongoid.class_variable_set(:@@global_thread_pool_async_query_executor, nil) - end - - context 'when global_executor_concurrency option is set' do - config_override :global_executor_concurrency, 50 - - it 'returns an executor' do - executor = Mongoid.global_thread_pool_async_query_executor - expect(executor).not_to be_nil - expect(executor.max_length).to eq( 50 ) - end - end - - context 'when global_executor_concurrency option is not set' do - it 'returns an executor' do - executor = Mongoid.global_thread_pool_async_query_executor - expect(executor).not_to be_nil - expect(executor.max_length).to eq( 4 ) - end - end - end end From 83633daa75b53d94aeefe822cd82659fecd059a9 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 15 Sep 2022 14:27:12 +0200 Subject: [PATCH 21/35] Fix dependencies --- mongoid.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoid.gemspec b/mongoid.gemspec index 23f605d2f9..732de484dd 100644 --- a/mongoid.gemspec +++ b/mongoid.gemspec @@ -38,7 +38,7 @@ Gem::Specification.new do |s| # See: https://github.com/rails/rails/pull/43951 s.add_dependency("activemodel", ['>=5.1', '<7.1', '!= 7.0.0']) s.add_dependency("mongo", ['>=2.18.0', '<3.0.0']) - s.add_dependency("concurrent-ruby", ['>= 1.1.10', '<1.2']) + s.add_dependency("concurrent-ruby", ['>= 1.0.5', '<1.2']) # The ruby2_keywords gem is recommended for handling argument delegation issues, # especially if support for 2.6 or prior is required. From b178fbf97f8d5bc7c10dd2893b88acb4dcc291f2 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Fri, 16 Sep 2022 09:29:38 +0200 Subject: [PATCH 22/35] Fix locking --- .../contextual/mongo/documents_loader.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index c09731c7a0..bb8f3959ec 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -82,11 +82,8 @@ def initialize(view, klass, criteria, executor: self.class.executor) @criteria = criteria @mutex = Mutex.new @state = :pending - @future = Concurrent::Promises.future_on(executor, self) do |task| - if task.pending? - task.send(:start) - task.execute - end + @future = Concurrent::Promises.future_on(executor) do + start && execute end end @@ -152,10 +149,17 @@ def execute private - # Mark the loader as started. + # Mark the loader as started if possible. + # + # @return [ true | false ] Whether the state was changed to :started. def start @mutex.synchronize do - @state = :started + if @state == :pending + @state = :started + true + else + false + end end end end From 0c2d4ca0381d5239be93a2bc35f991a8f5b7ee12 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:27:30 +0200 Subject: [PATCH 23/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index af6a380d88..bafd264bdb 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2431,7 +2431,7 @@ on when the query results are being accessed: .. note:: - Even though ``load_async`` method returns ``Criteria`` object, you should not + Even though ``load_async`` method returns a ``Criteria`` object, you should not do any operations on this object except accessing query results. The query is scheduled for execution immediately after calling ``load_async``, therefore later changes to the `Criteria`` object may not be applied. From a8bd5889e7141cadf69604dcd361194e086dc5be Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:27:50 +0200 Subject: [PATCH 24/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index bafd264bdb..abf9a57d5c 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2441,7 +2441,7 @@ Configuring asynchronous query execution ---------------------------------------- Asynchronous queries are disabled by default. When asynchronous queries are -disabled, ``load_async`` will execute the query immediately he current thread, +disabled, ``load_async`` will execute the query immediately on the current thread, blocking as necessary. Therefore, calling ``load_async`` on criteria in this case is roughly the equivalent of calling ``to_a``. From a917015d08194cff6fafe6146e49b8c8eea8ca3c Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:27:56 +0200 Subject: [PATCH 25/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index abf9a57d5c..0465af9c01 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2443,7 +2443,7 @@ Configuring asynchronous query execution Asynchronous queries are disabled by default. When asynchronous queries are disabled, ``load_async`` will execute the query immediately on the current thread, blocking as necessary. Therefore, calling ``load_async`` on criteria in this case -is roughly the equivalent of calling ``to_a``. +is roughly the equivalent of calling ``to_a`` to force query execution. In order to enable asynchronous query execution, the following config options should be set: From 7ceb4c4762b07a9b78fbacf7937fdc9eb7693118 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:28:10 +0200 Subject: [PATCH 26/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 0465af9c01..13c5388f49 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2455,5 +2455,5 @@ should be set: options: # Execute asynchronous queries using a global thread pool. async_query_executor: :global_thread_pool - # Number of threads in the pool. This option is defaulted to 4. + # Number of threads in the pool. The default is 4. # global_executor_concurrency: 4 From 5a80cc444e180701dc21761f196224cc890fc6da Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:28:16 +0200 Subject: [PATCH 27/35] Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 13c5388f49..76de1f007e 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -2446,7 +2446,7 @@ blocking as necessary. Therefore, calling ``load_async`` on criteria in this cas is roughly the equivalent of calling ``to_a`` to force query execution. In order to enable asynchronous query execution, the following config options -should be set: +must be set: .. code-block:: yaml From c8d351b4045333922aedfad09d1dbe3b2e5ec054 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 27 Sep 2022 18:28:23 +0200 Subject: [PATCH 28/35] Update lib/mongoid/config/validators/async_query_executor.rb Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- lib/mongoid/config/validators/async_query_executor.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mongoid/config/validators/async_query_executor.rb b/lib/mongoid/config/validators/async_query_executor.rb index b98eb8d6fb..b1721b5af4 100644 --- a/lib/mongoid/config/validators/async_query_executor.rb +++ b/lib/mongoid/config/validators/async_query_executor.rb @@ -5,6 +5,8 @@ module Config module Validators # Validator for async query executor configuration. + # + # @api private module AsyncQueryExecutor extend self From 2374e42d59a280bcafeca15b4d9cf04101ff4b68 Mon Sep 17 00:00:00 2001 From: Neil Shweky Date: Thu, 15 Sep 2022 09:58:07 -0400 Subject: [PATCH 29/35] MONGOID-5164 Use None criteria instead of $in with empty list in HABTM association (#5467) * MONGOID-5164 see if test fails when executing a query * MONGOID-5164 fix tests * omit apply_scope --- .../association/referenced/has_and_belongs_to_many.rb | 8 ++++++-- .../referenced/has_and_belongs_to_many/buildable_spec.rb | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/mongoid/association/referenced/has_and_belongs_to_many.rb b/lib/mongoid/association/referenced/has_and_belongs_to_many.rb index 1daa94f963..a369fb6b17 100644 --- a/lib/mongoid/association/referenced/has_and_belongs_to_many.rb +++ b/lib/mongoid/association/referenced/has_and_belongs_to_many.rb @@ -263,8 +263,12 @@ def with_ordering(criteria) def query_criteria(id_list) crit = relation_class.criteria - crit = crit.apply_scope(scope) - crit = crit.all_of(primary_key => {"$in" => id_list || []}) + crit = if id_list + crit = crit.apply_scope(scope) + crit.all_of(primary_key => { "$in" => id_list }) + else + crit.none + end with_ordering(crit) end end diff --git a/spec/mongoid/association/referenced/has_and_belongs_to_many/buildable_spec.rb b/spec/mongoid/association/referenced/has_and_belongs_to_many/buildable_spec.rb index cf89504bdd..9a8df0be7e 100644 --- a/spec/mongoid/association/referenced/has_and_belongs_to_many/buildable_spec.rb +++ b/spec/mongoid/association/referenced/has_and_belongs_to_many/buildable_spec.rb @@ -111,7 +111,7 @@ end let(:criteria) do - Preference.all_of("_id" => { "$in" => [] }) + Preference.none end it "a criteria object" do From 73059a94aab0108329654c6c9295fe537bf772d1 Mon Sep 17 00:00:00 2001 From: Neil Shweky Date: Fri, 23 Sep 2022 14:16:36 -0400 Subject: [PATCH 30/35] RUBY-2846 use BSON _bson_to_i method if it exists (#5473) * RUBY-2896 use BSON _bson_to_i method if it exists * RUBY-2846 change bson_master gemfile to test changes * Update gemfiles/bson_master.gemfile --- lib/mongoid/extensions/time_with_zone.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/mongoid/extensions/time_with_zone.rb b/lib/mongoid/extensions/time_with_zone.rb index 48739bc141..10c17cc3e5 100644 --- a/lib/mongoid/extensions/time_with_zone.rb +++ b/lib/mongoid/extensions/time_with_zone.rb @@ -26,9 +26,10 @@ def mongoize end # This code is copied from Time class extension in bson-ruby gem. It - # should be removed from here when added to bson-ruby. - # See https://jira.mongodb.org/browse/RUBY-2846. + # should be removed from here when the minimum BSON version is 5+. + # See https://jira.mongodb.org/browse/MONGOID-5491. def _bson_to_i + return super if defined?(super) # Workaround for JRuby's #to_i rounding negative timestamps up # rather than down (https://github.com/jruby/jruby/issues/6104) if BSON::Environment.jruby? From f010f081027c6bb3ee41bcc78a6e61a454a0b0a6 Mon Sep 17 00:00:00 2001 From: Neil Shweky Date: Fri, 23 Sep 2022 15:22:29 -0400 Subject: [PATCH 31/35] MONGOID-5227 Add test and adjust docs for dotted attribute assignment in default scope (#5474) * MONGOID-5227 Add test and adjust docs for dotted attribute assignment in default scope * MONGOID-5227 adjust docs * MONGOID-5227 clarify the docs and add tests * Update docs/reference/queries.txt Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> Co-authored-by: Oleg Pudeyev <39304720+p-mongo@users.noreply.github.com> --- docs/reference/queries.txt | 43 ++++++++++++++++++--- spec/mongoid/scopable_spec.rb | 70 +++++++++++++++++++++++++++++++++++ spec/support/models/band.rb | 1 + 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/reference/queries.txt b/docs/reference/queries.txt index 76de1f007e..3b837b9594 100644 --- a/docs/reference/queries.txt +++ b/docs/reference/queries.txt @@ -1887,8 +1887,8 @@ in the default scope, the value in the default scope takes precedence: Band.new # => # Because a default scope initializes fields in new models as just described, -defining a default scope with a dotted key and a simple literal value is -not possible: +defining a default scope with a dotted key and a simple literal value, while +possible, is not recommended: .. code-block:: ruby @@ -1900,7 +1900,36 @@ not possible: default_scope ->{ where('tags.foo' => 'bar') } end - Band.create! # exception: BSON::String::IllegalKey ('tags.foo' is an illegal key in MongoDB. Keys may not start with '$' or contain a '.'.) + Band.create! + # => Created document: {"_id"=>BSON::ObjectId('632de48f3282a404bee1877b'), "tags.foo"=>"bar"} + Band.create!(tags: { 'foo' => 'bar' }) + # => Created document: {"_id"=>BSON::ObjectId('632de4ad3282a404bee1877c'), "tags.foo"=>"bar", "tags"=>{"foo"=>"bar"}} + Band.all.to_a + # => [ #"bar"}> ] + +Mongoid 8 allows dotted keys to be used in Mongoid, and when creating a document, +the scope is added as a dotted key in the attributes: + +.. code-block:: ruby + + Band.new.attribute + # => {"_id"=>BSON::ObjectId('632de97d3282a404bee1877d'), "tags.foo"=>"bar"} + +Whereas when querying, Mongoid looks for an embedded document: + +.. code-block:: ruby + + Band.create! + # => Created document: {"_id"=>BSON::ObjectId('632de48f3282a404bee1877b'), "tags.foo"=>"bar"} + Band.where + # => #"bar"} + options: {} + class: Band + embedded: false> + # This looks for something like: { tags: { "foo" => "bar" } } + Band.count + # => 0 A workaround is to define the default scope as a complex query: @@ -1914,9 +1943,11 @@ A workaround is to define the default scope as a complex query: default_scope ->{ where('tags.foo' => {'$eq' => 'bar'}) } end - Band.create!(tags: {hello: 'world'}) - Band.create!(tags: {foo: 'bar'}) - Band.count # => 1 + Band.create!(tags: { hello: 'world' }) + Band.create!(tags: { foo: 'bar' }) + # does not add a "tags.foo" dotted attribute + Band.count + # => 1 You can tell Mongoid not to apply the default scope by using ``unscoped``, which can be inline or take a block. diff --git a/spec/mongoid/scopable_spec.rb b/spec/mongoid/scopable_spec.rb index 132194e8d5..060fa0d960 100644 --- a/spec/mongoid/scopable_spec.rb +++ b/spec/mongoid/scopable_spec.rb @@ -121,6 +121,76 @@ expect(selector).to eq({'active' => true}) end end + + context "when the default scope is dotted" do + + let(:criteria) do + Band.where('tags.foo' => 'bar') + end + + before do + Band.default_scope ->{ criteria } + end + + after do + Band.default_scoping = nil + end + + let!(:band) do + Band.create! + end + + it "adds the scope as a dotted key attribute" do + expect(band.attributes['tags.foo']).to eq('bar') + end + + it "adds the default scope to the class" do + expect(Band.default_scoping.call).to eq(criteria) + end + + it "flags as being default scoped" do + expect(Band).to be_default_scoping + end + + it "does not find the correct document" do + expect(Band.count).to eq(0) + end + end + + context "when the default scope is dotted with a query" do + + let(:criteria) do + Band.where('tags.foo' => {'$eq' => 'bar'}) + end + + before do + Band.default_scope ->{ criteria } + end + + after do + Band.default_scoping = nil + end + + let!(:band) do + Band.create!('tags' => { 'foo' => 'bar' }) + end + + it "does not add the scope as a dotted key attribute" do + expect(band.attributes).to_not have_key('tags.foo') + end + + it "adds the default scope to the class" do + expect(Band.default_scoping.call).to eq(criteria) + end + + it "flags as being default scoped" do + expect(Band).to be_default_scoping + end + + it "finds the correct document" do + expect(Band.where.first).to eq(band) + end + end end describe ".default_scopable?" do diff --git a/spec/support/models/band.rb b/spec/support/models/band.rb index e142a0dc84..6723ffcbff 100644 --- a/spec/support/models/band.rb +++ b/spec/support/models/band.rb @@ -21,6 +21,7 @@ class Band field :founded, type: Date field :deleted, type: Boolean field :mojo, type: Object + field :tags, type: Hash field :fans embeds_many :records, cascade_callbacks: true From 58c2d49ca1efa090bef9ca819312302e1d3fc03e Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 28 Sep 2022 09:52:51 +0200 Subject: [PATCH 32/35] Fix code review remarks --- .../contextual/mongo/documents_loader.rb | 23 +++++++++++++------ .../contextual/mongo/documents_loader_spec.rb | 13 +++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index bb8f3959ec..a3bec79fbf 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -29,13 +29,22 @@ def self.immediate_executor # @return [ Concurrent::ThreadPoolExecutor ] The executor # to be used to execute document loading tasks. def self.global_thread_pool_async_query_executor + create_pool = Proc.new do |concurrency| + Concurrent::ThreadPoolExecutor.new( + min_threads: 0, + max_threads: concurrency, + max_queue: concurrency * 4, + fallback_policy: :caller_runs + ) + end concurrency = Mongoid.global_executor_concurrency || 4 - @@global_thread_pool_async_query_executor ||= Concurrent::ThreadPoolExecutor.new( - min_threads: 0, - max_threads: concurrency, - max_queue: concurrency * 4, - fallback_policy: :caller_runs - ) + @@global_thread_pool_async_query_executor ||= create_pool.call(concurrency) + + if @@global_thread_pool_async_query_executor.max_length != concurrency + @@global_thread_pool_async_query_executor = create_pool.call(concurrency) + + end + @@global_thread_pool_async_query_executor end # Returns suitable executor according to Mongoid config options. @@ -61,7 +70,7 @@ def self.executor(name = Mongoid.async_query_executor) # @return [ Mongoid::Criteria ] Criteria that specifies which documents should # be loaded. Exposed here because `eager_loadable?` method from - # `Association::EagerLoadable` expects is to be available. + # `Association::EagerLoadable` expects this to be available. attr_accessor :criteria # Instantiates the document loader instance and immediately schedules diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb index 0659525bb1..dd53d026b0 100644 --- a/spec/mongoid/contextual/mongo/documents_loader_spec.rb +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -167,5 +167,18 @@ expect(executor.max_length).to eq( 4 ) end end + + context 'when global_executor_concurrency option changes' do + config_override :global_executor_concurrency, 50 + + it 'creates new executor' do + first_executor = described_class.global_thread_pool_async_query_executor + Mongoid.global_executor_concurrency = 100 + second_executor = described_class.global_thread_pool_async_query_executor + + expect(first_executor).not_to eq(second_executor) + expect(second_executor.max_length).to eq( 100 ) + end + end end end From 25b8f75af6e7936ff7022d043c62082adc5e65e4 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 29 Sep 2022 11:04:52 +0200 Subject: [PATCH 33/35] Shutdown pool gracefully --- lib/mongoid/contextual/mongo/documents_loader.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mongoid/contextual/mongo/documents_loader.rb b/lib/mongoid/contextual/mongo/documents_loader.rb index a3bec79fbf..0ed7efea22 100644 --- a/lib/mongoid/contextual/mongo/documents_loader.rb +++ b/lib/mongoid/contextual/mongo/documents_loader.rb @@ -39,10 +39,10 @@ def self.global_thread_pool_async_query_executor end concurrency = Mongoid.global_executor_concurrency || 4 @@global_thread_pool_async_query_executor ||= create_pool.call(concurrency) - if @@global_thread_pool_async_query_executor.max_length != concurrency + old_pool = @@global_thread_pool_async_query_executor @@global_thread_pool_async_query_executor = create_pool.call(concurrency) - + old_pool.shutdown end @@global_thread_pool_async_query_executor end From 34a76da5d0b94ff5c127a8acf3713df01e6517ca Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 29 Sep 2022 15:11:24 +0200 Subject: [PATCH 34/35] 5445 --- lib/mongoid.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mongoid.rb b/lib/mongoid.rb index 22816bd9e2..fdea622c97 100644 --- a/lib/mongoid.rb +++ b/lib/mongoid.rb @@ -12,6 +12,8 @@ require "active_support/time_with_zone" require "active_model" +require 'concurrent-ruby' + require "mongo" require 'mongo/active_support' From 5e85f80be5ff0e823f6483ec46fbc2abf0f4bd08 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Fri, 30 Sep 2022 09:47:21 +0200 Subject: [PATCH 35/35] Disable some specs on JRuby --- spec/mongoid/contextual/mongo/documents_loader_spec.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/mongoid/contextual/mongo/documents_loader_spec.rb b/spec/mongoid/contextual/mongo/documents_loader_spec.rb index dd53d026b0..936186c54f 100644 --- a/spec/mongoid/contextual/mongo/documents_loader_spec.rb +++ b/spec/mongoid/contextual/mongo/documents_loader_spec.rb @@ -3,6 +3,9 @@ require "spec_helper" describe Mongoid::Contextual::Mongo::DocumentsLoader do + # https://jira.mongodb.org/browse/MONGOID-5505 + require_mri + let(:view) do double('view').tap do |view| allow(view).to receive(:map)