fix: correctly identify missing verification for bi-directional pacts

lib/pact_broker/matrix/query_builder.rb

@@ -0,0 +1,100 @@
+module PactBroker
+  module Matrix
+    class QueryBuilder
+      LV = :latest_verification_id_for_pact_version_and_provider_version
+      LP = :latest_pact_publication_ids_for_consumer_versions
+
+      def self.provider_or_provider_version_or_verification_in(selectors, allow_null_provider_version = false, qualifier)
+        most_specific_criteria = selectors.collect(&:most_specific_provider_criterion)
+
+        verification_ids = collect_ids(most_specific_criteria, :verification_ids)
+        provider_version_ids = collect_ids(most_specific_criteria, :pacticipant_version_id)
+        provider_ids = collect_ids(most_specific_criteria, :pacticipant_id)
+
+        ors = [
+          { verification_id: verification_ids },
+          { Sequel[qualifier][:provider_version_id] => provider_version_ids },
+          { Sequel[qualifier][:provider_id] => provider_ids }
+        ]
+
+        if allow_null_provider_version
+          ors << {
+            Sequel[qualifier][:provider_id] => selectors.collect{ |s| s[:pacticipant_id] },
+            Sequel[qualifier][:provider_version_id] => nil
+          }
+        end
+
+        Sequel.|(*ors)
+      end
+
+      def self.consumer_in_pacticipant_ids(selectors)
+        { consumer_id: selectors.collect(&:pacticipant_id) }
+      end
+
+      def self.consumer_or_consumer_version_or_pact_publication_in(selectors, qualifier)
+        most_specific_criteria = selectors.collect(&:most_specific_consumer_criterion)
+        pact_publication_ids = collect_ids(most_specific_criteria, :pact_publication_ids)
+        consumer_version_ids = collect_ids(most_specific_criteria, :pacticipant_version_id)
+        consumer_ids = collect_ids(most_specific_criteria, :pacticipant_id)
+
+        Sequel.|(
+          { Sequel[qualifier][:pact_publication_id] => pact_publication_ids },
+          { Sequel[qualifier][:consumer_version_id] => consumer_version_ids },
+          { Sequel[qualifier][:consumer_id] => consumer_ids }
+        )
+      end
+
+      # Some selecters are specified in the query, others are implied (when only one pacticipant is specified,
+      # the integrations are automatically worked out, and the selectors for these are of type :implied )
+      # When there are 3 pacticipants that each have dependencies on each other (A->B, A->C, B->C), the query
+      # to deploy C (implied A, implied B, specified C) was returning the A->B row because it matched the
+      # implied selectors as well.
+      # This extra filter makes sure that every row that is returned actually matches one of the specified
+      # selectors.
+      def self.either_consumer_or_provider_was_specified_in_query(selectors, qualifier = nil)
+        consumer_id_field = qualifier ? Sequel[qualifier][:consumer_id] : CONSUMER_ID
+        provider_id_field = qualifier ? Sequel[qualifier][:provider_id] : PROVIDER_ID
+        specified_pacticipant_ids = selectors.select(&:specified?).collect(&:pacticipant_id)
+        Sequel.|({ consumer_id_field => specified_pacticipant_ids } , { provider_id_field => specified_pacticipant_ids })
+      end
+
+      def self.consumer_or_consumer_version_or_provider_or_provider_or_provider_version_match_selector(s)
+        consumer_or_consumer_version_match = s[:pacticipant_version_id] ? { Sequel[LP][:consumer_version_id] => s[:pacticipant_version_id] } :  { Sequel[LP][:consumer_id] => s[:pacticipant_id] }
+        provider_or_provider_version_match = s[:pacticipant_version_id] ? { Sequel[:lv][:provider_version_id] => s[:pacticipant_version_id] } :  { Sequel[LP][:provider_id] => s[:pacticipant_id] }
+        Sequel.|(consumer_or_consumer_version_match , provider_or_provider_version_match)
+      end
+
+      def self.all_pacticipant_ids selectors
+        selectors.collect(&:pacticipant_id)
+      end
+
+      def self.collect_ids(hashes, key)
+        ids = hashes.collect{ |s| s[key] }.flatten.compact
+        # must avoid an empty IN list, or Sequel makes a (0 = 1) clause for us
+        ids.empty? ? [-1] : ids
+      end
+
+      def self.collect_the_ids selectors
+        most_specific_criteria = selectors.collect(&:most_specific_consumer_criterion)
+        verification_ids = collect_ids(most_specific_criteria, :verification_ids)
+        pact_publication_ids = collect_ids(most_specific_criteria, :pact_publication_ids)
+        pacticipant_version_ids = collect_ids(most_specific_criteria, :pacticipant_version_id)
+        pacticipant_ids = collect_ids(most_specific_criteria, :pacticipant_id)
+        all_pacticipant_ids = selectors.collect(&:pacticipant_id)
+
+        specified_pacticipant_ids = selectors.select(&:specified?).collect(&:pacticipant_id)
+
+        {
+          verification_ids: verification_ids,
+          pact_publication_ids: pact_publication_ids,
+          consumer_version_ids: pacticipant_version_ids,
+          provider_version_ids: pacticipant_version_ids,
+          consumer_ids: pacticipant_ids,
+          provider_ids: pacticipant_ids,
+          all_pacticipant_ids: all_pacticipant_ids,
+          specified_pacticipant_ids: specified_pacticipant_ids
+        }
+      end
+    end
+  end
+end

lib/pact_broker/matrix/quick_row.rb

@@ -21,26 +21,27 @@ module Matrix
     LV = :latest_verification_id_for_pact_version_and_provider_version
     LP = :latest_pact_publication_ids_for_consumer_versions
 
-    CONSUMER_COLUMNS = [Sequel[:lp][:consumer_id], Sequel[:consumers][:name].as(:consumer_name), Sequel[:lp][:pact_publication_id], Sequel[:lp][:pact_version_id]]
-    PROVIDER_COLUMNS = [Sequel[:lp][:provider_id], Sequel[:providers][:name].as(:provider_name), Sequel[:lv][:verification_id]]
-    CONSUMER_VERSION_COLUMNS = [Sequel[:lp][:consumer_version_id], Sequel[:cv][:number].as(:consumer_version_number), Sequel[:cv][:order].as(:consumer_version_order)]
+    CONSUMER_COLUMNS = [Sequel[LP][:consumer_id], Sequel[:consumers][:name].as(:consumer_name), Sequel[LP][:pact_publication_id], Sequel[LP][:pact_version_id]]
+    PROVIDER_COLUMNS = [Sequel[LP][:provider_id], Sequel[:providers][:name].as(:provider_name), Sequel[:lv][:verification_id]]
+    CONSUMER_VERSION_COLUMNS = [Sequel[LP][:consumer_version_id], Sequel[:cv][:number].as(:consumer_version_number), Sequel[:cv][:order].as(:consumer_version_order)]
     PROVIDER_VERSION_COLUMNS = [Sequel[:lv][:provider_version_id], Sequel[:pv][:number].as(:provider_version_number), Sequel[:pv][:order].as(:provider_version_order)]
     ALL_COLUMNS = CONSUMER_COLUMNS + CONSUMER_VERSION_COLUMNS + PROVIDER_COLUMNS + PROVIDER_VERSION_COLUMNS
 
-    LP_LV_JOIN = { Sequel[:lp][:pact_version_id] => Sequel[:lv][:pact_version_id] }
-    CONSUMER_JOIN = { Sequel[:lp][:consumer_id] => Sequel[:consumers][:id] }
-    PROVIDER_JOIN = { Sequel[:lp][:provider_id] => Sequel[:providers][:id] }
-    CONSUMER_VERSION_JOIN = { Sequel[:lp][:consumer_version_id] => Sequel[:cv][:id] }
+    LP_LV_JOIN = { Sequel[LP][:pact_version_id] => Sequel[:lv][:pact_version_id] }
+    CONSUMER_JOIN = { Sequel[LP][:consumer_id] => Sequel[:consumers][:id] }
+    PROVIDER_JOIN = { Sequel[LP][:provider_id] => Sequel[:providers][:id] }
+    CONSUMER_VERSION_JOIN = { Sequel[LP][:consumer_version_id] => Sequel[:cv][:id] }
     PROVIDER_VERSION_JOIN = { Sequel[:lv][:provider_version_id] => Sequel[:pv][:id] }
 
-    RAW_QUERY = Sequel::Model.db[Sequel.as(LP, :lp)]
+    RAW_QUERY = Sequel::Model.db[LP]
       .select(*ALL_COLUMNS)
       .left_outer_join(LV, LP_LV_JOIN, { table_alias: :lv } )
       .join(:pacticipants, CONSUMER_JOIN, { table_alias: :consumers })
       .join(:pacticipants, PROVIDER_JOIN, { table_alias: :providers })
       .join(:versions, CONSUMER_VERSION_JOIN, { table_alias: :cv })
       .left_outer_join(:versions, PROVIDER_VERSION_JOIN, { table_alias: :pv } )
 
+
     ALIASED_QUERY = Sequel.as(RAW_QUERY, :quick_rows)
 
     class QuickRow < Sequel::Model(ALIASED_QUERY)
@@ -101,36 +102,36 @@ class QueryHelper
           def self.consumer_and_provider_in selectors
             Sequel.&(
               Sequel.|(
-                *consumer_and_maybe_consumer_version_match_any_selector(selectors)
+                *consumer_or_consumer_version_match_any_selector(selectors)
               ),
               Sequel.|(
-                *provider_and_maybe_provider_version_match_any_selector_or_verification_is_missing(selectors)
+                *provider_or_provider_version_match_any_selector_or_verification_is_missing(selectors)
               ),
               either_consumer_or_provider_was_specified_in_query(selectors)
             )
           end
 
-          def self.consumer_and_maybe_consumer_version_match_any_selector(selectors)
-            selectors.collect { |s| consumer_and_maybe_consumer_version_match_selector(s) }
+          def self.consumer_or_consumer_version_match_any_selector(selectors)
+            selectors.collect { |s| most_specific_consumer_criterion(s) }
           end
 
-          def self.consumer_and_maybe_consumer_version_match_selector(s)
+          def self.most_specific_consumer_criterion(s, qualifier = :quick_rows)
             if s[:pact_publication_ids]
-              { PACT_PUBLICATION_ID => s[:pact_publication_ids] }
+              { Sequel[qualifier][:pact_publication_id] => s[:pact_publication_ids] }
             elsif s[:pacticipant_version_id]
-              { CONSUMER_ID => s[:pacticipant_id], CONSUMER_VERSION_ID => s[:pacticipant_version_id] }
+              { Sequel[qualifier][:consumer_version_id] => s[:pacticipant_version_id] }
             else
-              { CONSUMER_ID => s[:pacticipant_id] }
+              { Sequel[qualifier][:consumer_id] => s[:pacticipant_id] }
             end
           end
 
-          def self.provider_and_maybe_provider_version_match_selector(s)
-            if s[:verification_ids]
-              { VERIFICATION_ID => s[:verification_ids] }
-            elsif s[:pacticipant_version_id]
-              { PROVIDER_ID => s[:pacticipant_id], PROVIDER_VERSION_ID => s[:pacticipant_version_id] }
+          def self.most_specific_provider_criterion(selector, qualifier = :quick_rows)
+            if selector[:verification_ids]
+              { Sequel[qualifier][:verification_id] => selector[:verification_ids] }
+            elsif selector[:pacticipant_version_id]
+              { Sequel[qualifier][:provider_version_id] => selector[:pacticipant_version_id] }
             else
-              { PROVIDER_ID => s[:pacticipant_id] }
+              { Sequel[qualifier][:provider_id] => selector[:pacticipant_id] }
             end
           end
 
@@ -140,9 +141,9 @@ def self.provider_verification_is_missing_for_matching_selector(s)
             { PROVIDER_ID => s[:pacticipant_id], PROVIDER_VERSION_ID => nil }
           end
 
-          def self.provider_and_maybe_provider_version_match_any_selector_or_verification_is_missing(selectors)
+          def self.provider_or_provider_version_match_any_selector_or_verification_is_missing(selectors)
             selectors.collect { |s|
-              provider_and_maybe_provider_version_match_selector(s)
+              most_specific_provider_criterion(s)
             } + selectors.collect { |s|
               provider_verification_is_missing_for_matching_selector(s)
             }
@@ -155,16 +156,17 @@ def self.provider_and_maybe_provider_version_match_any_selector_or_verification_
           # implied selectors as well.
           # This extra filter makes sure that every row that is returned actually matches one of the specified
           # selectors.
-          def self.either_consumer_or_provider_was_specified_in_query(selectors)
-            specified_pacticipant_ids = selectors.select{ |s| s[:type] == :specified }.collect{ |s| s[:pacticipant_id] }
-            Sequel.|({ CONSUMER_ID => specified_pacticipant_ids } , { PROVIDER_ID => specified_pacticipant_ids })
+          def self.either_consumer_or_provider_was_specified_in_query(selectors, qualifier = nil)
+            consumer_id_field = qualifier ? Sequel[qualifier][:consumer_id] : CONSUMER_ID
+            provider_id_field = qualifier ? Sequel[qualifier][:provider_id] : PROVIDER_ID
+            specified_pacticipant_ids = selectors.select(&:specified?).collect(&:pacticipant_id)
+            Sequel.|({ consumer_id_field => specified_pacticipant_ids } , { provider_id_field => specified_pacticipant_ids })
           end
 
           def self.consumer_or_consumer_version_or_provider_or_provider_or_provider_version_match_selector(s)
-            Sequel.|(
-              s[:pacticipant_version_id] ? { CONSUMER_VERSION_ID => s[:pacticipant_version_id] } :  { CONSUMER_ID => s[:pacticipant_id] },
-              s[:pacticipant_version_id] ? { PROVIDER_VERSION_ID => s[:pacticipant_version_id] } :  { PROVIDER_ID => s[:pacticipant_id] }
-            )
+            consumer_or_consumer_version_match = s[:pacticipant_version_id] ? { CONSUMER_VERSION_ID => s[:pacticipant_version_id] } :  { CONSUMER_ID => s[:pacticipant_id] }
+            provider_or_provider_version_match = s[:pacticipant_version_id] ? { PROVIDER_VERSION_ID => s[:pacticipant_version_id] } :  { PROVIDER_ID => s[:pacticipant_id] }
+            Sequel.|(consumer_or_consumer_version_match , provider_or_provider_version_match)
           end
         end
 

lib/pact_broker/matrix/quick_row_2.rb

@@ -0,0 +1,124 @@
+require 'pact_broker/pacts/all_pact_publications'
+require 'pact_broker/repositories/helpers'
+require 'pact_broker/matrix/query_builder'
+
+# The difference between this query and the query for QuickRow2 is that
+# the left outer join is done on a pre-filtered dataset so that we
+# get a row with null verification fields for a pact that has not been verified
+# by the particular providers where're interested in, rather than being excluded
+# from the dataset.
+
+module PactBroker
+  module Matrix
+    class QuickRow2 < Sequel::Model(:latest_pact_publication_ids_for_consumer_versions)
+      LV = :latest_verification_id_for_pact_version_and_provider_version
+      LP = :latest_pact_publication_ids_for_consumer_versions
+
+      LP_LV_JOIN = { Sequel[LP][:pact_version_id] => Sequel[:lv][:pact_version_id] }
+      CONSUMER_JOIN = { Sequel[LP][:consumer_id] => Sequel[:consumers][:id] }
+      PROVIDER_JOIN = { Sequel[LP][:provider_id] => Sequel[:providers][:id] }
+      CONSUMER_VERSION_JOIN = { Sequel[LP][:consumer_version_id] => Sequel[:cv][:id] }
+      PROVIDER_VERSION_JOIN = { Sequel[:lv][:provider_version_id] => Sequel[:pv][:id] }
+
+      CONSUMER_COLUMNS = [Sequel[LP][:consumer_id], Sequel[:consumers][:name].as(:consumer_name), Sequel[LP][:pact_publication_id], Sequel[LP][:pact_version_id]]
+      PROVIDER_COLUMNS = [Sequel[LP][:provider_id], Sequel[:providers][:name].as(:provider_name), Sequel[:lv][:verification_id]]
+      CONSUMER_VERSION_COLUMNS = [Sequel[LP][:consumer_version_id], Sequel[:cv][:number].as(:consumer_version_number), Sequel[:cv][:order].as(:consumer_version_order)]
+      PROVIDER_VERSION_COLUMNS = [Sequel[:lv][:provider_version_id], Sequel[:pv][:number].as(:provider_version_number), Sequel[:pv][:order].as(:provider_version_order)]
+      ALL_COLUMNS = CONSUMER_COLUMNS + CONSUMER_VERSION_COLUMNS + PROVIDER_COLUMNS + PROVIDER_VERSION_COLUMNS
+
+      SELECT_ALL_COLUMN_ARGS = [:select_all_columns] + ALL_COLUMNS
+
+      dataset_module do
+        include PactBroker::Repositories::Helpers
+
+        select *SELECT_ALL_COLUMN_ARGS
+
+        def matching_selectors selectors
+          if selectors.size == 1
+            matching_one_selector(selectors.first)
+          else
+            matching_multiple_selectors(selectors)
+          end
+        end
+
+        # When we have one selector, we need to join ALL the verifications to find out
+        # what integrations exist
+        def matching_one_selector(selector)
+          select_all_columns
+            .join_verifications
+            .join_pacticipants_and_pacticipant_versions
+            .where {
+              QueryBuilder.consumer_or_consumer_version_or_provider_or_provider_or_provider_version_match_selector(selector)
+            }
+        end
+
+        # When the user has specified multiple selectors, we only want to join the verifications for
+        # the specified selectors. This is because of the behaviour of the left outer join.
+        # Imagine a pact has been verified by a provider version that was NOT specified in the selectors.
+        # If we join all the verifications and THEN filter the rows to only show the versions specified
+        # in the selectors, we won't get a row for that pact, and hence, we won't
+        # know that it hasn't been verified by the provider version we're interested in.
+        # Instead, we need to filter the verifications dataset down to only the ones specified in the selectors first,
+        # and THEN join them to the pacts, so that we get a row for the pact with null provider version
+        # and verification fields.
+        def matching_multiple_selectors(selectors)
+          select_all_columns
+            .join_verifications_for(selectors)
+            .join_pacticipants_and_pacticipant_versions
+            .where {
+              Sequel.&(
+                QueryBuilder.consumer_or_consumer_version_or_pact_publication_in(selectors, LP),
+                QueryBuilder.either_consumer_or_provider_was_specified_in_query(selectors, LP)
+              )
+            }
+            .from_self(alias: :t9)
+            .where {
+              QueryBuilder.provider_or_provider_version_or_verification_in(selectors, true, :t9)
+            }
+        end
+
+        def join_pacticipants_and_pacticipant_versions
+          join_consumers
+            .join_providers
+            .join_consumer_versions
+            .join_provider_versions
+        end
+
+        def join_consumers
+          join(:pacticipants, CONSUMER_JOIN, { table_alias: :consumers })
+        end
+
+        def join_providers
+          join(:pacticipants, PROVIDER_JOIN, { table_alias: :providers })
+        end
+
+        def join_consumer_versions
+          join(:versions, CONSUMER_VERSION_JOIN, { table_alias: :cv })
+        end
+
+        def join_provider_versions
+          left_outer_join(:versions, PROVIDER_VERSION_JOIN, { table_alias: :pv } )
+        end
+
+        def join_verifications_for(selectors)
+          left_outer_join(verifications_for(selectors), LP_LV_JOIN, { table_alias: :lv } )
+        end
+
+        def join_verifications
+          left_outer_join(LV, LP_LV_JOIN, { table_alias: :lv } )
+        end
+
+        def verifications_for(selectors)
+          db[LV]
+            .select(:verification_id, :provider_version_id, :pact_version_id)
+            .where {
+              Sequel.&(
+                QueryBuilder.consumer_in_pacticipant_ids(selectors),
+                QueryBuilder.provider_or_provider_version_or_verification_in(selectors, false, LV)
+              )
+            }
+        end
+      end
+    end
+  end
+end

lib/pact_broker/matrix/repository.rb

@@ -9,6 +9,7 @@
 require 'pact_broker/matrix/resolved_selector'
 require 'pact_broker/verifications/latest_verification_id_for_pact_version_and_provider_version'
 require 'pact_broker/pacts/latest_pact_publications_by_consumer_version'
+require 'pact_broker/matrix/quick_row_2'
 
 module PactBroker
   module Matrix
@@ -81,8 +82,8 @@ def find_compatible_pacticipant_versions selectors
       # If two or more are specified, just return the integrations that involve the specified pacticipants
       def find_integrations_for_specified_selectors(resolved_specified_selectors)
         specified_pacticipant_names = resolved_specified_selectors.collect(&:pacticipant_name)
-        QuickRow
-          .pacticipant_names_and_ids
+
+        QuickRow2
           .matching_selectors(resolved_specified_selectors)
           .distinct
           .all