From a3aea484a8562843cddc04e36379a86db4b7adeb Mon Sep 17 00:00:00 2001 From: Beth Skurrie Date: Thu, 25 Jan 2018 15:53:19 +1100 Subject: [PATCH] feat(dashboard api): speed up loading of verification tags --- ...180119_update_latest_triggered_webhooks.rb | 2 +- ...0180123_create_latest_verification_tags.rb | 43 ---------- .../20180123_create_tags_with_latest_flag.rb | 28 ++++++ db/migrations/migration_helper.rb | 8 ++ .../api/decorators/dashboard_decorator.rb | 2 +- lib/pact_broker/index/service.rb | 5 +- lib/pact_broker/matrix/row.rb | 16 ++-- lib/pact_broker/tags/tag_with_latest_flag.rb | 18 ++++ .../decorators/dashboard_decorator_spec.rb | 2 +- ...23_create_latest_verification_tags_spec.rb | 85 ------------------- 10 files changed, 66 insertions(+), 143 deletions(-) delete mode 100644 db/migrations/20180123_create_latest_verification_tags.rb create mode 100644 db/migrations/20180123_create_tags_with_latest_flag.rb create mode 100644 lib/pact_broker/tags/tag_with_latest_flag.rb delete mode 100644 spec/migrations/20180123_create_latest_verification_tags_spec.rb diff --git a/db/migrations/20180119_update_latest_triggered_webhooks.rb b/db/migrations/20180119_update_latest_triggered_webhooks.rb index cc0ada6a7..5d2e0533f 100644 --- a/db/migrations/20180119_update_latest_triggered_webhooks.rb +++ b/db/migrations/20180119_update_latest_triggered_webhooks.rb @@ -18,7 +18,7 @@ ) create_or_replace_view(:latest_triggered_webhook_ids, - "select tw.webhook_uuid, tw.consumer_id, tw.provider_id, max(tw.id) as latest_triggered_webhook_id + "select tw.webhook_uuid, tw.consumer_id, tw.provider_id, ltwcd.latest_triggered_webhook_created_at, max(tw.id) as latest_triggered_webhook_id from latest_triggered_webhook_creation_dates ltwcd inner join triggered_webhooks tw on tw.consumer_id = ltwcd.consumer_id diff --git a/db/migrations/20180123_create_latest_verification_tags.rb b/db/migrations/20180123_create_latest_verification_tags.rb deleted file mode 100644 index 1d6628c40..000000000 --- a/db/migrations/20180123_create_latest_verification_tags.rb +++ /dev/null @@ -1,43 +0,0 @@ -Sequel.migration do - change do - # the provider version order of the latest verification for each consumer/provider/tag - create_view(:latest_tagged_verification_provider_version_orders, - " - select m.consumer_id, m.provider_id, t.name as tag_name, max(m.provider_version_order) as latest_provider_version_order - from latest_matrix m - inner join tags t - on m.provider_version_id = t.version_id - where m.provider_version_order is not null - group by m.consumer_id, m.provider_id, t.name - " - ) - -=begin - The tags for which the given verification is the latest of that tag - Imagine that: - - provider v1 has verification - has tag dev - has tag prod <- latest - provider v2 has verification - has tag dev <-latest - provider v3 has tag dev - - This table would contain the prod tag row for the v1 verification - This table would contain the dev tag row for the v2 verification -=end - create_view(:latest_verification_tags, - " - select t.*, m.verification_id - from latest_matrix m - inner join latest_tagged_verification_provider_version_orders l - on m.consumer_id = l.consumer_id - and m.provider_id = l.provider_id - and m.provider_version_order = l.latest_provider_version_order - inner join tags t - on l.tag_name = t.name - and m.provider_version_id = t.version_id - " - ) - end -end \ No newline at end of file diff --git a/db/migrations/20180123_create_tags_with_latest_flag.rb b/db/migrations/20180123_create_tags_with_latest_flag.rb new file mode 100644 index 000000000..ca151545c --- /dev/null +++ b/db/migrations/20180123_create_tags_with_latest_flag.rb @@ -0,0 +1,28 @@ +require_relative 'migration_helper' + +Sequel.migration do + change do + create_view(:latest_tagged_version_orders, + PactBroker::MigrationHelper.sqlite_safe(" + select v.pacticipant_id, t.name as tag_name, max(v.order) as latest_version_order, 1 as latest + from tags t + inner join versions v + on v.id = t.version_id + group by v.pacticipant_id, t.name + ") + ) + + create_view(:tags_with_latest_flag, + PactBroker::MigrationHelper.sqlite_safe(" + select t.*, ltvo.latest + from tags t + inner join versions v + on v.id = t.version_id + left outer join latest_tagged_version_orders ltvo + on t.name = ltvo.tag_name + and v.pacticipant_id = ltvo.pacticipant_id + and v.order = ltvo.latest_version_order + ") + ) + end +end diff --git a/db/migrations/migration_helper.rb b/db/migrations/migration_helper.rb index bbbae96c8..ed7971d68 100644 --- a/db/migrations/migration_helper.rb +++ b/db/migrations/migration_helper.rb @@ -21,5 +21,13 @@ def with_mysql def adapter Sequel::Model.db.adapter_scheme.to_s end + + def sqlite_safe string + if adapter == 'sqlite' + string.gsub(/\border\b/, '`order`') + else + string + end + end end end diff --git a/lib/pact_broker/api/decorators/dashboard_decorator.rb b/lib/pact_broker/api/decorators/dashboard_decorator.rb index 1d16f9510..5f8e3f7c0 100644 --- a/lib/pact_broker/api/decorators/dashboard_decorator.rb +++ b/lib/pact_broker/api/decorators/dashboard_decorator.rb @@ -131,7 +131,7 @@ def verification_tags(index_item, base_url) fake_tag = OpenStruct.new(name: tag.name, version: index_item.provider_version) { name: tag.name, - latest: true, + latest: tag.latest?, _links: { self: { href: tag_url(base_url, fake_tag) diff --git a/lib/pact_broker/index/service.rb b/lib/pact_broker/index/service.rb index bb04420df..861008469 100644 --- a/lib/pact_broker/index/service.rb +++ b/lib/pact_broker/index/service.rb @@ -23,6 +23,7 @@ def self.find_index_items options = {} .eager(:webhooks) .order(:consumer_name, :provider_name) .eager(:consumer_version_tags) + .eager(:provider_version_tags) .all end @@ -35,7 +36,7 @@ def self.find_index_items options = {} .eager(:webhooks) .order(:consumer_name, :provider_name) .eager(:consumer_version_tags) - .eager(:latest_verification_tags) + .eager(:provider_version_tags) if options[:tags].is_a?(Array) tagged_rows = tagged_rows.where(Sequel[:head_pact_publications][:tag_name] => options[:tags]).or(Sequel[:head_pact_publications][:tag_name] => nil) @@ -67,7 +68,7 @@ def self.find_index_items options = {} row.webhooks, row.latest_triggered_webhooks, tag_names, - row.latest_verification_tags + row.provider_version_tags.select(&:latest?) ) end diff --git a/lib/pact_broker/matrix/row.rb b/lib/pact_broker/matrix/row.rb index 939e7a364..b12fabe5c 100644 --- a/lib/pact_broker/matrix/row.rb +++ b/lib/pact_broker/matrix/row.rb @@ -1,20 +1,16 @@ require 'pact_broker/repositories/helpers' require 'pact_broker/webhooks/latest_triggered_webhook' require 'pact_broker/tags/latest_verification_tag' +require 'pact_broker/tags/tag_with_latest_flag' module PactBroker module Matrix class Row < Sequel::Model(:matrix) - associate(:one_to_many, :latest_triggered_webhooks, :class => "PactBroker::Webhooks::LatestTriggeredWebhook", primary_key: :pact_publication_id, key: :pact_publication_id) - - # already have this - # associate(:many_to_one, :latest_verification, :class => "PactBroker::Verifications::Repository::LatestVerificationsByConsumerVersion", primary_key: :pact_version_id, key: :pact_version_id) - # TODO modify this to work with single pacticipant webhooks associate(:one_to_many, :webhooks, :class => "PactBroker::Webhooks::Webhook", primary_key: [:consumer_id, :provider_id], key: [:consumer_id, :provider_id]) - associate(:one_to_many, :consumer_version_tags, :class => "PactBroker::Domain::Tag", primary_key: :consumer_version_id, key: :version_id) - associate(:one_to_many, :latest_verification_tags, :class => "PactBroker::Tags::LatestVerificationTag", primary_key: :verification_id, key: :verification_id) + associate(:one_to_many, :consumer_version_tags, :class => "PactBroker::Tags::TagWithLatestFlag", primary_key: :consumer_version_id, key: :version_id) + associate(:one_to_many, :provider_version_tags, :class => "PactBroker::Tags::TagWithLatestFlag", primary_key: :provider_version_id, key: :version_id) dataset_module do include PactBroker::Repositories::Helpers @@ -71,9 +67,9 @@ def consumer_head_tag_names= consumer_head_tag_names @consumer_head_tag_names = consumer_head_tag_names end - def latest_triggered_webhooks - @latest_triggered_webhooks ||= [] - end + # def latest_triggered_webhooks + # @latest_triggered_webhooks ||= [] + # end def summary "#{consumer_name}#{consumer_version_number} #{provider_name}#{provider_version_number || '?'} (r#{pact_revision_number}n#{verification_number || '?'})" diff --git a/lib/pact_broker/tags/tag_with_latest_flag.rb b/lib/pact_broker/tags/tag_with_latest_flag.rb new file mode 100644 index 000000000..d005c2ee3 --- /dev/null +++ b/lib/pact_broker/tags/tag_with_latest_flag.rb @@ -0,0 +1,18 @@ +require 'pact_broker/db' +require 'pact_broker/repositories/helpers' + +module PactBroker + module Tags + # The tag associated with the latest verification for a given tag + class TagWithLatestFlag < Sequel::Model(:tags_with_latest_flag) + + dataset_module do + include PactBroker::Repositories::Helpers + end + + def latest? + !values[:latest].nil? + end + end + end +end diff --git a/spec/lib/pact_broker/api/decorators/dashboard_decorator_spec.rb b/spec/lib/pact_broker/api/decorators/dashboard_decorator_spec.rb index 5dd969fc8..466f54340 100644 --- a/spec/lib/pact_broker/api/decorators/dashboard_decorator_spec.rb +++ b/spec/lib/pact_broker/api/decorators/dashboard_decorator_spec.rb @@ -21,7 +21,7 @@ module Decorators provider_version_number: provider_version.number, consumer_version_number: consumer_version.number, tag_names: ['prod'], - latest_verification_latest_tags: [double('tag', name: 'dev')] + latest_verification_latest_tags: [double('tag', name: 'dev', latest?: true)] ) end let(:consumer) { instance_double('PactBroker::Domain::Pacticipant', name: 'Foo') } diff --git a/spec/migrations/20180123_create_latest_verification_tags_spec.rb b/spec/migrations/20180123_create_latest_verification_tags_spec.rb deleted file mode 100644 index b7a9ea83f..000000000 --- a/spec/migrations/20180123_create_latest_verification_tags_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -describe 'latest tagged verifications', migration: true do - before do - PactBroker::Database.migrate(20180123) - end - - let(:now) { DateTime.new(2018, 2, 2) } - let!(:consumer) { create(:pacticipants, {name: 'C', created_at: now, updated_at: now}) } - let!(:provider) { create(:pacticipants, {name: 'P', created_at: now, updated_at: now}) } - let!(:consumer_version_1) { create(:versions, {number: '1', order: 1, pacticipant_id: consumer[:id], created_at: now, updated_at: now}) } - let!(:consumer_version_2) { create(:versions, {number: '2', order: 2, pacticipant_id: consumer[:id], created_at: now, updated_at: now}) } - - let!(:provider_version_1) { create(:versions, {number: '1', order: 1, pacticipant_id: provider[:id], created_at: now, updated_at: now}) } - let!(:provider_version_2) { create(:versions, {number: '2', order: 2, pacticipant_id: provider[:id], created_at: now, updated_at: now}) } - let!(:provider_version_3) { create(:versions, {number: '3', order: 3, pacticipant_id: provider[:id], created_at: now, updated_at: now}) } - - let!(:provider_version_1_prod_tag) { create(:tags, {version_id: provider_version_1[:id], name: 'prod', created_at: now, updated_at: now}, nil) } - let!(:provider_version_1_dev_tag) { create(:tags, {version_id: provider_version_1[:id], name: 'dev', created_at: now, updated_at: now}, nil) } - let!(:provider_version_2_dev_tag) { create(:tags, {version_id: provider_version_2[:id], name: 'dev', created_at: now, updated_at: now}, nil) } - - let!(:pact_version_1) { create(:pact_versions, {content: {some: 'json'}.to_json, sha: '1', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } - let!(:pact_version_2) { create(:pact_versions, {content: {some: 'json other'}.to_json, sha: '2', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } - #let!(:pact_version_3) { create(:pact_versions, {content: {some: 'json more'}.to_json, sha: '3', consumer_id: consumer[:id], provider_id: provider[:id], created_at: now}) } - let!(:pact_publication_1) do - create(:pact_publications, { - consumer_version_id: consumer_version_1[:id], - provider_id: provider[:id], - revision_number: 1, - pact_version_id: pact_version_1[:id], - created_at: now - }) - end - - let!(:pact_publication_2) do - create(:pact_publications, { - consumer_version_id: consumer_version_2[:id], - provider_id: provider[:id], - revision_number: 1, - pact_version_id: pact_version_2[:id], - created_at: now - }) - end - - # provider v1 - let!(:verification_1) do - create(:verifications, { - number: 1, - success: true, - provider_version_id: provider_version_1[:id], - pact_version_id: pact_version_1[:id], - execution_date: now, - created_at: now - }) - end - - # provider v2 - let!(:verification_2) do - create(:verifications, { - number: 2, - success: true, - provider_version_id: provider_version_2[:id], - pact_version_id: pact_version_1[:id], - execution_date: now, - created_at: now - }) - end - - # provider v2 - let!(:verification_3) do - create(:verifications, { - number: 3, - success: true, - provider_version_id: provider_version_2[:id], - pact_version_id: pact_version_1[:id], - execution_date: now, - created_at: now - }) - end - - it "includes the tag rows for which the related verification is the latest of that tag" do - rows = database[:latest_verification_tags].all - expect(rows).to contain_hash(verification_id: verification_1[:id], name: 'prod') - expect(rows).to contain_hash(verification_id: verification_3[:id], name: 'dev') - expect(rows.size).to eq 2 - end -end