diff --git a/DEVELOPER_SETUP.md b/DEVELOPER_SETUP.md index 161466f0f..09d0e6e38 100644 --- a/DEVELOPER_SETUP.md +++ b/DEVELOPER_SETUP.md @@ -36,7 +36,7 @@ Remember to rebuild the image if you change any of the gems or gem versions. ### With native install -* You will need to install Ruby 2.5, and preferably a ruby version manager. I recommend using [chruby][chruby] and [ruby-install][ruby-install]. +* You will need to install Ruby 2.7, and preferably a ruby version manager. I recommend using [chruby][chruby] and [ruby-install][ruby-install]. * Install bundler (the Ruby gem dependency manager) `gem install bundler` * Check out the pact_broker repository and cd into it. * Run `bundle install`. If you have any gem conflict issues, run `bundle update`. @@ -104,5 +104,56 @@ psql postgres://postgres:postgres@postgres/postgres bundle exec rspec path_to_your_spec.rb ``` +## Running the regression tests + +The regression tests record a series of API requests/responses using a real exported database (not included in the git repository because of the size) and store the expectations in files using the Approvals gem. They allow you to make sure that the changes you have made have not made any (unexpected) changes to the interface. + +The tests and files are stored in the [regression](regression) directory. + +To run: + +1. Set up your local development environment as described above, making sure you have `INSTALL_PG=true` exported in your shell. + +1. Make sure you have the master branch checked out. + +1. Load an exported real database into a postgres docker image. The exported file must be in the pg dump format to use this script, and it must be located in the project root directory for it to be found via the mounted directory. + + ``` + script/docker/reload.sh + + ``` +1. Clear any previously generated approvals. + + ``` + regression/script/clear.sh + ``` + +1. Run the tests. They will fail because there are no approval files yet. + + ``` + regression/script/run.sh + ``` + +1. Approval all the things. + + ``` + regression/script/approval-all.sh + ``` + +1. Run the tests again to make sure that the same results can be expected each time. + + ``` + regression/script/run.sh + ``` + +1. Check out the feature branch (or enable the feature toggle) + +1. Run the tests again. + ``` + regression/script/run.sh + ``` + +1. If there is a diff, you can set `SHOW_REGRESSION_DIFF=true`, but the output is quite noisy, and you're probably better off using diff or diffmerge to view the differences. + [chruby]: https://github.com/postmodern/chruby [ruby-install]: https://github.com/postmodern/ruby-install diff --git a/db/ddl_statements/latest_tagged_pact_consumer_version_orders.rb b/db/ddl_statements/latest_tagged_pact_consumer_version_orders.rb index 5998f9a9f..82aa4f435 100644 --- a/db/ddl_statements/latest_tagged_pact_consumer_version_orders.rb +++ b/db/ddl_statements/latest_tagged_pact_consumer_version_orders.rb @@ -21,3 +21,14 @@ def latest_tagged_pact_consumer_version_orders_v3(connection) .join(:versions, { Sequel[:lp][:consumer_version_id] => Sequel[:cv][:id] }, { table_alias: :cv} ) .join(:tags, { Sequel[:t][:version_id] => Sequel[:lp][:consumer_version_id] }, { table_alias: :t }) end + +def latest_tagged_pact_consumer_version_orders_v4(connection) + view = Sequel.as(:latest_pact_publication_ids_for_consumer_versions, :lp) + connection.from(view) + .select_group( + Sequel[:lp][:provider_id], + Sequel[:lp][:consumer_id], + Sequel[:t][:name].as(:tag_name)) + .select_append{ max(version_order).as(latest_consumer_version_order) } + .join(:tags, { Sequel[:t][:version_id] => Sequel[:lp][:consumer_version_id] }, { table_alias: :t }) +end diff --git a/db/ddl_statements/latest_tagged_pact_publications.rb b/db/ddl_statements/latest_tagged_pact_publications.rb new file mode 100644 index 000000000..c86f8a456 --- /dev/null +++ b/db/ddl_statements/latest_tagged_pact_publications.rb @@ -0,0 +1,6 @@ +LATEST_TAGGED_PACT_PUBLICATIONS = "select lp.*, o.tag_name + from latest_pact_publications_by_consumer_versions lp + inner join latest_tagged_pact_consumer_version_orders o + on lp.consumer_id = o.consumer_id + and lp.provider_id = o.provider_id + and lp.consumer_version_order = latest_consumer_version_order" diff --git a/db/ddl_statements/latest_verification_ids_for_consumer_version_tags.rb b/db/ddl_statements/latest_verification_ids_for_consumer_version_tags.rb index f542eec80..cc44bd8a1 100644 --- a/db/ddl_statements/latest_verification_ids_for_consumer_version_tags.rb +++ b/db/ddl_statements/latest_verification_ids_for_consumer_version_tags.rb @@ -51,3 +51,16 @@ on v.provider_version_id = pv.id where v.id in (select latest_verification_id from latest_verification_ids_for_pact_versions) group by pv.pacticipant_id, lpp.consumer_id, t.name" + +LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V4 = "select + lpp.provider_id, + lpp.consumer_id, + t.name as consumer_version_tag_name, + max(lv.latest_verification_id) as latest_verification_id + from latest_verification_ids_for_pact_versions lv + join latest_pact_publication_ids_for_consumer_versions lpp + on lv.pact_version_id = lpp.pact_version_id + join tags t + on lpp.consumer_version_id = t.version_id + group by lpp.provider_id, lpp.consumer_id, t.name +" diff --git a/db/migrations/20210205_add_pacticipant_id_to_tag.rb b/db/migrations/20210205_add_pacticipant_id_to_tag.rb new file mode 100644 index 000000000..78d98b11a --- /dev/null +++ b/db/migrations/20210205_add_pacticipant_id_to_tag.rb @@ -0,0 +1,17 @@ +require_relative 'migration_helper' + +include PactBroker::MigrationHelper + +Sequel.migration do + change do + alter_table(:tags) do + # TODO set_column_not_null(:pacticipant_id) + # TODO set_column_not_null(:version_order) + add_column(:pacticipant_id, Integer) + add_column(:version_order, Integer) + add_index(:version_order, name: "tags_version_order_index") + add_index(:version_id, name: "tags_version_id_index") + add_index(:pacticipant_id, name: "tags_pacticipant_id_index") + end + end +end diff --git a/db/migrations/20210206_add_index_to_tags_and_versions.rb b/db/migrations/20210206_add_index_to_tags_and_versions.rb new file mode 100644 index 000000000..b181c88e8 --- /dev/null +++ b/db/migrations/20210206_add_index_to_tags_and_versions.rb @@ -0,0 +1,27 @@ +require_relative 'migration_helper' + +include PactBroker::MigrationHelper + +Sequel.migration do + up do + if postgres? + run("CREATE INDEX tags_pacticipant_id_name_version_order_desc_index ON tags (pacticipant_id, name, version_order DESC);") + run("CREATE INDEX versions_pacticipant_id_order_desc_index ON versions (pacticipant_id, \"order\" DESC);") + else + alter_table(:tags) do + add_index([:pacticipant_id, :name, :version_order], name: "tags_pacticipant_id_name_version_order_index") + end + end + end + + down do + if postgres? + run("DROP INDEX tags_pacticipant_id_name_version_order_desc_index") + run("DROP INDEX versions_pacticipant_id_order_desc_index") + else + alter_table(:tags) do + drop_index([:pacticipant_id, :name, :version_order], name: "tags_pacticipant_id_name_version_order_index") + end + end + end +end diff --git a/db/migrations/20210207_optimise_latest_verification_ids_for_consumer_version_tags.rb b/db/migrations/20210207_optimise_latest_verification_ids_for_consumer_version_tags.rb new file mode 100644 index 000000000..dc50ea73d --- /dev/null +++ b/db/migrations/20210207_optimise_latest_verification_ids_for_consumer_version_tags.rb @@ -0,0 +1,13 @@ +require_relative '../ddl_statements' + +Sequel.migration do + up do + create_or_replace_view(:latest_verification_ids_for_consumer_version_tags, + LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V4) + end + + down do + create_or_replace_view(:latest_verification_ids_for_consumer_version_tags, + LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V3) + end +end diff --git a/db/migrations/20210208_optimise_latest_tagged_pact_cv_orders.rb b/db/migrations/20210208_optimise_latest_tagged_pact_cv_orders.rb new file mode 100644 index 000000000..6fe0135ec --- /dev/null +++ b/db/migrations/20210208_optimise_latest_tagged_pact_cv_orders.rb @@ -0,0 +1,13 @@ +require_relative '../ddl_statements' + +Sequel.migration do + up do + create_or_replace_view(:latest_tagged_pact_consumer_version_orders, + latest_tagged_pact_consumer_version_orders_v4(self)) + end + + down do + create_or_replace_view(:latest_tagged_pact_consumer_version_orders, + latest_tagged_pact_consumer_version_orders_v3(self)) + end +end diff --git a/lib/pact_broker/certificates/certificate.rb b/lib/pact_broker/certificates/certificate.rb index 686cc3b67..9b58b7f0c 100644 --- a/lib/pact_broker/certificates/certificate.rb +++ b/lib/pact_broker/certificates/certificate.rb @@ -9,7 +9,7 @@ class Certificate < Sequel::Model # Table: certificates # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('certificates_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # uuid | text | NOT NULL # description | text | # content | text | NOT NULL diff --git a/lib/pact_broker/config/setting.rb b/lib/pact_broker/config/setting.rb index 33fd27292..f63598b7e 100644 --- a/lib/pact_broker/config/setting.rb +++ b/lib/pact_broker/config/setting.rb @@ -73,7 +73,7 @@ def self.get_db_type(object) # Table: config # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('config_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # name | text | NOT NULL # type | text | NOT NULL # value | text | diff --git a/lib/pact_broker/db/data_migrations/helpers.rb b/lib/pact_broker/db/data_migrations/helpers.rb index 0465594bc..ea4f8452c 100644 --- a/lib/pact_broker/db/data_migrations/helpers.rb +++ b/lib/pact_broker/db/data_migrations/helpers.rb @@ -5,6 +5,10 @@ module Helpers def column_exists?(connection, table, column) connection.table_exists?(table) && connection.schema(table).find{|col| col.first == column } end + + def columns_exist?(connection, table, columns) + columns.all?{ | column | column_exists?(connection, table, column) } + end end end end diff --git a/lib/pact_broker/db/data_migrations/set_extra_columns_for_tags.rb b/lib/pact_broker/db/data_migrations/set_extra_columns_for_tags.rb new file mode 100644 index 000000000..9033f8365 --- /dev/null +++ b/lib/pact_broker/db/data_migrations/set_extra_columns_for_tags.rb @@ -0,0 +1,29 @@ +require 'pact_broker/db/data_migrations/helpers' + +module PactBroker + module DB + module DataMigrations + class SetExtraColumnsForTags + extend Helpers + + def self.call(connection) + if columns_exist?(connection, :tags, [:version_id, :pacticipant_id]) && + columns_exist?(connection, :versions, [:id, :pacticipant_id]) + connection[:tags].update( + pacticipant_id: connection[:versions].select(:pacticipant_id) + .where(Sequel[:versions][:id] => Sequel[:tags][:version_id]) + ) + end + + if columns_exist?(connection, :tags, [:version_id, :version_order]) && + columns_exist?(connection, :versions, [:id, :order]) + connection[:tags].update( + version_order: connection[:versions].select(:order) + .where(Sequel[:versions][:id] => Sequel[:tags][:version_id]) + ) + end + end + end + end + end +end diff --git a/lib/pact_broker/db/migrate_data.rb b/lib/pact_broker/db/migrate_data.rb index 5ab10d12f..906eeb3d4 100644 --- a/lib/pact_broker/db/migrate_data.rb +++ b/lib/pact_broker/db/migrate_data.rb @@ -21,6 +21,7 @@ def self.call database_connection, options = {} DataMigrations::DeleteDeprecatedWebhookExecutions.call(database_connection) DataMigrations::SetCreatedAtForLatestPactPublications.call(database_connection) DataMigrations::SetCreatedAtForLatestVerifications.call(database_connection) + DataMigrations::SetExtraColumnsForTags.call(database_connection) end end end diff --git a/lib/pact_broker/domain/pacticipant.rb b/lib/pact_broker/domain/pacticipant.rb index 1be083036..410133c7f 100644 --- a/lib/pact_broker/domain/pacticipant.rb +++ b/lib/pact_broker/domain/pacticipant.rb @@ -63,13 +63,9 @@ def any_versions? end end - - - - # Table: pacticipants # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('pacticipants_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # name | text | # repository_url | text | # created_at | timestamp without time zone | NOT NULL diff --git a/lib/pact_broker/domain/tag.rb b/lib/pact_broker/domain/tag.rb index 898d50784..a7a50775f 100644 --- a/lib/pact_broker/domain/tag.rb +++ b/lib/pact_broker/domain/tag.rb @@ -1,5 +1,6 @@ require 'pact_broker/db' require 'pact_broker/repositories/helpers' +require 'pact_broker/tags/eager_loaders' module PactBroker module Domain @@ -10,47 +11,43 @@ class Tag < Sequel::Model unrestrict_primary_key associate(:many_to_one, :version, :class => "PactBroker::Domain::Version", :key => :version_id, :primary_key => :id) + # The tag for the latest version that has a tag with a matching name + many_to_one :head_tag, read_only: true, key: [:name, :pacticipant_id], + class: Tag, + dataset: lambda { + Tag.where(name: name, pacticipant_id: pacticipant_id) + .order(Sequel.desc(:version_order)) + .limit(1) + }, + eager_loader: PactBroker::Tags::EagerLoaders::HeadTag + dataset_module do include PactBroker::Repositories::Helpers - def latest_tags - tags_versions_join = { - Sequel[:tags][:version_id] => Sequel[:versions][:id], - } + def join_pact_publications + join(:pact_publications, { Sequel[:tags][:version_id] => Sequel[:pact_publications][:consumer_version_id] } ) + end - latest_tags_versions_join = { - Sequel[:latest_tags][:name] => Sequel[:tags][:name], - Sequel[:latest_tags][:latest_order] => Sequel[:versions][:order], - Sequel[:latest_tags][:pacticipant_id] => Sequel[:versions][:pacticipant_id], - } + def for(pacticipant_name, version_number, tag_name) + where( + version_id: db[:versions].select(:id).where( + number: version_number, + pacticipant_id: db[:pacticipants].select(:id).where(name_like(:name, pacticipant_name)) + ), + name: tag_name + ).single_record + end - latest_tags = PactBroker::Domain::Tag - .select_group(Sequel[:tags][:name], Sequel[:versions][:pacticipant_id]) - .select_append{ max(order).as(latest_order) } - .join(:versions, tags_versions_join) + def latest_tags + self_join = { + Sequel[:tags][:pacticipant_id] => Sequel[:tags_2][:pacticipant_id], + Sequel[:tags][:name] => Sequel[:tags_2][:name] + } PactBroker::Domain::Tag .select_all_qualified - .join(:versions, - { Sequel[:tags][:version_id] => Sequel[:versions][:id] } - ) - .join(latest_tags, latest_tags_versions_join, { table_alias: :latest_tags }) - end - - # Ron's fancy join - # performs every so slightly better - def latest_tags_2 - tag_versions = PactBroker::Domain::Tag - .select_all_qualified - .select_append(Sequel[:versions][:pacticipant_id]) - .select_append(Sequel[:versions][:order]) - .join(:versions, - { Sequel[:tags][:version_id] => Sequel[:versions][:id] } - ) - - tag_versions - .left_join(tag_versions, { Sequel[:tags][:name] => Sequel[:tags_2][:name], Sequel[:versions][:pacticipant_id] => Sequel[:tags_2][:pacticipant_id] }, { table_alias: :tags_2 }) do | table, joined_table, js | - Sequel.qualify(table, :order) > Sequel.qualify(joined_table, :order) + .left_join(:tags, self_join, { table_alias: :tags_2 }) do + Sequel[:tags_2][:version_order] > Sequel[:tags][:version_order] end .where(Sequel[:tags_2][:name] => nil) end @@ -58,32 +55,40 @@ def latest_tags_2 # Does NOT care about whether or not there is a pact publication # for the version def latest_tags_for_pacticipant_ids(pacticipant_ids) - tags_versions_join = { - Sequel[:tags][:version_id] => Sequel[:versions][:id], - Sequel[:versions][:pacticipant_id] => pacticipant_ids + self_join = { + Sequel[:tags][:pacticipant_id] => Sequel[:tags_2][:pacticipant_id], + Sequel[:tags][:name] => Sequel[:tags_2][:name], + Sequel[:tags_2][:pacticipant_id] => pacticipant_ids, } - latest_tags_versions_join = { - Sequel[:latest_tags][:name] => Sequel[:tags][:name], - Sequel[:latest_tags][:latest_order] => Sequel[:versions][:order], - Sequel[:latest_tags][:pacticipant_id] => Sequel[:versions][:pacticipant_id], - Sequel[:versions][:pacticipant_id] => pacticipant_ids - } + Tag + .select_all_qualified + .left_join(:tags, self_join, { table_alias: :tags_2 }) do + Sequel[:tags_2][:version_order] > Sequel[:tags][:version_order] + end + .where(Sequel[:tags_2][:name] => nil) + .where(Sequel[:tags][:pacticipant_id] => pacticipant_ids) + end - latest_tags = PactBroker::Domain::Tag - .select_group(Sequel[:tags][:name], Sequel[:versions][:pacticipant_id]) - .select_append{ max(order).as(latest_order) } - .join(:versions, tags_versions_join) + def latest_tags_for_pacticipant_ids_and_tag_names(pacticipant_ids, tag_names) + self_join = { + Sequel[:tags][:pacticipant_id] => Sequel[:tags_2][:pacticipant_id], + Sequel[:tags][:name] => Sequel[:tags_2][:name], + Sequel[:tags_2][:pacticipant_id] => pacticipant_ids, + Sequel[:tags_2][:name] => tag_names + } - PactBroker::Domain::Tag + Tag .select_all_qualified - .join(:versions, - { Sequel[:tags][:version_id] => Sequel[:versions][:id], - Sequel[:versions][:pacticipant_id] => pacticipant_ids - }) - .join(latest_tags, latest_tags_versions_join, { table_alias: :latest_tags }) + .left_join(:tags, self_join, { table_alias: :tags_2 }) do + Sequel[:tags_2][:version_order] > Sequel[:tags][:version_order] + end + .where(Sequel[:tags_2][:name] => nil) + .where(Sequel[:tags][:pacticipant_id] => pacticipant_ids) + .where(Sequel[:tags][:name] => tag_names) end + # ignores tags that don't have a pact publication def head_tags_for_consumer_id(consumer_id) lp = :latest_pact_publication_ids_for_consumer_versions tags_versions_join = { @@ -98,7 +103,7 @@ def head_tags_for_consumer_id(consumer_id) # head tags for this consumer # the latest tag, pacticipant_id, version order # for versions that have a pact publication - PactBroker::Domain::Tag + Tag .select_group(Sequel[:tags][:name], Sequel[:versions][:pacticipant_id]) .select_append{ max(order).as(latest_consumer_version_order) } .join(:versions, tags_versions_join) @@ -106,25 +111,74 @@ def head_tags_for_consumer_id(consumer_id) end def head_tags_for_pact_publication(pact_publication) - head_tags_versions_join = { - Sequel[:head_tags][:latest_consumer_version_order] => Sequel[:versions][:order], - Sequel[:head_tags][:pacticipant_id] => Sequel[:versions][:pacticipant_id], - Sequel[:versions][:pacticipant_id] => pact_publication.consumer_id - } + Tag.where(version_id: pact_publication.consumer_version_id).all.select do | tag | + tag_pp_join = { + Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id], + Sequel[:pact_publications][:consumer_id] => pact_publication.consumer_id, + Sequel[:pact_publications][:provider_id] => pact_publication.provider_id, + Sequel[:tags][:name] => tag.name + } + Tag.join(:pact_publications, tag_pp_join) do + Sequel[:tags][:version_order] > tag.version_order + end + .where(pacticipant_id: pact_publication.consumer_id) + .limit(1) + .empty? + end + end + end - # Find the head tags that belong to this pact publication - # Note: The tag model has the name and version_id, - # but does not have the created_at value set - but don't need it for now - head_tags_for_consumer_id(pact_publication.consumer_id).from_self(alias: :head_tags) - .select(Sequel[:head_tags][:name], Sequel[:versions][:id].as(:version_id)) - .join(:versions, head_tags_versions_join) - .where(Sequel[:versions][:id] => pact_publication.consumer_version_id) + def before_save + if version + if version.order && self.version_order.nil? + self.version_order = version.order + end + + if self.pacticipant_id.nil? + if version.pacticipant_id + self.pacticipant_id = version.pacticipant_id + elsif version&.pacticipant&.id + self.pacticipant_id = version.pacticipant.id + end + end + end + + if version_order.nil? || pacticipant_id.nil? + raise PactBroker::Error.new("Need to set version_order and pacticipant_id for tags now") + else + super + end + end + + def latest_for_pacticipant? + head_tag == self + end + + alias_method :latest?, :latest_for_pacticipant? + + def latest_for_pact_publication?(pact_publication) + tag_pp_join = { + Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id], + Sequel[:pact_publications][:consumer_id] => pact_publication.consumer_id, + Sequel[:pact_publications][:provider_id] => pact_publication.provider_id, + Sequel[:tags][:name] => name + } + own_version_order = self.version_order + Tag.join(:pact_publications, tag_pp_join) do + Sequel[:tags][:version_order] > own_version_order end + .where(pacticipant_id: pact_publication.consumer_id) + .limit(1) + .empty? end def <=> other name <=> other.name end + + def pacticipant + version.pacticipant + end end end end @@ -132,12 +186,18 @@ def <=> other # Table: tags # Primary Key: (name, version_id) # Columns: -# name | text | -# version_id | integer | -# created_at | timestamp without time zone | NOT NULL -# updated_at | timestamp without time zone | NOT NULL +# name | text | +# version_id | integer | +# created_at | timestamp without time zone | NOT NULL +# updated_at | timestamp without time zone | NOT NULL +# pacticipant_id | integer | +# version_order | integer | # Indexes: -# tags_pk | PRIMARY KEY btree (version_id, name) -# ndx_tag_name | btree (name) +# tags_pk | PRIMARY KEY btree (version_id, name) +# ndx_tag_name | btree (name) +# tags_pacticipant_id_index | btree (pacticipant_id) +# tags_pacticipant_id_name_version_order_desc_index | btree (pacticipant_id, name, version_order DESC) +# tags_version_id_index | btree (version_id) +# tags_version_order_index | btree (version_order) # Foreign key constraints: # tags_version_id_fkey | (version_id) REFERENCES versions(id) diff --git a/lib/pact_broker/domain/verification.rb b/lib/pact_broker/domain/verification.rb index a863fc7d6..4292eb982 100644 --- a/lib/pact_broker/domain/verification.rb +++ b/lib/pact_broker/domain/verification.rb @@ -16,7 +16,7 @@ class Verification < Sequel::Model associate(:many_to_one, :provider_version, class: "PactBroker::Domain::Version", key: :provider_version_id, primary_key: :id) associate(:many_to_one, :provider, class: "PactBroker::Domain::Pacticipant", key: :provider_id, primary_key: :id) associate(:many_to_one, :consumer, class: "PactBroker::Domain::Pacticipant", key: :consumer_id, primary_key: :id) - associate(:one_to_many, :provider_version_tags, :class => "PactBroker::Tags::TagWithLatestFlag", primary_key: :provider_version_id, key: :version_id) + associate(:one_to_many, :provider_version_tags, :class => "PactBroker::Domain::Tag", primary_key: :provider_version_id, key: :version_id) plugin :serialization, :json, :test_results def before_create @@ -197,7 +197,7 @@ def method_missing(m, *args, &block) # Table: verifications # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('verifications_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # number | integer | # success | boolean | NOT NULL # provider_version | text | @@ -209,6 +209,7 @@ def method_missing(m, *args, &block) # test_results | text | # consumer_id | integer | # provider_id | integer | +# wip | boolean | NOT NULL DEFAULT false # Indexes: # verifications_pkey | PRIMARY KEY btree (id) # verifications_pact_version_id_number_index | UNIQUE btree (pact_version_id, number) diff --git a/lib/pact_broker/domain/version.rb b/lib/pact_broker/domain/version.rb index 9244a5988..35c150b21 100644 --- a/lib/pact_broker/domain/version.rb +++ b/lib/pact_broker/domain/version.rb @@ -31,34 +31,31 @@ class Version < Sequel::Model associate(:many_to_one, :pacticipant, :class => "PactBroker::Domain::Pacticipant", :key => :pacticipant_id, :primary_key => :id) one_to_many :tags, :reciprocal => :version, order: :created_at - one_to_many :tags_with_latest_flag, :class => "PactBroker::Tags::TagWithLatestFlag", primary_keys: [:id], key: [:version_id], :eager_loader => (proc do |eo_opts| - tags_for_versions = PactBroker::Domain::Tag.where(version_id: eo_opts[:key_hash][:id].keys) - latest_tag_for_pacticipants = PactBroker::Domain::Tag.latest_tags_for_pacticipant_ids(eo_opts[:rows].collect(&:pacticipant_id)).all - - eo_opts[:rows].each{|row| row.associations[:tags_with_latest_flag] = [] } - - tags_for_versions.each do | tag | - latest = latest_tag_for_pacticipants.any? { |latest_tag| latest_tag.name == tag.name && latest_tag.version_id == tag.version_id } - eo_opts[:id_map][tag.version_id].each do | version | - version.associations[:tags_with_latest_flag] << EagerTagWithLatestFlag.new(tag, latest) - end - end - end) + many_to_one :latest_version_for_pacticipant, read_only: true, key: :id, + class: Version, + dataset: lambda { Version.latest_version_for_pacticipant(pacticipant) }, + eager_loader: PactBroker::Versions::EagerLoaders::LatestVersionForPacticipant many_to_one :latest_version_for_branch, read_only: true, key: :id, class: Version, dataset: PactBroker::Versions::LazyLoaders::LATEST_VERSION_FOR_BRANCH, - eager_loader: PactBroker::Versions::EagerLoaders::LatestVersionForBranchEagerLoader + eager_loader: PactBroker::Versions::EagerLoaders::LatestVersionForBranch dataset_module do include PactBroker::Repositories::Helpers + def latest_version_for_pacticipant(pacticipant) + where(pacticipant: pacticipant) + .order(Sequel.desc(:order)) + .limit(1) + end + def for(pacticipant_name, version_number) where_pacticipant_name(pacticipant_name).where_number(version_number).single_record end - def latest_versions_for_pacticipant_branches(pacticipant, branches) - query = Version.where(pacticipant: pacticipant, Sequel[:versions][:branch] => branches) + def latest_versions_for_pacticipant_branches(pacticipant_id, branches) + query = Version.where(Sequel[:versions][:pacticipant_id] => pacticipant_id, Sequel[:versions][:branch] => branches) self_join = { Sequel[:versions][:pacticipant_id] => Sequel[:versions_2][:pacticipant_id], @@ -71,7 +68,7 @@ def latest_versions_for_pacticipant_branches(pacticipant, branches) end def where_pacticipant_name(pacticipant_name) - where(pacticipant_id: db[:pacticipants].select(:id).where(name_like(:name, pacticipant_name))) + where(Sequel[:versions][:pacticipant_id] => db[:pacticipants].select(:id).where(name_like(:name, pacticipant_name))) # If we do a join, we get the extra columns from the pacticipant table that then # make == not work # join(:pacticipants) do | p | @@ -137,7 +134,7 @@ def calculate_max_version_order_and_join_back_to_versions(query, selector) Sequel[:versions][:order] => Sequel[:latest][:latest_version_order] } - group_by_cols = selector.tag == true ? [:pacticipant_id, Sequel[:tags][:name]] : [:pacticipant_id] + group_by_cols = selector.tag == true ? [Sequel[:versions][:pacticipant_id], Sequel[:tags][:name]] : [Sequel[:versions][:pacticipant_id]] max_order_for_each_pacticipant = query .select_group(*group_by_cols) @@ -166,6 +163,10 @@ def version_and_updated_date "Version #{number} - #{updated_at.to_time.localtime.strftime("%d/%m/%Y")}" end + def head_tags + tags.select(&:latest_for_pacticipant?) + end + # What about provider??? This makes no sense def latest_pact_publication pact_publications.last @@ -174,25 +175,33 @@ def latest_pact_publication def latest_for_branch? branch ? latest_version_for_branch.order == order : nil end + + def latest_for_pacticipant? + latest_version_for_pacticipant == self + end end end end # Table: versions # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('versions_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # number | text | # repository_ref | text | # pacticipant_id | integer | NOT NULL # order | integer | # created_at | timestamp without time zone | NOT NULL # updated_at | timestamp without time zone | NOT NULL +# branch | text | +# build_url | text | # Indexes: -# versions_pkey | PRIMARY KEY btree (id) -# uq_ver_ppt_ord | UNIQUE btree (pacticipant_id, "order") -# versions_pacticipant_id_number_index | UNIQUE btree (pacticipant_id, number) -# ndx_ver_num | btree (number) -# ndx_ver_ord | btree ("order") +# versions_pkey | PRIMARY KEY btree (id) +# uq_ver_ppt_ord | UNIQUE btree (pacticipant_id, "order") +# versions_pacticipant_id_number_index | UNIQUE btree (pacticipant_id, number) +# ndx_ver_num | btree (number) +# ndx_ver_ord | btree ("order") +# versions_pacticipant_id_branch_order_index | btree (pacticipant_id, branch, "order") +# versions_pacticipant_id_order_desc_index | btree (pacticipant_id, "order" DESC) # Foreign key constraints: # versions_pacticipant_id_fkey | (pacticipant_id) REFERENCES pacticipants(id) # Referenced By: diff --git a/lib/pact_broker/index/service.rb b/lib/pact_broker/index/service.rb index 03ada3312..9883be0e4 100644 --- a/lib/pact_broker/index/service.rb +++ b/lib/pact_broker/index/service.rb @@ -13,14 +13,6 @@ class Service extend PactBroker::Services extend PactBroker::Logging - COLS = [:id, :consumer_name, :provider_name, :consumer_version_order] - LATEST_PPS = Sequel::Model.db[:latest_pact_publications].select(*COLS) - LATEST_TAGGED_PPS = Sequel::Model.db[:latest_tagged_pact_publications].select(*COLS) - HEAD_PP_ORDER_COLUMNS = [ - Sequel.asc(Sequel.function(:lower, :consumer_name)), - Sequel.desc(:consumer_version_order), - Sequel.asc(Sequel.function(:lower, :provider_name)) - ].freeze DEFAULT_PAGE_SIZE = 30 DEFAULT_PAGE_NUMBER = 1 @@ -40,27 +32,26 @@ def self.find_all_index_items def self.find_index_items options = {} latest_verifications_for_cv_tags = latest_verifications_for_consumer_version_tags(options) - latest_pact_publication_ids = latest_pact_publications.select(:id).all.collect{ |h| h[:id] } + latest_pp_ids = latest_pact_publication_ids # We only need to know if a webhook exists for an integration, not what its properties are webhooks = PactBroker::Webhooks::Webhook.select(:consumer_id, :provider_id).distinct.all - pact_publication_ids = head_pact_publication_ids(options) - pagination_record_count = pact_publication_ids.pagination_record_count + pact_publication_query = head_pact_publications(options) + pagination_record_count = pact_publication_query.pagination_record_count - pact_publications = pact_publication_scope - .where(id: pact_publication_ids) - .select_all_qualified + pact_publications = pact_publication_query .eager(:consumer) .eager(:provider) .eager(:pact_version) .eager(integration: [{latest_verification: :provider_version}, :latest_triggered_webhooks]) - .eager(consumer_version: [:latest_version_for_branch]) - .eager(latest_verification: { provider_version: [:latest_version_for_branch, :tags_with_latest_flag] }) - .eager(:head_pact_tags) + .eager(consumer_version: [:latest_version_for_branch, { tags: :head_tag }]) + .eager(latest_verification: { provider_version: [:latest_version_for_branch, { tags: :head_tag } ] }) + .eager(:head_pact_publications_for_tags) index_items = pact_publications.all.collect do | pact_publication | - is_overall_latest_for_integration = latest_pact_publication_ids.include?(pact_publication.id) + is_overall_latest_for_integration = latest_pp_ids.include?(pact_publication.id) + latest_verification = latest_verification_for_pseudo_branch(pact_publication, is_overall_latest_for_integration, latest_verifications_for_cv_tags, options[:tags]) webhook = webhooks.find{ |webhook| webhook.is_for?(pact_publication.integration) } @@ -73,7 +64,7 @@ def self.find_index_items options = {} webhook ? [webhook]: [], pact_publication.integration.latest_triggered_webhooks, consumer_version_tags(pact_publication, options[:tags]).sort_by(&:created_at).collect(&:name), - options[:tags] && latest_verification ? latest_verification.provider_version.tags_with_latest_flag.select(&:latest?).sort_by(&:created_at) : [], + options[:tags] && latest_verification ? latest_verification.provider_version.tags.select(&:latest_for_pacticipant?).sort_by(&:created_at) : [], pact_publication.latest_for_branch? ) end.sort @@ -101,30 +92,24 @@ def self.consumer_version_tags(pact_publication, tags_option) if tags_option == true pact_publication.head_pact_tags elsif tags_option.is_a?(Array) - pact_publication.head_pact_tags.select { |tag| tags_option.include?(tag.name) } + pact_publication.head_pact_tags.select{ |tag| tags_option.include?(tag.name)} else [] end end def self.find_index_items_for_api(consumer_name: nil, provider_name: nil, **ignored) - latest_pact_publication_ids = latest_pact_publications.select(:id).all.collect{ |h| h[:id] } - pact_publication_ids = head_pact_publication_ids(consumer_name: consumer_name, provider_name: provider_name, tags: true) - - pact_publications = pact_publication_scope - .where(id: pact_publication_ids) - .select_all_qualified + latest_pp_ids = latest_pact_publication_ids + pact_publications = head_pact_publications(consumer_name: consumer_name, provider_name: provider_name, tags: true) .eager(:consumer) .eager(:provider) .eager(:pact_version) - .eager(consumer_version: [:latest_version_for_branch]) - .eager(latest_verification: { provider_version: [:tags_with_latest_flag, :latest_version_for_branch]}) - .eager(:head_pact_tags) - + .eager(consumer_version: [:latest_version_for_branch, { tags: :head_tag }]) + .eager(latest_verification: { provider_version: [:latest_version_for_branch, { tags: :head_tag }]}) + .eager(:head_pact_publications_for_tags) pact_publications.all.collect do | pact_publication | - - is_overall_latest_for_integration = latest_pact_publication_ids.include?(pact_publication.id) + is_overall_latest_for_integration = latest_pp_ids.include?(pact_publication.id) PactBroker::Domain::IndexItem.create( pact_publication.consumer, @@ -135,41 +120,61 @@ def self.find_index_items_for_api(consumer_name: nil, provider_name: nil, **igno [], [], pact_publication.head_pact_tags.sort_by(&:created_at).collect(&:name), - pact_publication.latest_verification ? pact_publication.latest_verification.provider_version.tags_with_latest_flag.select(&:latest?).sort_by(&:created_at) : [] + pact_publication.latest_verification ? pact_publication.latest_verification.provider_version.tags.select(&:latest_for_pacticipant?).sort_by(&:created_at) : [] ) end.sort end def self.latest_pact_publications - db[:latest_pact_publications] + PactBroker::Pacts::PactPublication.overall_latest + end + + def self.latest_pact_publication_ids + PactBroker::Pacts::PactPublication.select(Sequel[:pact_publications][:id]).overall_latest.collect(&:id) end def self.db PactBroker::Pacts::PactPublication.db end - def self.head_pact_publication_ids(options = {}) - query = if options[:tags].is_a?(Array) - LATEST_PPS.union(LATEST_TAGGED_PPS.where(tag_name: options[:tags])) - elsif options[:tags] - LATEST_PPS.union(LATEST_TAGGED_PPS) - else - LATEST_PPS - end + def self.head_pact_publications(options = {}) + base = PactBroker::Pacts::PactPublication.select(Sequel[:pact_publications][:id]) if options[:consumer_name] - query = query.where(PactBroker::Repositories::Helpers.name_like(:consumer_name, options[:consumer_name])) + consumer = pacticipant_repository.find_by_name!(options[:consumer_name]) + base = base.for_consumer(consumer) end if options[:provider_name] - query = query.where(PactBroker::Repositories::Helpers.name_like(:provider_name, options[:provider_name])) + provider = pacticipant_repository.find_by_name!(options[:provider_name]) + base = base.for_provider(provider) end - query.order(*HEAD_PP_ORDER_COLUMNS) + latest = base.overall_latest + ids_query = if options[:tags].is_a?(Array) + latest.union(base.latest_for_consumer_tag(options[:tags])) + elsif options[:tags] + latest.union(base.latest_by_consumer_tag) + else + latest + end + + query = PactBroker::Pacts::PactPublication.select_all_qualified.where(Sequel[:pact_publications][:id] => ids_query) + .join_consumers(:consumers) + .join_providers(:providers) + .join(:versions, { Sequel[:pact_publications][:consumer_version_id] => Sequel[:cv][:id] }, { table_alias: :cv } ) + + order_columns = [ + Sequel.asc(Sequel.function(:lower, Sequel[:consumers][:name])), + Sequel.desc(Sequel[:cv][:order]), + Sequel.asc(Sequel.function(:lower, Sequel[:providers][:name])) + ] + + query.order(*order_columns) .paginate(options[:page_number] || DEFAULT_PAGE_NUMBER, options[:page_size] || DEFAULT_PAGE_SIZE) - .select(:id) end + # eager loading the tag stuff doesn't seem to make it quicker def self.latest_verifications_for_consumer_version_tags(options) # server side rendered index page with tags[]=a&tags=[]b if options[:tags].is_a?(Array) diff --git a/lib/pact_broker/metrics/service.rb b/lib/pact_broker/metrics/service.rb index 6e445fdb0..ba7a9ac81 100644 --- a/lib/pact_broker/metrics/service.rb +++ b/lib/pact_broker/metrics/service.rb @@ -58,7 +58,7 @@ def metrics tags: { count: PactBroker::Domain::Tag.count, distinctCount: PactBroker::Domain::Tag.select(:name).distinct.count, - distinctWithPacticipantCount: PactBroker::Domain::Tag.join(:versions, { id: :version_id }).select_group(:name, :pacticipant_id).count + distinctWithPacticipantCount: PactBroker::Domain::Tag.join(:versions, { id: :version_id }).select_group(:name, Sequel[:versions][:pacticipant_id]).count }, triggeredWebhooks: { count: PactBroker::Webhooks::TriggeredWebhook.count diff --git a/lib/pact_broker/pacts/eager_loaders.rb b/lib/pact_broker/pacts/eager_loaders.rb new file mode 100644 index 000000000..dca907cb9 --- /dev/null +++ b/lib/pact_broker/pacts/eager_loaders.rb @@ -0,0 +1,52 @@ +module PactBroker + module Pacts + module EagerLoaders + class HeadPactPublicationsForTags + def self.call(eo) + pact_publications = eo[:rows] + initialize_association(pact_publications) + populate_associations(group_by_consumer_and_provider_ids(pact_publications)) + end + + def self.initialize_association(pact_publications) + pact_publications.each { |pp| pp.associations[:head_pact_publications_for_tags] = [] } + end + + def self.group_by_consumer_and_provider_ids(pact_publications) + pact_publications.group_by{ |pact_publication| [pact_publication.consumer_id, pact_publication.provider_id] } + end + + def self.populate_associations(grouped_pact_publications) + grouped_pact_publications.each do | key, pact_publications | + populate_associations_for_consumer_and_provider(key, pact_publications) + end + end + + def self.populate_associations_for_consumer_and_provider(key, pact_publications) + head_pact_publications_by_tag = hash_of_head_pact_publications( + pact_publications.first.class, + pact_publications.first.consumer, + pact_publications.first.provider, + pact_publications.flat_map{ |pp| pp.consumer_version_tags.collect(&:name) } + ) + + pact_publications.each do | pact_publication | + pact_publication.consumer_version_tags.collect(&:name).sort.each do | tag_name | + pact_publication.associations[:head_pact_publications_for_tags] << head_pact_publications_by_tag[tag_name] + end + end + end + + def self.hash_of_head_pact_publications pact_publication_class, consumer, provider, tag_names + pact_publication_class + .for_consumer(consumer) + .for_provider(provider) + .latest_for_consumer_tag(tag_names) + .each_with_object({}) do | head_pact_publication, hash | + hash[head_pact_publication.values.fetch(:tag_name)] = head_pact_publication + end + end + end + end + end +end diff --git a/lib/pact_broker/pacts/latest_pact_publication_id_for_consumer_version.rb b/lib/pact_broker/pacts/latest_pact_publication_id_for_consumer_version.rb index 2593eac0f..9dc4e0ace 100644 --- a/lib/pact_broker/pacts/latest_pact_publication_id_for_consumer_version.rb +++ b/lib/pact_broker/pacts/latest_pact_publication_id_for_consumer_version.rb @@ -15,17 +15,22 @@ class LatestPactPublicationIdForConsumerVersion < Sequel::Model(:latest_pact_pub end end -# Table: latest_pact_publications_by_consumer_versions +# Table: latest_pact_publication_ids_for_consumer_versions +# Primary Key: (provider_id, consumer_version_id) # Columns: -# id | integer | -# consumer_id | integer | -# consumer_name | text | -# consumer_version_id | integer | -# consumer_version_number | text | -# consumer_version_order | integer | -# provider_id | integer | -# provider_name | text | -# revision_number | integer | -# pact_version_id | integer | -# pact_version_sha | text | -# created_at | timestamp without time zone | +# consumer_id | integer | NOT NULL +# consumer_version_id | integer | NOT NULL +# provider_id | integer | NOT NULL +# pact_publication_id | integer | NOT NULL +# pact_version_id | integer | NOT NULL +# created_at | timestamp without time zone | +# Indexes: +# latest_pact_publication_ids_for_consume_pact_publication_id_key | UNIQUE btree (pact_publication_id) +# unq_latest_ppid_prov_conver | UNIQUE btree (provider_id, consumer_version_id) +# lpp_provider_id_consumer_id_index | btree (provider_id, consumer_id) +# Foreign key constraints: +# latest_pact_publication_ids_for_consum_consumer_version_id_fkey | (consumer_version_id) REFERENCES versions(id) ON DELETE CASCADE +# latest_pact_publication_ids_for_consum_pact_publication_id_fkey | (pact_publication_id) REFERENCES pact_publications(id) ON DELETE CASCADE +# latest_pact_publication_ids_for_consumer_v_pact_version_id_fkey | (pact_version_id) REFERENCES pact_versions(id) ON DELETE CASCADE +# latest_pact_publication_ids_for_consumer_versi_consumer_id_fkey | (consumer_id) REFERENCES pacticipants(id) ON DELETE CASCADE +# latest_pact_publication_ids_for_consumer_versi_provider_id_fkey | (provider_id) REFERENCES pacticipants(id) ON DELETE CASCADE diff --git a/lib/pact_broker/pacts/lazy_loaders.rb b/lib/pact_broker/pacts/lazy_loaders.rb new file mode 100644 index 000000000..3401eab6c --- /dev/null +++ b/lib/pact_broker/pacts/lazy_loaders.rb @@ -0,0 +1,14 @@ +module PactBroker + module Pacts + module LazyLoaders + HEAD_PACT_PUBLICATIONS_FOR_TAGS = lambda { + consumer_version_tag_names = PactBroker::Domain::Tag.select(:name).where(version_id: consumer_version_id) + PactPublication + .for_consumer(consumer) + .for_provider(provider) + .latest_for_consumer_tag(consumer_version_tag_names) + .from_self.order_by(:tag_name) + } + end + end +end diff --git a/lib/pact_broker/pacts/pact_publication.rb b/lib/pact_broker/pacts/pact_publication.rb index cf24f3cb1..d4cc6c162 100644 --- a/lib/pact_broker/pacts/pact_publication.rb +++ b/lib/pact_broker/pacts/pact_publication.rb @@ -5,6 +5,8 @@ require 'pact_broker/integrations/integration' require 'pact_broker/tags/head_pact_tags' require 'pact_broker/pacts/pact_publication_dataset_module' +require 'pact_broker/pacts/eager_loaders' +require 'pact_broker/pacts/lazy_loaders' module PactBroker module Pacts @@ -21,7 +23,13 @@ class PactPublication < Sequel::Model(:pact_publications) associate(:many_to_one, :pact_version, class: "PactBroker::Pacts::PactVersion", :key => :pact_version_id, :primary_key => :id) associate(:many_to_one, :integration, class: "PactBroker::Integrations::Integration", key: [:consumer_id, :provider_id], primary_key: [:consumer_id, :provider_id]) one_to_one(:latest_verification, class: "PactBroker::Verifications::LatestVerificationForPactVersion", key: :pact_version_id, primary_key: :pact_version_id) - one_to_many(:head_pact_tags, class: "PactBroker::Tags::HeadPactTag", primary_key: :id, key: :pact_publication_id) + + one_to_many(:head_pact_publications_for_tags, + class: PactPublication, + read_only: true, + dataset: PactBroker::Pacts::LazyLoaders::HEAD_PACT_PUBLICATIONS_FOR_TAGS, + eager_loader: PactBroker::Pacts::EagerLoaders::HeadPactPublicationsForTags + ) plugin :upsert, identifying_columns: [:consumer_version_id, :provider_id, :revision_number] plugin :timestamps, update_on_create: true @@ -29,7 +37,11 @@ class PactPublication < Sequel::Model(:pact_publications) dataset_module do include PactBroker::Repositories::Helpers include PactPublicationDatasetModule + end + def self.subtract(a, b) + b_ids = b.collect(&:id) + a.reject{ |pact_publication| b_ids.include?(pact_publication.id) } end def before_create @@ -37,10 +49,16 @@ def before_create self.revision_number ||= 1 end + def head_pact_tags + consumer_version.tags.select{ |tag| head_tag_names.include?(tag.name) } + end + # The names of the tags for which this pact is the latest pact with that tag # (ie. it is not necessarily the pact for the latest consumer version with the given tag) def head_tag_names - @head_tag_names ||= PactBroker::Domain::Tag.head_tags_for_pact_publication(self).collect(&:name) + @head_tag_names ||= head_pact_publications_for_tags + .select { |head_pact_publication| head_pact_publication.id == id } + .collect { | head_pact_publication| head_pact_publication.values.fetch(:tag_name) } end def consumer_version_tags @@ -114,7 +132,7 @@ def cached_domain_for_delegation # Table: pact_publications # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('pact_publications_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # consumer_version_id | integer | NOT NULL # provider_id | integer | NOT NULL # revision_number | integer | NOT NULL diff --git a/lib/pact_broker/pacts/pact_publication_dataset_module.rb b/lib/pact_broker/pacts/pact_publication_dataset_module.rb index 5f33329f0..de5eab08c 100644 --- a/lib/pact_broker/pacts/pact_publication_dataset_module.rb +++ b/lib/pact_broker/pacts/pact_publication_dataset_module.rb @@ -25,39 +25,41 @@ def latest_by_consumer_branch Sequel[:pact_publications][:consumer_version_id] => Sequel[:cv][:id] } - base_query = select_all_qualified - .select_append(Sequel[:cv][:branch], Sequel[:cv][:order]) - .remove_overridden_revisions - .join(:versions, versions_join, { table_alias: :cv }) do - Sequel.lit("cv.branch is not null") - end + base_query = join(:versions, versions_join, { table_alias: :cv }) do + Sequel.lit("cv.branch is not null") + end self_join = { Sequel[:pact_publications][:consumer_id] => Sequel[:pp2][:consumer_id], Sequel[:cv][:branch] => Sequel[:pp2][:branch] } - base_query.left_join(base_query, self_join, { table_alias: :pp2 } ) do + if no_columns_selected? + base_query = base_query.select_all_qualified.select_append(Sequel[:cv][:branch]) + end + + base_query.left_join(base_query.select(:consumer_id, :branch, :order), self_join, { table_alias: :pp2 } ) do Sequel[:pp2][:order] > Sequel[:cv][:order] end .where(Sequel[:pp2][:order] => nil) + .remove_overridden_revisions_from_complete_query end def overall_latest - base_query = select_all_qualified - .select_append(Sequel[:cv][:order]) - .remove_overridden_revisions - .join_consumer_versions # won't need to do this when we add the order to latest_pact_publication_ids_for_consumer_versions + base_query = join_consumer_versions # won't need to do this when we add the order to latest_pact_publication_ids_for_consumer_versions self_join = { Sequel[:pact_publications][:consumer_id] => Sequel[:pp2][:consumer_id], Sequel[:pact_publications][:provider_id] => Sequel[:pp2][:provider_id] } - base_query.left_join(base_query, self_join, { table_alias: :pp2 } ) do + base_query = base_query.select_all_qualified if no_columns_selected? + + base_query.left_join(base_query.select(:consumer_id, :provider_id, :order), self_join, { table_alias: :pp2 } ) do Sequel[:pp2][:order] > Sequel[:cv][:order] end .where(Sequel[:pp2][:order] => nil) + .remove_overridden_revisions_from_complete_query end def latest_for_consumer_branch(branch) @@ -66,74 +68,85 @@ def latest_for_consumer_branch(branch) Sequel[:cv][:branch] => branch } - base_query = select_all_qualified - .select_append(Sequel[:cv][:branch], Sequel[:cv][:order]) - .remove_overridden_revisions - .join(:versions, versions_join, { table_alias: :cv }) do - Sequel.lit("cv.branch is not null") - end + base_query = join(:versions, versions_join, { table_alias: :cv }) do + Sequel.lit("cv.branch is not null") + end self_join = { Sequel[:pact_publications][:consumer_id] => Sequel[:pp2][:consumer_id], Sequel[:cv][:branch] => Sequel[:pp2][:branch] } - base_query.left_join(base_query, self_join, { table_alias: :pp2 } ) do + if no_columns_selected? + base_query = base_query.select_all_qualified.select_append(Sequel[:cv][:branch]) + end + + base_query.left_join(base_query.select(:consumer_id, :branch, :order), self_join, { table_alias: :pp2 } ) do Sequel[:pp2][:order] > Sequel[:cv][:order] end .where(Sequel[:pp2][:order] => nil) + .remove_overridden_revisions_from_complete_query end def latest_by_consumer_tag - versions_join = { - Sequel[:pact_publications][:consumer_version_id] => Sequel[:cv][:id] - } - tags_join = { - Sequel[:cv][:id] => Sequel[:tags][:version_id] + Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id] } - base_query = select_all_qualified - .select_append(Sequel[:cv][:order], Sequel[:tags][:name].as(:tag_name)) - .remove_overridden_revisions - .join(:versions, versions_join, { table_alias: :cv }) - .join(:tags, tags_join) + base_query = join(:tags, tags_join) + + if no_columns_selected? + base_query = base_query.select_all_qualified.select_append(Sequel[:tags][:name].as(:tag_name)) + end + + joined_query = base_query.select( + Sequel[:pact_publications][:consumer_id], + Sequel[:tags][:version_order], + Sequel[:tags][:name].as(:tag_name) + ) self_join = { Sequel[:pact_publications][:consumer_id] => Sequel[:pp2][:consumer_id], Sequel[:tags][:name] => Sequel[:pp2][:tag_name] } - base_query.left_join(base_query, self_join, { table_alias: :pp2 } ) do - Sequel[:pp2][:order] > Sequel[:cv][:order] + base_query.left_join(joined_query, self_join, { table_alias: :pp2 } ) do + Sequel[:pp2][:version_order] > Sequel[:tags][:version_order] end - .where(Sequel[:pp2][:order] => nil) + .where(Sequel[:pp2][:version_order] => nil) + .remove_overridden_revisions_from_complete_query end def latest_for_consumer_tag(tag_name) - versions_join = { - Sequel[:pact_publications][:consumer_version_id] => Sequel[:cv][:id] - } - tags_join = { - Sequel[:cv][:id] => Sequel[:tags][:version_id], + Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id], Sequel[:tags][:name] => tag_name } - base_query = select_all_qualified - .select_append(Sequel[:cv][:order], Sequel[:tags][:name].as(:tag_name)) - .remove_overridden_revisions - .join(:versions, versions_join, { table_alias: :cv }) + base_query = self + if no_columns_selected? + base_query = base_query.select_all_qualified.select_append(Sequel[:tags][:name].as(:tag_name)) + end + + base_query = base_query .join(:tags, tags_join) .where(Sequel[:tags][:name] => tag_name) + joined_query = base_query.select( + Sequel[:pact_publications][:consumer_id], + Sequel[:tags][:name].as(:tag_name), + Sequel[:tags][:version_order] + ) + self_join = { Sequel[:pact_publications][:consumer_id] => Sequel[:pp2][:consumer_id], Sequel[:tags][:name] => Sequel[:pp2][:tag_name] } - base_query.left_join(base_query, self_join, { table_alias: :pp2 } ) do - Sequel[:pp2][:order] > Sequel[:cv][:order] + + base_query.left_join(joined_query, self_join, { table_alias: :pp2 } ) do + Sequel[:pp2][:version_order] > Sequel[:tags][:version_order] end - .where(Sequel[:pp2][:order] => nil) + .where(Sequel[:pp2][:version_order] => nil) + .remove_overridden_revisions_from_complete_query end def successfully_verified_by_provider_branch(provider_id, provider_version_branch) @@ -179,8 +192,14 @@ def created_after date where(Sequel.lit("#{first_source_alias}.created_at > ?", date)) end - def remove_overridden_revisions - join(:latest_pact_publication_ids_for_consumer_versions, { Sequel[:lp][:pact_publication_id] => Sequel[:pact_publications][:id] }, { table_alias: :lp}) + def remove_overridden_revisions(pact_publications_alias = :pact_publications) + join(:latest_pact_publication_ids_for_consumer_versions, { Sequel[:lp][:pact_publication_id] => Sequel[pact_publications_alias][:id] }, { table_alias: :lp}) + end + + def remove_overridden_revisions_from_complete_query + from_self(alias: :p) + .select(Sequel[:p].*) + .remove_overridden_revisions(:p) end def join_consumer_versions(table_alias = :cv, extra_join_criteria = {}, &block) @@ -203,12 +222,18 @@ def join_consumer_version_tags_with_names(consumer_version_tag_names) }) end - def join_providers(table_alias = :providers) - join(:pacticipants, { Sequel[:pact_publications][:provider_id] => Sequel[table_alias][:id] }, { table_alias: table_alias }) + def join_providers(table_alias = :providers, base_table = :pact_publications, extra_join_criteria = {}) + provider_join = { + Sequel[base_table][:provider_id] => Sequel[table_alias][:id] + }.merge(extra_join_criteria) + join(:pacticipants, provider_join, { table_alias: table_alias }) end - def join_consumers(table_alias = :consumers) - join(:pacticipants, { Sequel[:pact_publications][:consumer_id] => Sequel[table_alias][:id] }, { table_alias: table_alias }) + def join_consumers(table_alias = :consumers, base_table = :pact_publications, extra_join_criteria = {}) + consumer_join = { + Sequel[base_table][:consumer_id] => Sequel[table_alias][:id] + }.merge(extra_join_criteria) + join(:pacticipants, consumer_join, { table_alias: table_alias }) end def join_pact_versions @@ -261,6 +286,12 @@ def delete PactBroker::Webhooks::TriggeredWebhook.where(pact_publication: self).delete super end + + private + + def no_columns_selected? + opts[:select].nil? + end end end end diff --git a/lib/pact_broker/pacts/pact_version.rb b/lib/pact_broker/pacts/pact_version.rb index 6fb7b8e4a..8671fa6f2 100644 --- a/lib/pact_broker/pacts/pact_version.rb +++ b/lib/pact_broker/pacts/pact_version.rb @@ -85,10 +85,9 @@ def verified_successfully_by_any_provider_version? end end - # Table: pact_versions # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('pact_versions_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # consumer_id | integer | NOT NULL # provider_id | integer | NOT NULL # sha | text | NOT NULL diff --git a/lib/pact_broker/pacts/pacts_for_verification_repository.rb b/lib/pact_broker/pacts/pacts_for_verification_repository.rb index 50e44b9ad..9372894dd 100644 --- a/lib/pact_broker/pacts/pacts_for_verification_repository.rb +++ b/lib/pact_broker/pacts/pacts_for_verification_repository.rb @@ -177,7 +177,6 @@ def provider_tag_objects_for(provider, provider_tags_names) .select_group(Sequel[:tags][:name], Sequel[:pacticipant_id]) .select_append(Sequel.function(:min, Sequel[:tags][:created_at]).as(:created_at)) .distinct - .join(:versions, { Sequel[:tags][:version_id] => Sequel[:versions][:id] } ) .where(pacticipant_id: provider.id) .where(name: provider_tags_names) .all @@ -276,11 +275,11 @@ def find_all_pact_versions_for_provider_with_consumer_version_tags provider_name end def remove_already_verified_by_branch(pact_publications, provider, provider_version_branch) - pact_publications.all - pact_publications.successfully_verified_by_provider_branch(provider.id, provider_version_branch).all + PactPublication.subtract(pact_publications.all, pact_publications.successfully_verified_by_provider_branch(provider.id, provider_version_branch).all) end def remove_already_verified_by_tag(pact_publications, query, provider, tag) - pact_publications - query.successfully_verified_by_provider_tag(provider.id, tag).all + PactPublication.subtract(pact_publications, query.successfully_verified_by_provider_tag(provider.id, tag).all) end end end diff --git a/lib/pact_broker/repositories/helpers.rb b/lib/pact_broker/repositories/helpers.rb index cadd68ff3..a19ab65be 100644 --- a/lib/pact_broker/repositories/helpers.rb +++ b/lib/pact_broker/repositories/helpers.rb @@ -43,6 +43,10 @@ def select_all_qualified select(Sequel[model.table_name].*) end + def select_append_all_qualified + select_append(Sequel[model.table_name].*) + end + def select_for_subquery column if mysql? #stoopid mysql doesn't allow you to modify datasets with subqueries column_name = column.respond_to?(:alias) ? column.alias : column diff --git a/lib/pact_broker/tags/eager_loaders.rb b/lib/pact_broker/tags/eager_loaders.rb new file mode 100644 index 000000000..5a02954cd --- /dev/null +++ b/lib/pact_broker/tags/eager_loaders.rb @@ -0,0 +1,47 @@ +module PactBroker + module Tags + module EagerLoaders + class HeadTag + def self.call(eo, **_other) + initialize_association(eo[:rows]) + populate_associations(eo[:rows]) + end + + def self.initialize_association(tags) + tags.each{|tag| tag.associations[:head_tag] = nil } + end + + def self.populate_associations(tags) + group_by_pacticipant_id(tags).each do | pacticipant_id, tags | + populate_associations_by_pacticipant(pacticipant_id, tags) + end + end + + def self.group_by_pacticipant_id(tags) + tags.to_a.group_by(&:pacticipant_id) + end + + def self.populate_associations_by_pacticipant(pacticipant_id, tags) + latest_tags_for_tags = latest_tags_for_pacticipant_id( + pacticipant_id, + tags.collect(&:name).uniq.compact, + tags.first.class + ) + self.populate_tags(tags, latest_tags_for_tags) + end + + def self.populate_tags(tags, latest_tags_for_tags) + tags.each do | tag | + tag.associations[:head_tag] = latest_tags_for_tags[[tag.pacticipant_id, tag.name]] + end + end + + def self.latest_tags_for_pacticipant_id(pacticipant_id, tag_names, tag_class) + tag_class.latest_tags_for_pacticipant_ids_and_tag_names(pacticipant_id, tag_names).each_with_object({}) do | tag, hash | + hash[[tag.pacticipant_id, tag.name]] = tag + end + end + end + end + end +end diff --git a/lib/pact_broker/tags/repository.rb b/lib/pact_broker/tags/repository.rb index 493b5e48a..1b432753d 100644 --- a/lib/pact_broker/tags/repository.rb +++ b/lib/pact_broker/tags/repository.rb @@ -10,7 +10,9 @@ class Repository def create args params = { name: args.fetch(:name), - version_id: args.fetch(:version).id + version_id: args.fetch(:version).id, + version_order: args.fetch(:version).order, + pacticipant_id: args.fetch(:version).pacticipant_id } Domain::Tag.new(params).insert_ignore end diff --git a/lib/pact_broker/tags/tag_with_latest_flag.rb b/lib/pact_broker/tags/tag_with_latest_flag.rb index e76dcef6f..018e9ae17 100644 --- a/lib/pact_broker/tags/tag_with_latest_flag.rb +++ b/lib/pact_broker/tags/tag_with_latest_flag.rb @@ -4,6 +4,7 @@ module PactBroker module Tags # The tag associated with the latest verification for a given tag + # TODO remove this class now we have eager loaders for head_tag class TagWithLatestFlag < Sequel::Model(:tags_with_latest_flag) associate(:many_to_one, :version, :class => "PactBroker::Domain::Version", :key => :version_id, :primary_key => :id) diff --git a/lib/pact_broker/test/test_data_builder.rb b/lib/pact_broker/test/test_data_builder.rb index 9343eff2e..6bd25bea7 100644 --- a/lib/pact_broker/test/test_data_builder.rb +++ b/lib/pact_broker/test/test_data_builder.rb @@ -167,7 +167,8 @@ def create_consumer_version version_number = "1.0.#{model_counter}", params = {} ) set_created_at_if_set params[:created_at], :versions, { id: @consumer_version.id } tag_names.each do | tag_name | - PactBroker::Domain::Tag.create(name: tag_name, version: @consumer_version) + tag = PactBroker::Domain::Tag.create(name: tag_name, version: consumer_version) + set_created_at_if_set(params[:created_at], :tags, { name: tag.name, version_id: consumer_version.id }) end self end @@ -178,7 +179,8 @@ def create_provider_version version_number = "1.0.#{model_counter}", params = {} @version = PactBroker::Domain::Version.create(:number => version_number, :pacticipant => @provider) @provider_version = @version tag_names.each do | tag_name | - PactBroker::Domain::Tag.create(name: tag_name, version: @provider_version) + tag = PactBroker::Domain::Tag.create(name: tag_name, version: provider_version) + set_created_at_if_set(params[:created_at], :tags, { name: tag.name, version_id: provider_version.id }) end self end @@ -195,7 +197,7 @@ def use_provider_version version_number def create_tag tag_name, params = {} params.delete(:comment) - @tag = PactBroker::Domain::Tag.create(name: tag_name, version: @version) + @tag = PactBroker::Domain::Tag.create(name: tag_name, version: @version, version_order: @version.order, pacticipant_id: @version.pacticipant_id) set_created_at_if_set params[:created_at], :tags, { name: @tag.name, version_id: @tag.version_id } self end @@ -397,6 +399,18 @@ def find_version(pacticipant_name, version_number) PactBroker::Domain::Version.for(pacticipant_name, version_number) end + def find_pact(consumer_name, consumer_version_number, provider_name) + pact_repository.find_pact(consumer_name, consumer_version_number, provider_name) + end + + def find_pact_publication(consumer_name, consumer_version_number, provider_name) + PactBroker::Pacts::PactPublication + .remove_overridden_revisions + .where(provider: find_pacticipant(provider_name)) + .where(consumer_version: find_version(consumer_name, consumer_version_number)) + .single_record + end + def model_counter @@model_counter ||= 0 @@model_counter += 1 diff --git a/lib/pact_broker/verifications/latest_verification_id_for_pact_version_and_provider_version.rb b/lib/pact_broker/verifications/latest_verification_id_for_pact_version_and_provider_version.rb index 6cd714bd4..9d08621f4 100644 --- a/lib/pact_broker/verifications/latest_verification_id_for_pact_version_and_provider_version.rb +++ b/lib/pact_broker/verifications/latest_verification_id_for_pact_version_and_provider_version.rb @@ -16,12 +16,14 @@ class LatestVerificationIdForPactVersionAndProviderVersion < Sequel::Model(:late end # Table: latest_verification_id_for_pact_version_and_provider_version +# Primary Key: (pact_version_id, provider_version_id) # Columns: -# consumer_id | integer | NOT NULL -# pact_version_id | integer | NOT NULL -# provider_id | integer | NOT NULL -# provider_version_id | integer | NOT NULL -# verification_id | integer | NOT NULL +# consumer_id | integer | NOT NULL +# pact_version_id | integer | NOT NULL +# provider_id | integer | NOT NULL +# provider_version_id | integer | NOT NULL +# verification_id | integer | NOT NULL +# created_at | timestamp without time zone | # Indexes: # latest_v_id_for_pv_and_pv_pv_id_pv_id_unq | UNIQUE btree (pact_version_id, provider_version_id) # latest_v_id_for_pv_and_pv_v_id_unq | UNIQUE btree (verification_id) diff --git a/lib/pact_broker/versions/eager_loaders.rb b/lib/pact_broker/versions/eager_loaders.rb index 772570d93..aa0857ffb 100644 --- a/lib/pact_broker/versions/eager_loaders.rb +++ b/lib/pact_broker/versions/eager_loaders.rb @@ -1,7 +1,7 @@ module PactBroker module Versions module EagerLoaders - class LatestVersionForBranchEagerLoader + class LatestVersionForBranch def self.call(eo, **other) initialize_association(eo[:rows]) populate_associations(eo[:rows]) @@ -12,19 +12,18 @@ def self.initialize_association(versions) end def self.populate_associations(versions) - group_by_pacticipant(versions).each do | pacticipant, versions | + group_by_pacticipant_id(versions).each do | pacticipant, versions | populate_associations_by_pacticipant(pacticipant, versions) end end - def self.group_by_pacticipant(versions) - versions.to_a.group_by(&:pacticipant) + def self.group_by_pacticipant_id(versions) + versions.to_a.group_by(&:pacticipant_id) end - def self.populate_associations_by_pacticipant(pacticipant, versions) - # Only has pacticipant_id, branch and order populated + def self.populate_associations_by_pacticipant(pacticipant_id, versions) latest_versions_for_branches = latest_versions_for_pacticipant_branches( - pacticipant, + pacticipant_id, versions.collect(&:branch).uniq.compact, versions.first.class ) @@ -37,12 +36,36 @@ def self.populate_versions_with_branches(versions, latest_versions_for_branches) end end - def self.latest_versions_for_pacticipant_branches(pacticipant, branches, version_class) - version_class.latest_versions_for_pacticipant_branches(pacticipant, branches).each_with_object({}) do | row, hash | + def self.latest_versions_for_pacticipant_branches(pacticipant_id, branches, version_class) + version_class.latest_versions_for_pacticipant_branches(pacticipant_id, branches).each_with_object({}) do | row, hash | hash[[row.pacticipant_id, row.branch]] = row end end end + + class LatestVersionForPacticipant + def self.call(eo, **other) + populate_associations(eo[:rows]) + end + + def self.populate_associations(versions) + group_by_pacticipant(versions).each do | pacticipant, versions | + populate_associations_by_pacticipant(pacticipant, versions) + end + end + + def self.group_by_pacticipant(versions) + versions.to_a.group_by(&:pacticipant) + end + + def self.populate_associations_by_pacticipant(pacticipant, versions) + latest_version = versions.first.class.latest_version_for_pacticipant(pacticipant).single_record + + versions.each do | version | + version.associations[:latest_version_for_pacticipant] = latest_version + end + end + end end end end diff --git a/lib/pact_broker/versions/repository.rb b/lib/pact_broker/versions/repository.rb index 789f8ce8d..174b2afb8 100644 --- a/lib/pact_broker/versions/repository.rb +++ b/lib/pact_broker/versions/repository.rb @@ -88,7 +88,7 @@ def delete_orphan_versions consumer, provider version_ids_to_keep = (version_ids_with_pact_publications + version_ids_with_verifications).uniq PactBroker::Domain::Version - .where(pacticipant_id: [consumer.id, provider.id]) + .where(Sequel[:versions][:pacticipant_id] => [consumer.id, provider.id]) .exclude(id: (version_ids_with_pact_publications + version_ids_with_verifications).uniq) .delete end diff --git a/lib/pact_broker/webhooks/execution.rb b/lib/pact_broker/webhooks/execution.rb index 01999aff4..ad0f93215 100644 --- a/lib/pact_broker/webhooks/execution.rb +++ b/lib/pact_broker/webhooks/execution.rb @@ -32,13 +32,14 @@ def <=> other # Table: webhook_executions # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('webhook_executions_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # triggered_webhook_id | integer | # success | boolean | NOT NULL # logs | text | # created_at | timestamp without time zone | NOT NULL # Indexes: -# webhook_executions_pkey | PRIMARY KEY btree (id) +# webhook_executions_pkey | PRIMARY KEY btree (id) +# webhook_executions_triggered_webhook_id_index | btree (triggered_webhook_id) # Foreign key constraints: # webhook_executions_consumer_id_fkey | (consumer_id) REFERENCES pacticipants(id) # webhook_executions_pact_publication_id_fkey | (pact_publication_id) REFERENCES pact_publications(id) diff --git a/lib/pact_broker/webhooks/latest_triggered_webhook.rb b/lib/pact_broker/webhooks/latest_triggered_webhook.rb index 00d548c53..4af989039 100644 --- a/lib/pact_broker/webhooks/latest_triggered_webhook.rb +++ b/lib/pact_broker/webhooks/latest_triggered_webhook.rb @@ -21,3 +21,5 @@ class LatestTriggeredWebhook < TriggeredWebhook # status | text | # created_at | timestamp without time zone | # updated_at | timestamp without time zone | +# verification_id | integer | +# event_name | text | diff --git a/lib/pact_broker/webhooks/triggered_webhook.rb b/lib/pact_broker/webhooks/triggered_webhook.rb index 4e10a05e8..42d1c5dd8 100644 --- a/lib/pact_broker/webhooks/triggered_webhook.rb +++ b/lib/pact_broker/webhooks/triggered_webhook.rb @@ -110,7 +110,7 @@ def number_of_attempts_remaining # Table: triggered_webhooks # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('triggered_webhooks_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # trigger_uuid | text | NOT NULL # trigger_type | text | NOT NULL # pact_publication_id | integer | NOT NULL @@ -122,10 +122,17 @@ def number_of_attempts_remaining # created_at | timestamp without time zone | NOT NULL # updated_at | timestamp without time zone | NOT NULL # verification_id | integer | +# event_name | text | +# event_context | text | # Indexes: -# triggered_webhooks_pkey | PRIMARY KEY btree (id) -# uq_triggered_webhook_ppi_wi | UNIQUE btree (pact_publication_id, webhook_id, trigger_uuid) -# uq_triggered_webhook_wi | UNIQUE btree (webhook_id, trigger_uuid) +# triggered_webhooks_pkey | PRIMARY KEY btree (id) +# uq_triggered_webhook_ppi_wi | UNIQUE btree (pact_publication_id, webhook_id, trigger_uuid) +# uq_triggered_webhook_wi | UNIQUE btree (webhook_id, trigger_uuid) +# triggered_webhooks_consumer_id_index | btree (consumer_id) +# triggered_webhooks_pact_publication_id_index | btree (pact_publication_id) +# triggered_webhooks_provider_id_index | btree (provider_id) +# triggered_webhooks_verification_id_index | btree (verification_id) +# triggered_webhooks_webhook_id_index | btree (webhook_id) # Foreign key constraints: # triggered_webhooks_consumer_id_fkey | (consumer_id) REFERENCES pacticipants(id) # triggered_webhooks_pact_publication_id_fkey | (pact_publication_id) REFERENCES pact_publications(id) diff --git a/lib/pact_broker/webhooks/webhook.rb b/lib/pact_broker/webhooks/webhook.rb index ad4973c48..5d51cbe7a 100644 --- a/lib/pact_broker/webhooks/webhook.rb +++ b/lib/pact_broker/webhooks/webhook.rb @@ -94,7 +94,7 @@ def self.properties_hash_from_domain webhook # Table: webhooks # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('webhooks_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # uuid | text | NOT NULL # method | text | NOT NULL # url | text | NOT NULL diff --git a/lib/pact_broker/webhooks/webhook_event.rb b/lib/pact_broker/webhooks/webhook_event.rb index f422f3076..607a52545 100644 --- a/lib/pact_broker/webhooks/webhook_event.rb +++ b/lib/pact_broker/webhooks/webhook_event.rb @@ -44,7 +44,7 @@ def provider_verification_failed? # Table: webhook_events # Columns: -# id | integer | PRIMARY KEY DEFAULT nextval('webhook_events_id_seq'::regclass) +# id | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY # webhook_id | integer | # name | text | # created_at | timestamp without time zone | NOT NULL diff --git a/script/pry.rb b/script/pry.rb index 0cd8237c9..23212904c 100755 --- a/script/pry.rb +++ b/script/pry.rb @@ -19,7 +19,7 @@ require 'pact_broker/db' require 'pact_broker' require 'support/test_data_builder' -puts "bout to pry" +puts "about to pry" require 'pry'; pry(binding); -puts "fater pry" +puts "after pry" diff --git a/spec/lib/pact_broker/domain/tag_spec.rb b/spec/lib/pact_broker/domain/tag_spec.rb index 039580af6..43a49b14b 100644 --- a/spec/lib/pact_broker/domain/tag_spec.rb +++ b/spec/lib/pact_broker/domain/tag_spec.rb @@ -3,22 +3,23 @@ module PactBroker module Domain describe Tag do - before do - td.create_consumer("foo") - .create_consumer_version("1") - .create_consumer_version_tag("dev") - .create_consumer_version_tag("prod") - .create_consumer_version("2") - .create_consumer_version_tag("dev") - .create_consumer_version_tag("bloop") - .create_consumer_version("3") - .create_consumer_version_tag("dev") - .create_consumer("bar") - .create_consumer_version("1") - .create_consumer_version_tag("test") - end describe "#latest_tags_for_pacticipant_ids" do + before do + td.create_consumer("foo") + .create_consumer_version("1") + .create_consumer_version_tag("dev") + .create_consumer_version_tag("prod") + .create_consumer_version("2") + .create_consumer_version_tag("dev") + .create_consumer_version_tag("bloop") + .create_consumer_version("3") + .create_consumer_version_tag("dev") + .create_consumer("bar") + .create_consumer_version("1") + .create_consumer_version_tag("test") + end + it "returns the latest tags for the given pacticipant ids" do pacticipant = PactBroker::Domain::Pacticipant.order(:id).first tags = Tag.latest_tags_for_pacticipant_ids([pacticipant.id]).all @@ -32,6 +33,21 @@ module Domain end describe "latest_tags" do + before do + td.create_consumer("foo") + .create_consumer_version("1") + .create_consumer_version_tag("dev") + .create_consumer_version_tag("prod") + .create_consumer_version("2") + .create_consumer_version_tag("dev") + .create_consumer_version_tag("bloop") + .create_consumer_version("3") + .create_consumer_version_tag("dev") + .create_consumer("bar") + .create_consumer_version("1") + .create_consumer_version_tag("test") + end + it "returns the tags that belong to the most recent version with that tag/pacticipant" do tags = Tag.latest_tags.all expect(tags.collect(&:name).sort).to eq %w{bloop dev prod test} @@ -42,19 +58,77 @@ module Domain expect(tags.collect(&:created_at).compact.size).to eq 4 end end + + describe "latest_for_pacticipant?" do + before do + # Foo v1 Bar1 + # Foo v1 Bar2 + # Foo v2 Bar1 + td.create_pact_with_verification_and_tags("Foo", "1", ["dev", "prod"], "Bar1") + .create_provider("Bar2") + .create_pact + .create_pact_with_verification_and_tags("Foo", "2", ["dev"], "Bar1") + .create_consumer_version("3") + .create_consumer_version_tag("dev") + end + + it "returns true if there are no other tags with that name for that pacticipant for a later version" do + version_1 = PactBroker::Versions::Repository.new.find_by_pacticipant_name_and_number("Foo", "1") + expect(version_1.tags.find { |t| t.name == "prod" }.latest_for_pacticipant?).to be true + expect(version_1.tags.find { |t| t.name == "dev" }.latest_for_pacticipant?).to be false + end + end + + describe "head_tags_for_pact_publication" do + before do + # Foo v1 Bar1 + # Foo v1 Bar2 + # Foo v2 Bar1 + td.create_pact_with_verification_and_tags("Foo", "1", ["dev", "prod"], "Bar1") + .create_provider("Bar2") + .create_pact + .create_pact_with_verification_and_tags("Foo", "2", ["dev"], "Bar1") + .create_consumer_version("3") + .create_consumer_version_tag("dev") + end + + it "returns the names of the tags for which this pact publication is the latest" do + pact_0 = PactBroker::Pacts::PactPublication.find(id: PactBroker::Pacts::Repository.new.find_pact("Foo", "1", "Bar1").id) + expect(pact_0.consumer_version.tags.collect(&:name).sort).to eq ["dev", "prod"] + expect(Tag.head_tags_for_pact_publication(pact_0).collect(&:name).sort).to eq ["prod"] + + pact_1 = PactBroker::Pacts::PactPublication.find(id: PactBroker::Pacts::Repository.new.find_pact("Foo", "2", "Bar1").id) + expect(Tag.head_tags_for_pact_publication(pact_1).collect(&:name).sort).to eq ["dev"] + + pact_2 = PactBroker::Pacts::PactPublication.find(id: PactBroker::Pacts::Repository.new.find_pact("Foo", "1", "Bar2").id) + expect(Tag.head_tags_for_pact_publication(pact_2).collect(&:name).sort).to eq ["dev", "prod"] + end + end + + describe "head_tag" do + before do + td.create_consumer("Foo") + .create_consumer_version("1", tag_names: ["main", "test"]) + .create_consumer_version("2", tag_names: ["main", "test"]) + .create_consumer_version("3", tag_names: ["main"]) + .create_provider("Bar") + .create_provider_version("1", tag_names: ["main", "test"]) + .create_provider_version("2", tag_names: ["main", "test"]) + .create_provider_version("3", tag_names: ["main"]) + end + + it "lazy loads" do + expect(Tag.for("Foo", "2", "main").head_tag).to eq Tag.for("Foo", "3", "main") + expect(Tag.for("Foo", "3", "main").head_tag).to eq Tag.for("Foo", "3", "main") + expect(Tag.for("Bar", "1", "test").head_tag).to eq Tag.for("Bar", "2", "test") + end + + it "eager loads" do + tags = Tag.order(:version_order, :name).eager(:head_tag).all + expect(tags[0].head_tag).to eq Tag.for("Foo", "3", "main") + expect(tags[1].head_tag).to eq Tag.for("Foo", "2", "test") + end + end end end end - -# Table: tags -# Primary Key: (name, version_id) -# Columns: -# name | text | -# version_id | integer | -# created_at | timestamp without time zone | NOT NULL -# updated_at | timestamp without time zone | NOT NULL -# Indexes: -# tags_pk | PRIMARY KEY btree (version_id, name) -# ndx_tag_name | btree (name) -# Foreign key constraints: -# tags_version_id_fkey | (version_id) REFERENCES versions(id) diff --git a/spec/lib/pact_broker/domain/version_spec.rb b/spec/lib/pact_broker/domain/version_spec.rb index b6c903c02..9ef11ca9c 100644 --- a/spec/lib/pact_broker/domain/version_spec.rb +++ b/spec/lib/pact_broker/domain/version_spec.rb @@ -142,6 +142,54 @@ def version_numbers end end + describe "latest_for_pacticipant?" do + before do + td.create_consumer("Foo") + .create_consumer_version("1") + .create_consumer_version("2") + .create_consumer("Bar") + .create_consumer_version("5") + .create_consumer_version("6") + .create_consumer_version("7") + end + + context "when the version is the latest for the pacticipant" do + it "returns true" do + expect(Version.for("Foo", "2").latest_for_pacticipant?).to be true + end + end + + context "when the version is not the latest version for the pacticipant" do + it "returns false" do + expect(Version.for("Foo", "1").latest_for_pacticipant?).to be false + end + end + end + + describe "latest_version_for_pacticipant" do + before do + td.create_consumer("Foo") + .create_consumer_version("1") + .create_consumer_version("2") + .create_consumer("Bar") + .create_consumer_version("5") + .create_consumer_version("6") + .create_consumer_version("7") + end + + subject { Version.order(:order) } + + it "lazy loads" do + expect(subject.all[0].latest_version_for_pacticipant.number).to eq "2" + end + + it "eager loads" do + all = subject.eager(:latest_version_for_pacticipant).all + expect(all[0].associations[:latest_version_for_pacticipant]).to_not be nil + expect(all[0].latest_version_for_pacticipant.number).to eq "2" + end + end + describe "latest_version_for_branch" do before do td.create_consumer("Foo") @@ -207,29 +255,6 @@ def version_numbers end end - describe "tags_with_latest_flag" do - before do - td.create_consumer("foo") - .create_consumer_version("1") - .create_consumer_version_tag("dev") - .create_consumer_version_tag("prod") - .create_consumer_version("2") - .create_consumer_version_tag("dev") - end - - it "uneager loads" do - version = Version.first(number: "1") - expect(version.tags.collect(&:name).sort).to eq %w{dev prod} - expect(version.tags_with_latest_flag.select(&:latest).collect(&:name)).to eq %w{prod} - end - - it "eager loads" do - version = Version.eager(:tags, :tags_with_latest_flag).where(number: "1").all.first - expect(version.tags.collect(&:name).sort).to eq %w{dev prod} - expect(version.tags_with_latest_flag.select(&:latest).collect(&:name)).to eq %w{prod} - end - end - describe "latest_for_branch?" do before do td.create_consumer("Foo") diff --git a/spec/lib/pact_broker/index/service_spec.rb b/spec/lib/pact_broker/index/service_spec.rb index 5ca0511c9..c9b53f0f3 100644 --- a/spec/lib/pact_broker/index/service_spec.rb +++ b/spec/lib/pact_broker/index/service_spec.rb @@ -192,6 +192,43 @@ module Index expect(rows[1].provider_name).to eq "Wiffle" end end + + context "with tags[]=" do + before do + td.create_pact_with_hierarchy("Foo", "1.0.0", "Bar") + .create_verification(provider_version: "4.5.6") + .create_consumer_version("2.0.0") + .create_consumer_version_tag("dev") + .create_pact + .revise_pact + .create_consumer_version("2.1.0") + .create_consumer_version_tag("prod") + .create_consumer_version_tag("not-prod") + .create_pact + .revise_pact + .create_verification(provider_version: "4.5.6", number: 1) + .create_verification(provider_version: "4.5.7", number: 2) + .create_verification(provider_version: "4.5.8", number: 3) + .create_verification(provider_version: "4.5.9", number: 4) + .create_provider("Wiffle") + .create_pact + end + + let(:options) { {tags: ["prod"]} } + + it "returns a row for for the latest pact and a row for the prod pact (maybe it shouldn't return the latest as well)" do + expect(rows.size).to eq 2 + + expect(rows[0].latest?).to be true + expect(rows[0].provider_name).to eq "Bar" + expect(rows[0].tag_names).to eq ["prod"] + expect(rows[0].provider_version_number).to eq "4.5.9" + + expect(rows[1].latest?).to be true + expect(rows[1].provider_name).to eq "Wiffle" + expect(rows[1].tag_names).to eq ["prod"] + end + end end context "when a pact with a tag has been verified, and then a new changed version has been published with the same tag" do @@ -263,29 +300,66 @@ module Index let(:page_size) { 2 } let(:tags) { nil } - before do - td.create_pact_with_hierarchy("Foo1", "1", "Bar1") - .create_pact_with_hierarchy("Foo2", "1", "Bar2") - .create_pact_with_hierarchy("Foo3", "1", "Bar3") - end + context "with no tags" do + before do + td.create_pact_with_hierarchy("Foo1", "1", "Bar1") + .create_pact_with_hierarchy("Foo2", "1", "Bar2") + .create_pact_with_hierarchy("Foo3", "1", "Bar3") + end - it "it returns the total number of records" do - expect(rows.pagination_record_count).to eq 3 - end + it "it returns the total number of records" do + expect(rows.pagination_record_count).to eq 3 + end - describe "the first page" do - it "contains 2 rows" do - expect(rows.count).to eq 2 + describe "the first page" do + it "contains 2 rows" do + expect(rows.count).to eq 2 + end + end + + describe "the second page" do + let(:page_number) { 2 } + + it "contains 1 row" do + expect(rows.count).to eq 1 + end end end - describe "the second page" do - let(:page_number) { 2 } + context "with tags" do + before do + td.create_pact_with_hierarchy("Foo1", "1", "Bar1") + .create_consumer_version_tag("prod") + .create_consumer_version_tag("dev") + .create_pact_with_hierarchy("Foo2", "1", "Bar2") + .create_consumer_version_tag("prod") + .create_consumer_version_tag("dev") + .create_pact_with_hierarchy("Foo3", "1", "Bar3") + .create_consumer_version_tag("prod") + .create_consumer_version_tag("dev") + end + + let(:tags) { true } - it "contains 1 row" do - expect(rows.count).to eq 1 + it "it returns the total number of records" do + expect(rows.pagination_record_count).to eq 3 + end + + describe "the first page" do + it "contains 2 rows" do + expect(rows.count).to eq 2 + end + end + + describe "the second page" do + let(:page_number) { 2 } + + it "contains 1 row" do + expect(rows.count).to eq 1 + end end end + end end diff --git a/spec/lib/pact_broker/pacts/pact_publication_dataset_module_spec.rb b/spec/lib/pact_broker/pacts/pact_publication_dataset_module_spec.rb index 498f1bd69..fecddcffe 100644 --- a/spec/lib/pact_broker/pacts/pact_publication_dataset_module_spec.rb +++ b/spec/lib/pact_broker/pacts/pact_publication_dataset_module_spec.rb @@ -44,6 +44,10 @@ module Pacts expect(subject.find { |pp| pp.consumer_id == foo_z.id && pp[:branch] == "main" }.consumer_version.number).to eq "6" end + it "does not return extra columns" do + expect(subject.first.values.keys.sort).to eq (PactPublication.columns + [:branch]).sort + end + context "chained with created after" do subject { PactPublication.created_after(DateTime.new(2020, 1, 3)).latest_by_consumer_branch.all } @@ -88,6 +92,18 @@ module Pacts expect(all.last.consumer_version.number).to eq "11" end + it "does not return extra columns" do + expect(subject.first.values.keys.sort).to eq (PactPublication.columns + [:branch]).sort + end + + context "when columns are already selected" do + subject { PactPublication.select(Sequel[:pact_publications][:id]).latest_for_consumer_branch("main") } + + it "does not override them" do + expect(subject.all.first.values.keys).to eq [:id] + end + end + context "when chained" do it "works" do all = PactPublication.for_provider(td.find_pacticipant("Bar")).latest_for_consumer_branch("main").all @@ -130,6 +146,18 @@ module Pacts expect(subject.find { |pp| pp.consumer_id == foo.id && pp[:tag_name] == "feat/x" }.consumer_version.number).to eq "4" expect(subject.find { |pp| pp.consumer_id == foo_z.id && pp[:tag_name] == "main" }.consumer_version.number).to eq "6" end + + it "does not return extra columns" do + expect(subject.first.values.keys.sort).to eq (PactPublication.columns + [:tag_name]).sort + end + + context "when columns are already selected" do + subject { PactPublication.select(Sequel[:pact_publications][:id]).latest_by_consumer_tag } + + it "does not override them" do + expect(subject.all.first.values.keys).to eq [:id] + end + end end describe "overall_latest" do @@ -157,6 +185,18 @@ module Pacts expect(all.size).to eq 2 end + it "does not return extra columns" do + expect(subject.all.first.values.keys.sort).to eq PactPublication.columns.sort + end + + context "when columns are already selected" do + subject { PactPublication.select(Sequel[:pact_publications][:id]).overall_latest } + + it "does not override them" do + expect(subject.all.first.values.keys).to eq [:id] + end + end + context "when chained" do it "works with a consumer" do expect(PactPublication.for_consumer(td.find_pacticipant("Foo")).overall_latest.all.first.consumer.name).to eq "Foo" @@ -183,6 +223,7 @@ module Pacts .create_pact .create_consumer_version("2", tag_names: ["main"]) .create_pact + .revise_pact .create_consumer_version("3", tag_names: ["feat/x"]) .create_pact .create_consumer("Foo2") @@ -207,6 +248,18 @@ module Pacts expect(all.last.consumer_version.number).to eq "11" end + it "does not return extra columns" do + expect(subject.all.first.values.keys.sort).to eq (PactPublication.columns + [:tag_name]).sort + end + + context "when columns are already selected" do + subject { PactPublication.select(Sequel[:pact_publications][:id]).latest_for_consumer_tag("main") } + + it "does not override them" do + expect(subject.all.first.values.keys).to eq [:id] + end + end + context "when chained" do it "works" do all = PactPublication.for_provider(td.find_pacticipant("Bar")).latest_for_consumer_tag("main").all @@ -234,6 +287,10 @@ module Pacts expect(subject.size).to eq 1 expect(subject.first.consumer_version.number).to eq "2" end + + it "does not return extra columns" do + expect(subject.first.values.keys.sort).to eq PactPublication.columns.sort + end end @@ -320,7 +377,7 @@ module Pacts it "with branches" do potential = PactPublication.for_provider(bar).latest_by_consumer_branch already_verified = potential.successfully_verified_by_provider_branch(bar.id, "provider-main") - not_verified = potential.all - already_verified.all + not_verified = PactPublication.subtract(potential.all, already_verified.all) expect(not_verified.size).to eq 1 expect(not_verified.first.consumer_version_number).to eq "3" @@ -329,7 +386,7 @@ module Pacts it "with tags" do potential = PactPublication.for_provider(bar).latest_by_consumer_tag already_verified = potential.successfully_verified_by_provider_branch(bar.id, "provider-main") - not_verified = potential.all - already_verified.all + not_verified = PactPublication.subtract(potential.all, already_verified.all) expect(not_verified.size).to eq 1 expect(not_verified.first.consumer_version_number).to eq "3" diff --git a/spec/lib/pact_broker/pacts/pact_publication_spec.rb b/spec/lib/pact_broker/pacts/pact_publication_spec.rb index b1de3d8af..08c7e9cd9 100644 --- a/spec/lib/pact_broker/pacts/pact_publication_spec.rb +++ b/spec/lib/pact_broker/pacts/pact_publication_spec.rb @@ -131,22 +131,22 @@ module Pacts end describe "#head_tag_names" do - before do - td.create_pact_with_hierarchy("Foo", "1.2.3", "Bar") - .create_consumer_version_tag("no") - .create_consumer_version("3.4.5") - .create_consumer_version_tag("yes") - .create_pact - .create_consumer_version("5.6.7") - .create_consumer_version_tag("no") - .create_consumer("Foo2") - .create_consumer_version("3.4.5") - .create_consumer_version_tag("yes", comment: "actually no, just here to make sure it selects the right one") - end let(:pact_publication) { PactPublication.find(id: td.pact.id) } context "when the pact is the latest for a tag" do + before do + td.create_pact_with_hierarchy("Foo", "1.2.3", "Bar") + .create_consumer_version_tag("no") + .create_consumer_version("3.4.5") + .create_consumer_version_tag("yes") + .create_pact + .create_consumer_version("5.6.7") + .create_consumer_version_tag("no") + .create_consumer("Foo2") + .create_consumer_version("3.4.5") + .create_consumer_version_tag("yes", comment: "actually no, just here to make sure it selects the right one") + end it "returns the relevant tag names" do expect(pact_publication.head_tag_names).to eq ["yes"] expect(pact_publication.head_pact_tags.collect(&:name)).to eq ["yes"] @@ -154,9 +154,46 @@ module Pacts end context "when the pact is not the latest for a tag" do + before do + td.create_pact_with_hierarchy("Foo", "1", "Bar") + .create_consumer_version_tag("prod") + .create_pact_with_hierarchy("Foo", "2", "Bar") + .create_consumer_version_tag("prod") + + end + let(:pact_publication) { PactPublication.where(consumer_version_id: PactBroker::Domain::Version.for("Foo", "1").id).single_record } + it "returns the relevant tag names" do - expect(pact_publication.head_tag_names).to eq ["yes"] - expect(pact_publication.head_pact_tags.collect(&:name)).to eq ["yes"] + expect(pact_publication.head_tag_names).to eq [] + end + end + end + + describe "#head_pact_publications_for_tags" do + before do + td.create_pact_with_hierarchy("Foo", "1", "Bar") + .create_consumer_version_tag("main") + .create_consumer_version_tag("prod") + .create_pact_with_hierarchy("Foo", "2", "Bar") + .create_consumer_version_tag("main") + .create_pact_with_hierarchy("Foo2", "6", "Bar") + .create_consumer_version_tag("main") + end + + let(:pact_publication) { td.find_pact_publication("Foo", "1", "Bar") } + + context "lazy loading" do + it "sets the head_pact_publications_for_tags" do + expect(pact_publication.head_pact_publications_for_tags.first.consumer_version.number).to eq "2" + expect(pact_publication.head_pact_publications_for_tags.last.consumer_version.number).to eq "1" + end + end + + context "eager loading" do + it "sets the head_pact_publications_for_tags" do + all = PactPublication.eager(:head_pact_publications_for_tags).order(:id).all + expect(all.first.associations[:head_pact_publications_for_tags].first.consumer_version.number).to eq "2" + expect(all.first.associations[:head_pact_publications_for_tags].last.consumer_version.number).to eq "1" end end end diff --git a/spec/lib/pact_broker/tags/repository_spec.rb b/spec/lib/pact_broker/tags/repository_spec.rb index d06be53ae..60a95a582 100644 --- a/spec/lib/pact_broker/tags/repository_spec.rb +++ b/spec/lib/pact_broker/tags/repository_spec.rb @@ -23,6 +23,8 @@ module Tags it "sets the properties" do expect(subject.name).to eq "prod" expect(subject.version.id).to eq td.version.id + expect(subject.version_order).to eq td.version.order + expect(subject.pacticipant_id).to eq td.version.pacticipant_id end context "when the tag already exists" do diff --git a/tasks/db.rake b/tasks/db.rake index e36f34636..16c9165a8 100644 --- a/tasks/db.rake +++ b/tasks/db.rake @@ -98,6 +98,7 @@ namespace :db do desc 'Annotate the Sequel domain classes with schema information' task :annotate do begin + raise "Need to set INSTALL_PG=true" unless ENV["INSTALL_PG"] == "true" ENV['RACK_ENV'] = 'test' ENV['DATABASE_ADAPTER'] = 'docker_postgres' load 'tasks/docker_database.rb'