feat: add duplicate columns to tags table to reduce joins

* feat: add pacticipant id and version order columns to tags

* chore: update setting of extra columns in tags

* feat: optimise query to find latest tags

* chore: more optimisations

* chore: optimise latest tagged pact consumer version orders

* chore: make tests pass

* chore: rename migrations and fix tests

* chore: remove commented old code [ci-skip]

* chore: add custom index for tag ordering

* chore: optimise index by determining latest tag status using eager loaded head_tag

* chore: update sequel model annotations

* chore: update indexes

* chore: raise error if running db:annotate without INSTALL_PG=true [ci-skip]

* chore: code climate

* chore: undo changes to pry.rb [ci-skip]

* chore: update pry.rb [ci-skip]

* docs: add docs about the regression tests

[ci-skip]

* chore: use eager loaded head_pact_publications for tags to determine the head pact tags

* chore: optimise 'latest pact publication by tag' query by removing versions join

* chore: optimise PactPublication.latest_for_consumer_tag

* refactor: rewrite queries for selecting pacts for index page

* docs: comment [ci-skip]

DEVELOPER_SETUP.md

@@ -36,7 +36,7 @@ Remember to rebuild the image if you change any of the gems or gem versions.
 
 ### With native install
 
-* You will need to install Ruby 2.5, and preferably a ruby version manager. I recommend using [chruby][chruby] and [ruby-install][ruby-install].
+* You will need to install Ruby 2.7, and preferably a ruby version manager. I recommend using [chruby][chruby] and [ruby-install][ruby-install].
 * Install bundler (the Ruby gem dependency manager) `gem install bundler`
 * Check out the pact_broker repository and cd into it.
 * Run `bundle install`. If you have any gem conflict issues, run `bundle update`.
@@ -104,5 +104,56 @@ psql postgres://postgres:postgres@postgres/postgres
   bundle exec rspec path_to_your_spec.rb
   ```
 
+## Running the regression tests
+
+The regression tests record a series of API requests/responses using a real exported database (not included in the git repository because of the size) and store the expectations in files using the Approvals gem. They allow you to make sure that the changes you have made have not made any (unexpected) changes to the interface.
+
+The tests and files are stored in the [regression](regression) directory.
+
+To run:
+
+1. Set up your local development environment as described above, making sure you have `INSTALL_PG=true` exported in your shell.
+
+1. Make sure you have the master branch checked out.
+
+1. Load an exported real database into a postgres docker image. The exported file must be in the pg dump format to use this script, and it must be located in the project root directory for it to be found via the mounted directory.
+
+    ```
+    script/docker/reload.sh <export>
+
+    ```
+1. Clear any previously generated approvals.
+
+    ```
+    regression/script/clear.sh
+    ```
+
+1. Run the tests. They will fail because there are no approval files yet.
+
+    ```
+    regression/script/run.sh
+    ```
+
+1. Approval all the things.
+
+    ```
+    regression/script/approval-all.sh
+    ```
+
+1. Run the tests again to make sure that the same results can be expected each time.
+
+    ```
+    regression/script/run.sh
+    ```
+
+1. Check out the feature branch (or enable the feature toggle)
+
+1. Run the tests again.
+    ```
+    regression/script/run.sh
+    ```
+
+1. If there is a diff, you can set `SHOW_REGRESSION_DIFF=true`, but the output is quite noisy, and you're probably better off using diff or diffmerge to view the differences.
+
 [chruby]: https://github.com/postmodern/chruby
 [ruby-install]: https://github.com/postmodern/ruby-install

db/ddl_statements/latest_tagged_pact_consumer_version_orders.rb

@@ -21,3 +21,14 @@ def latest_tagged_pact_consumer_version_orders_v3(connection)
     .join(:versions, { Sequel[:lp][:consumer_version_id] => Sequel[:cv][:id] }, { table_alias: :cv} )
     .join(:tags, { Sequel[:t][:version_id] => Sequel[:lp][:consumer_version_id] }, { table_alias: :t })
 end
+
+def latest_tagged_pact_consumer_version_orders_v4(connection)
+  view = Sequel.as(:latest_pact_publication_ids_for_consumer_versions, :lp)
+  connection.from(view)
+    .select_group(
+      Sequel[:lp][:provider_id],
+      Sequel[:lp][:consumer_id],
+      Sequel[:t][:name].as(:tag_name))
+    .select_append{ max(version_order).as(latest_consumer_version_order) }
+    .join(:tags, { Sequel[:t][:version_id] => Sequel[:lp][:consumer_version_id] }, { table_alias: :t })
+end

db/ddl_statements/latest_tagged_pact_publications.rb

@@ -0,0 +1,6 @@
+LATEST_TAGGED_PACT_PUBLICATIONS = "select lp.*, o.tag_name
+      from latest_pact_publications_by_consumer_versions lp
+      inner join latest_tagged_pact_consumer_version_orders o
+      on lp.consumer_id = o.consumer_id
+        and lp.provider_id = o.provider_id
+        and lp.consumer_version_order = latest_consumer_version_order"

db/ddl_statements/latest_verification_ids_for_consumer_version_tags.rb

@@ -51,3 +51,16 @@
         on v.provider_version_id = pv.id
       where v.id in (select latest_verification_id from latest_verification_ids_for_pact_versions)
       group by pv.pacticipant_id, lpp.consumer_id, t.name"
+
+LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V4 = "select
+    lpp.provider_id,
+    lpp.consumer_id,
+    t.name as consumer_version_tag_name,
+    max(lv.latest_verification_id) as latest_verification_id
+  from latest_verification_ids_for_pact_versions lv
+  join latest_pact_publication_ids_for_consumer_versions lpp
+    on lv.pact_version_id = lpp.pact_version_id
+  join tags t
+    on lpp.consumer_version_id = t.version_id
+  group by lpp.provider_id, lpp.consumer_id, t.name
+"

db/migrations/20210205_add_pacticipant_id_to_tag.rb

@@ -0,0 +1,17 @@
+require_relative 'migration_helper'
+
+include PactBroker::MigrationHelper
+
+Sequel.migration do
+  change do
+    alter_table(:tags) do
+      # TODO set_column_not_null(:pacticipant_id)
+      # TODO set_column_not_null(:version_order)
+      add_column(:pacticipant_id, Integer)
+      add_column(:version_order, Integer)
+      add_index(:version_order, name: "tags_version_order_index")
+      add_index(:version_id, name: "tags_version_id_index")
+      add_index(:pacticipant_id, name: "tags_pacticipant_id_index")
+    end
+  end
+end

db/migrations/20210206_add_index_to_tags_and_versions.rb

@@ -0,0 +1,27 @@
+require_relative 'migration_helper'
+
+include PactBroker::MigrationHelper
+
+Sequel.migration do
+  up do
+    if postgres?
+      run("CREATE INDEX tags_pacticipant_id_name_version_order_desc_index ON tags (pacticipant_id, name, version_order DESC);")
+      run("CREATE INDEX versions_pacticipant_id_order_desc_index ON versions (pacticipant_id, \"order\" DESC);")
+    else
+      alter_table(:tags) do
+        add_index([:pacticipant_id, :name, :version_order], name: "tags_pacticipant_id_name_version_order_index")
+      end
+    end
+  end
+
+  down do
+    if postgres?
+      run("DROP INDEX tags_pacticipant_id_name_version_order_desc_index")
+      run("DROP INDEX versions_pacticipant_id_order_desc_index")
+    else
+      alter_table(:tags) do
+        drop_index([:pacticipant_id, :name, :version_order], name: "tags_pacticipant_id_name_version_order_index")
+      end
+    end
+  end
+end

db/migrations/20210207_optimise_latest_verification_ids_for_consumer_version_tags.rb

@@ -0,0 +1,13 @@
+require_relative '../ddl_statements'
+
+Sequel.migration do
+  up do
+    create_or_replace_view(:latest_verification_ids_for_consumer_version_tags,
+      LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V4)
+  end
+
+  down do
+    create_or_replace_view(:latest_verification_ids_for_consumer_version_tags,
+      LATEST_VERIFICATION_IDS_FOR_CONSUMER_VERSION_TAGS_V3)
+  end
+end

db/migrations/20210208_optimise_latest_tagged_pact_cv_orders.rb

@@ -0,0 +1,13 @@
+require_relative '../ddl_statements'
+
+Sequel.migration do
+  up do
+    create_or_replace_view(:latest_tagged_pact_consumer_version_orders,
+      latest_tagged_pact_consumer_version_orders_v4(self))
+  end
+
+  down do
+    create_or_replace_view(:latest_tagged_pact_consumer_version_orders,
+      latest_tagged_pact_consumer_version_orders_v3(self))
+  end
+end

lib/pact_broker/certificates/certificate.rb

@@ -9,7 +9,7 @@ class Certificate < Sequel::Model
 
 # Table: certificates
 # Columns:
-#  id          | integer                     | PRIMARY KEY DEFAULT nextval('certificates_id_seq'::regclass)
+#  id          | integer                     | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
 #  uuid        | text                        | NOT NULL
 #  description | text                        |
 #  content     | text                        | NOT NULL

lib/pact_broker/config/setting.rb

@@ -73,7 +73,7 @@ def self.get_db_type(object)
 
 # Table: config
 # Columns:
-#  id         | integer                     | PRIMARY KEY DEFAULT nextval('config_id_seq'::regclass)
+#  id         | integer                     | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
 #  name       | text                        | NOT NULL
 #  type       | text                        | NOT NULL
 #  value      | text                        |

lib/pact_broker/db/data_migrations/helpers.rb

@@ -5,6 +5,10 @@ module Helpers
         def column_exists?(connection, table, column)
           connection.table_exists?(table) && connection.schema(table).find{|col| col.first == column }
         end
+
+        def columns_exist?(connection, table, columns)
+          columns.all?{ | column | column_exists?(connection, table, column) }
+        end
       end
     end
   end

lib/pact_broker/db/data_migrations/set_extra_columns_for_tags.rb

@@ -0,0 +1,29 @@
+require 'pact_broker/db/data_migrations/helpers'
+
+module PactBroker
+  module DB
+    module DataMigrations
+      class SetExtraColumnsForTags
+        extend Helpers
+
+        def self.call(connection)
+          if columns_exist?(connection, :tags, [:version_id, :pacticipant_id]) &&
+              columns_exist?(connection, :versions, [:id, :pacticipant_id])
+            connection[:tags].update(
+              pacticipant_id: connection[:versions].select(:pacticipant_id)
+                                .where(Sequel[:versions][:id] => Sequel[:tags][:version_id])
+            )
+          end
+
+          if columns_exist?(connection, :tags, [:version_id, :version_order]) &&
+              columns_exist?(connection, :versions, [:id, :order])
+            connection[:tags].update(
+              version_order: connection[:versions].select(:order)
+                                .where(Sequel[:versions][:id] => Sequel[:tags][:version_id])
+            )
+          end
+        end
+      end
+    end
+  end
+end

lib/pact_broker/db/migrate_data.rb

@@ -21,6 +21,7 @@ def self.call database_connection, options = {}
         DataMigrations::DeleteDeprecatedWebhookExecutions.call(database_connection)
         DataMigrations::SetCreatedAtForLatestPactPublications.call(database_connection)
         DataMigrations::SetCreatedAtForLatestVerifications.call(database_connection)
+        DataMigrations::SetExtraColumnsForTags.call(database_connection)
       end
     end
   end

lib/pact_broker/domain/pacticipant.rb

@@ -63,13 +63,9 @@ def any_versions?
   end
 end
 
-
-
-
-
 # Table: pacticipants
 # Columns:
-#  id             | integer                     | PRIMARY KEY DEFAULT nextval('pacticipants_id_seq'::regclass)
+#  id             | integer                     | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
 #  name           | text                        |
 #  repository_url | text                        |
 #  created_at     | timestamp without time zone | NOT NULL