Fix/previous years (#712)

https://eaflood.atlassian.net/browse/RUBY-2554
https://eaflood.atlassian.net/browse/RUBY-2364
https://eaflood.atlassian.net/browse/RUBY-2556

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-09-26 11:37:04 UTC using RuboCop version 1.56.3.
+# on 2023-10-05 14:45:57 UTC using RuboCop version 1.56.4.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -14,29 +14,13 @@ FactoryBot/AssociationStyle:
   Exclude:
     - 'spec/factories/projects.rb'
 
-# Offense count: 1
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: EnforcedStyle.
-# SupportedStyles: final_newline, final_blank_line
-Layout/TrailingEmptyLines:
-  Exclude:
-    - 'spec/steps/pafs_core/flood_protection_outcomes_details_spec.rb'
-
 # Offense count: 7
 # Configuration parameters: AllowComments, AllowEmptyLambdas.
 Lint/EmptyBlock:
   Exclude:
     - 'spec/factories/steps.rb'
     - 'spec/helpers/pafs_core/application_helper_spec.rb'
 
-# Offense count: 3
-# This cop supports safe autocorrection (--autocorrect).
-Lint/RedundantCopDisableDirective:
-  Exclude:
-    - 'spec/steps/pafs_core/coastal_erosion_protection_outcomes_detail_spec.rb'
-    - 'spec/steps/pafs_core/flood_protection_outcomes_details_spec.rb'
-    - 'spec/steps/pafs_core/funding_sources_and_spending_detail_spec.rb'
-
 # Offense count: 18
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
 Metrics/AbcSize:
@@ -52,7 +36,7 @@ Metrics/ClassLength:
 Metrics/CyclomaticComplexity:
   Max: 17
 
-# Offense count: 31
+# Offense count: 33
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 234
@@ -67,7 +51,7 @@ Metrics/ModuleLength:
 Metrics/PerceivedComplexity:
   Max: 17
 
-# Offense count: 662
+# Offense count: 661
 # Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
 # SupportedStyles: snake_case, normalcase, non_integer
 # AllowedIdentifiers: capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64
@@ -118,11 +102,11 @@ RSpec/MessageSpies:
     - 'spec/services/pafs_core/file_storage_service_spec.rb'
     - 'spec/steps/pafs_core/funding_calculator_step_spec.rb'
 
-# Offense count: 213
+# Offense count: 214
 RSpec/MultipleExpectations:
   Max: 9
 
-# Offense count: 66
+# Offense count: 68
 # Configuration parameters: AllowSubject.
 RSpec/MultipleMemoizedHelpers:
   Max: 19
@@ -223,7 +207,7 @@ Rails/InverseOf:
     - 'app/models/pafs_core/funding_value.rb'
     - 'app/models/pafs_core/user.rb'
 
-# Offense count: 20
+# Offense count: 21
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: Include.
 # Include: app/**/*.rb, config/**/*.rb, db/**/*.rb, lib/**/*.rb
@@ -233,6 +217,7 @@ Rails/Output:
     - 'lib/pafs_core/data_migration/export_to_pol.rb'
     - 'lib/pafs_core/data_migration/generate_funding_contributor_fcerm.rb'
     - 'lib/pafs_core/data_migration/move_funding_sources.rb'
+    - 'lib/pafs_core/data_migration/remove_previous_years.rb'
     - 'lib/pafs_core/data_migration/update_areas.rb'
     - 'lib/pafs_core/data_migration/update_projects.rb'
     - 'lib/pafs_core/files.rb'

Gemfile.lock

@@ -266,7 +266,7 @@ GEM
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     parallel (1.23.0)
-    parser (3.2.2.3)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
     pg (1.5.4)
@@ -346,7 +346,7 @@ GEM
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
     rspec-support (3.12.1)
-    rubocop (1.56.3)
+    rubocop (1.56.4)
       base64 (~> 0.1.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -364,7 +364,7 @@ GEM
       rubocop (~> 1.41)
     rubocop-factory_bot (2.24.0)
       rubocop (~> 1.33)
-    rubocop-rails (2.21.1)
+    rubocop-rails (2.21.2)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
@@ -409,7 +409,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.2)
-    unicode-display_width (2.4.2)
+    unicode-display_width (2.5.0)
     vcr (6.2.0)
     webmock (3.19.1)
       addressable (>= 2.8.0)

app/helpers/pafs_core/projects_helper.rb

@@ -104,11 +104,19 @@ def str_year(year)
     end
 
     def formatted_financial_year(year)
-      "#{year} to #{year + 1}"
+      if year.negative?
+        t("previous_years_label")
+      else
+        "#{year} to #{year + 1}"
+      end
     end
 
     def formatted_financial_month_and_year(year)
-      "April #{year} to March #{year + 1}"
+      if year.negative?
+        t("previous_years_label")
+      else
+        "April #{year} to March #{year + 1}"
+      end
     end
 
     def urgency_flag(project)

app/presenters/pafs_core/camc3_presenter.rb

@@ -207,7 +207,7 @@ def attributes
     attr_accessor :project, :fcerm1_presenter, :pf_calculator_presenter, :fcerm1_mapper, :funding_sources_mapper
 
     def financial_years
-      Time.zone.today.uk_financial_year..project.project_end_financial_year
+      [-1] + (2015..project.project_end_financial_year).to_a
     end
   end
 end

app/services/pafs_core/program_upload_service.rb

@@ -145,7 +145,7 @@ def build_project_from_row(row)
         if range
           # handle ranges
           start_column = column
-          years = (Time.zone.today.uk_financial_year..2027)
+          years = [-1].concat((2015..2027).to_a)
           values = []
           years.each_with_index do |_year, i|
             cell = row[start_column + i]

app/services/pafs_core/remove_previous_years_service.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module PafsCore
+  class RemovePreviousYearsService
+    attr_reader :project, :current_year
+
+    def initialize(project)
+      @project = project
+      @current_year = Time.zone.today.uk_financial_year
+    end
+
+    def run
+      return if project.state.state == "submitted"
+
+      ActiveRecord::Base.transaction do
+        prune_years(:funding_values)
+        prune_years(:flood_protection_outcomes)
+        prune_years(:flood_protection2040_outcomes)
+        prune_years(:coastal_erosion_protection_outcomes)
+      end
+    end
+
+    private
+
+    def prune_years(annual_attribute)
+      project.send(annual_attribute).where("financial_year < ?", current_year).destroy_all
+    end
+  end
+end

app/services/pafs_core/spreadsheet/contributors/coerce/financial_year.rb

@@ -6,6 +6,8 @@ module Contributors
       module Coerce
         class FinancialYear < Base
           def perform
+            return -1 if value == "Previous years"
+
             raise("unknown year") if matches.nil?
 
             matches[0]

app/services/pafs_core/spreadsheet/contributors/export.rb

@@ -34,6 +34,8 @@ def generate
         private
 
         def financial_year(value)
+          return "Previous years" if value == -1
+
           "#{value} - #{value + 1}"
         end
 

app/services/pafs_core/spreadsheet_service.rb

@@ -131,7 +131,7 @@ def add_project_to_sheet(sheet, project, row_no)
 
         if range
           start_column = column_index(col[:column])
-          years = (Time.zone.today.uk_financial_year..2027)
+          years = [-1].concat((2015..2027).to_a)
           years.each_with_index do |year, i|
             value = use_value ? project.send(name, year) : 0
             sheet[row_no][start_column + i].change_contents(value)

app/views/pafs_core/projects/steps/coastal_erosion_protection_outcomes_summary.html.erb

@@ -28,7 +28,6 @@
                 </thead>
                 <tbody>
                   <% @project.current_coastal_erosion_protection_outcomes.each do |cepo| %>
-                    <% next if cepo.financial_year == -1 %>
                     <tr>
                       <td class="financial_year">
                         <%= formatted_financial_year(cepo.financial_year) %>

app/views/pafs_core/projects/steps/funding_values.html.erb

@@ -26,7 +26,6 @@
                   <tbody>
                     <%= f.fields_for :funding_values, @project.current_funding_values do |builder| %>
                       <tr>
-                      <% next if builder.object.financial_year == -1 %>
                         <td class="financial_year">
                           <%= builder.hidden_field :financial_year %>
                           <%= builder.hidden_field :id %>

app/views/shared/_contributor_values_form.html.erb

@@ -36,7 +36,6 @@
 
                       <% contributors.each do |contributor| %>
                         <%= f.fields_for :funding_contributors, contributor do |builder| %>
-                          <% next if builder.object.funding_value.financial_year == -1 %>
                           <tr>
                             <td class="financial_year">
                               <%= formatted_financial_year(builder.object.funding_value.financial_year) %>

config/locales/en.yml

@@ -21,6 +21,7 @@ en:
   back_to_home_label: "Back to home page"
   back_to_start_label: "Back to start page"
   overview_link: "Project overview"
+  previous_years_label: "Previous years"
   complete_project_label: "Submit"
   submit_project_label: "Submit"
   unlock_project_label: "Revert to draft"

lib/pafs_core/data_migration/remove_previous_years.rb

@@ -1,35 +1,45 @@
 # frozen_string_literal: true
 
-require "csv"
-
 module PafsCore
   module DataMigration
     class RemovePreviousYears
-      attr_reader :project, :current_year
 
-      def self.perform_all
-        PafsCore::Project.find_each do |project|
-          new(project).perform
-        end
-      end
+      def self.perform(max_projects = 100)
+        processed_project_count = 0
+
+        # The funding_values cleanup for a given project will also clear out any previous
+        # years data for the other annualised attributes. We run the service for each of
+        # those attributes to handle any cases where previous years funding_values were not
+        # present but previous years data was present for another attribute.
+        %i[funding_values
+           flood_protection_outcomes
+           flood_protection2040_outcomes
+           coastal_erosion_protection_outcomes].each do |attribute|
+          projects = qualifying_projects(attribute, max_projects - processed_project_count)
+          puts "Processing #{projects.length}/#{max_projects} projects based on #{attribute}" unless Rails.env.test?
+          projects.each do |project|
+            PafsCore::RemovePreviousYearsService.new(project).run
+          end
 
-      def initialize(project)
-        @project = project
+          processed_project_count += projects.length
+          break if processed_project_count >= max_projects
+        end
       end
 
-      def perform
-        @current_year = Time.zone.today.uk_financial_year
+      def self.qualifying_projects(attribute, project_limit)
+        sanitized_attribute = ActiveRecord::Base.sanitize_sql(attribute)
 
-        ActiveRecord::Base.transaction do
-          prune_years(:funding_values)
-          prune_years(:flood_protection_outcomes)
-          prune_years(:flood_protection2040_outcomes)
-          prune_years(:coastal_erosion_protection_outcomes)
-        end
+        PafsCore::Project
+          .joins(:state)
+          .where.not(pafs_core_states: { state: :submitted })
+          .joins(attribute)
+          .distinct
+          .where(["pafs_core_#{sanitized_attribute}.financial_year < ?", current_year])
+          .limit(project_limit)
       end
 
-      def prune_years(annual_attribute)
-        @project.send(annual_attribute).where("financial_year < ?", current_year).destroy_all
+      def self.current_year
+        @current_year = Time.zone.today.uk_financial_year
       end
     end
   end

lib/tasks/pafs_core_tasks.rake

@@ -40,8 +40,13 @@ namespace :pafs do
   end
 
   desc "Remove data for financial years before the current financial year"
-  task remove_previous_years: :environment do
-    PafsCore::DataMigration::RemovePreviousYears.perform_all
+  task :remove_previous_years, [:max_projects] => :environment do |_t, args|
+    project_limit = if args[:max_projects].present?
+                      args[:max_projects].to_i
+                    else
+                      ENV.fetch("REMOVE_PREVIOUS_YEARS_MAX_PROJECTS", 100).to_i
+                    end
+    PafsCore::DataMigration::RemovePreviousYears.perform(project_limit)
   end
 end
 # rubocop:enable Metrics/BlockLength

spec/helpers/pafs_core/projects_helper_spec.rb

@@ -35,6 +35,12 @@ module PafsCore
     end
 
     describe "#formatted_financial_year" do
+      context "when given a year less than zero" do
+        it "returns the string 'Previous years'" do
+          expect(helper.formatted_financial_year(-1)).to eq "Previous years"
+        end
+      end
+
       it "returns the financial year range as a string" do
         expect(helper.formatted_financial_year(2016)).to eq "2016 to 2017"
       end