Vfep 1019 - fix date issues with export from GIDS dashboard, fix Fetc…

…h api for ScorecardDegreeProgram file (#1016)

* fix date processing

* Fix date issues with DateTimeConverter and fetch function for Scorecard Degree Program

---------

Co-authored-by: nfstern02 <[email protected]>

app/models/converters/date_converter.rb

@@ -9,6 +9,10 @@ def self.convert(value)
     begin
       return value if value.instance_of?(Date)
 
+      # When exporting and importing a date, it can come back as a string. Currently observed yyyy-mm-dd.
+      # Add more formats as needed.
+      return Date.parse(value) if value.instance_of?(String) && value.match?(/\d{4}-\d{2}-\d{2}/)
+
       Date.strptime(value, '%m/%d/%Y')
     rescue ArgumentError
       nil

app/models/converters/date_time_converter.rb

@@ -10,6 +10,12 @@ def self.convert(value)
 
       return value if value.instance_of?(Date)
 
+      # Added this here because you can't dynamically decide to use the DateConverter or DateTimeConverter.
+      # It is declared as part of the model's upload. There's a situation with the complaint model where the
+      # initial upload type is a date/time but when an export is performed, it gets exported as a date. The upload
+      # of the exxport is a string of the form "yyyy-mm-dd". We have to be able to handle this situation.
+      return Date.parse(value) if value.instance_of?(String) && value.match?(/\d{4}-\d{2}-\d{2}/)
+
       Date.strptime(value, '%m/%d/%Y')
     rescue ArgumentError
       nil

app/utilities/scorecard_api/service.rb

@@ -135,11 +135,24 @@ def self.api_mappings
     end
 
     def self.populate
+      # This method is used by both ScorecardApi::Service and ScorecardDegreeProgramApi::Service
+      # that we know about. There could also be other services.
+      # The pagination does NOT work for the ScorecardDegreeProgramApi::Service but the entire
+      # result set is returned  with schools_api_call(0). Since the pagination is broken for it
+      # and we have all the data we need, we can skip retrieving the results with pagination and
+      # map the results we have.
       results = []
       response_body = schools_api_call(0) #  call for page 0 to get initial @total
       results.push(*response_body[:results])
-      number_of_pages = (response_body[:metadata][:total] / MAX_PAGE_SIZE).to_f.ceil
-      (1..number_of_pages).each { |page_num| results.push(*schools_api_call(page_num)[:results]) }
+
+      # Rubocop is just wrong about this, it doesn't work if you remove the self.
+      # rubocop:disable Style/RedundantSelf
+      unless self.name.eql?('ScorecardDegreeProgramApi::Service')
+        number_of_pages = (response_body[:metadata][:total] / MAX_PAGE_SIZE).to_f.ceil
+        (1..number_of_pages).each { |page_num| results.push(*schools_api_call(page_num)[:results]) }
+      end
+      # rubocop:enable Style/RedundantSelf
+
       map_results(results)
     end
 

spec/models/converters/date_converter_spec.rb

@@ -5,10 +5,14 @@
 RSpec.describe DateConverter do
   subject { described_class }
 
-  it 'converts strings to dates' do
+  it 'converts m/d/yyyy strings to dates' do
     expect(described_class.convert('4/17/2017')).to eq(Date.parse('April 17, 2017'))
   end
 
+  it 'converts yyyy-mm-dd strings to dates' do
+    expect(described_class.convert('2017-04-17')).to eq(Date.parse('April 17, 2017'))
+  end
+
   it 'converts blank value to nil' do
     expect(described_class.convert('')).to be_nil
   end

spec/models/converters/date_time_converter_spec.rb

@@ -17,6 +17,10 @@
     expect(described_class.convert(Date.current)).to eq(Date.current)
   end
 
+  it 'converts yyyy-mm-dd strings to dates' do
+    expect(described_class.convert('2017-04-17')).to eq(Date.parse('April 17, 2017'))
+  end
+
   it 'converts blank value to nil' do
     expect(described_class.convert('')).to be_nil
   end

spec/utilities/scorecard_degree_program_api/service_spec.rb

@@ -5,50 +5,17 @@
 describe ScorecardDegreeProgramApi::Service do
   describe 'populate scorecard degree programs' do
     let(:result_1) do
-      {
-        'latest.programs.cip_4_digit': [{
-          unitid: 1,
-          school: {},
-          credential: {}
-        }]
-      }
+      { 'latest.programs.cip_4_digit': [{ unitid: 1, school: {}, credential: {} }] }
     end
     let(:result_2) do
-      {
-        'latest.programs.cip_4_digit': [{
-          unitid: 2,
-          school: {},
-          credential: {}
-        }]
-      }
+      { 'latest.programs.cip_4_digit': [{ unitid: 2, school: {}, credential: {} }] }
     end
 
     let(:response_results) { [result_1, result_2] }
     let(:client_instance) { instance_double(ScorecardApi::Client) }
 
-    context 'when total is greater than MAX_PAGE_SIZE' do
-      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE + 50 }
-      let(:body) { { results: response_results, metadata: { total: total } } }
-      let(:response) do
-        response = Faraday::Env.new
-        response[:body] = body
-        response
-      end
-
-      it 'calls ScorecardApi::Client twice' do
-        allow(ScorecardApi::Client).to receive(:new).and_return(client_instance)
-        allow(client_instance).to receive(:schools).and_return(response)
-
-        results = described_class.populate
-
-        expect(results.size).to eq(response_results.size * 2)
-        expect(results).to all(be_a(ScorecardDegreeProgram))
-      end
-    end
-
-    context 'when total is less than MAX_PAGE_SIZE' do
-      let(:total) { ScorecardApi::Service::MAX_PAGE_SIZE - 50 }
-      let(:body) { { results: response_results, metadata: { total: total } } }
+    context 'when calling the api' do
+      let(:body) { { results: response_results } }
       let(:response) do
         response = Faraday::Env.new
         response[:body] = body