Merge pull request #10690 from mfo/US/normalize-addresses-for-rnf-rna…

…-siret

ETQ Tech, les adresses des champs siret / rna / rnf sont normalisées pour une recherche homogène via les filtres

app/models/champs/rnf_champ.rb

@@ -13,6 +13,10 @@ def fetch_external_data
     RNFService.new.(rnf_id:)
   end
 
+  def update_with_external_data!(data:)
+    update!(data:, value_json: APIGeoService.parse_rnf_address(data[:address]))
+  end
+
   def fetch_external_data?
     true
   end

app/models/concerns/rna_champ_association_fetchable_concern.rb

@@ -10,7 +10,7 @@ def fetch_association!(rna)
     return clear_association!(:invalid) unless valid_champ_value?
     return clear_association!(:not_found) if (data = APIEntreprise::RNAAdapter.new(rna, procedure_id).to_params).blank?
 
-    update!(data: data)
+    update!(data: data, value_json: APIGeoService.parse_rna_address(data['adresse']))
   rescue APIEntreprise::API::Error => error
     error_key = :network_error if error.try(:network_error?) && !APIEntrepriseService.api_djepva_up?
     clear_association!(error_key)

app/models/concerns/siret_champ_etablissement_fetchable_concern.rb

@@ -9,10 +9,9 @@ def fetch_etablissement!(siret, user)
     return clear_etablissement!(:invalid_checksum) if invalid_because?(siret, :checksum) # i18n-tasks-use t('errors.messages.invalid_siret_checksum')
     return clear_etablissement!(:not_found) unless (etablissement = APIEntrepriseService.create_etablissement(self, siret, user&.id)) # i18n-tasks-use t('errors.messages.siret_not_found')
 
-    update!(etablissement: etablissement)
+    update!(etablissement: etablissement, value_json: APIGeoService.parse_etablissement_address(etablissement))
   rescue => error
     if error.try(:network_error?) && !APIEntrepriseService.api_insee_up?
-      # TODO: notify ops
       update!(
         etablissement: APIEntrepriseService.create_etablissement_as_degraded_mode(self, siret, user.id)
       )

app/services/api_geo_service.rb

@@ -122,6 +122,87 @@ def parse_ban_address(feature)
       }.merge(territory)
     end
 
+    def parse_rna_address(address)
+      postal_code = address[:code_postal]
+      city_name_fallback = address[:commune]
+      city_code = address[:code_insee]
+      departement_code, region_code = if postal_code.present? && city_code.present?
+        commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code }
+        if commune.present?
+          [commune[:departement_code], commune[:region_code]]
+        else
+          []
+        end
+      end
+
+      {
+        street_number: address[:numero_voie],
+        street_name: address[:libelle_voie],
+        street_address: address[:libelle_voie].present? ? [address[:numero_voie], address[:type_voie], address[:libelle_voie]].compact.join(' ') : nil,
+        postal_code: postal_code.presence || '',
+        city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback),
+        city_code: city_code.presence || '',
+        departement_code:,
+        departement_name: departement_name(departement_code),
+        region_code:,
+        region_name: region_name(region_code)
+      }
+    end
+
+    def parse_rnf_address(address)
+      postal_code = address[:postalCode]
+      city_name_fallback = address[:cityName]
+      city_code = address[:cityCode]
+      departement_code, region_code = if postal_code.present? && city_code.present?
+        commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code }
+        if commune.present?
+          [commune[:departement_code], commune[:region_code]]
+        else
+          []
+        end
+      end
+
+      {
+        street_number: address[:streetNumber],
+        street_name: address[:streetName],
+        street_address: address[:streetAddress],
+        postal_code: postal_code.presence || '',
+        city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback),
+        city_code: city_code.presence || '',
+        departement_code:,
+        departement_name: departement_name(departement_code),
+        region_code:,
+        region_name: region_name(region_code)
+      }
+    end
+
+    def parse_etablissement_address(etablissement)
+      postal_code = etablissement.code_postal
+      city_name_fallback = etablissement.localite.presence || ''
+      city_code = etablissement.code_insee_localite
+      departement_code, region_code = if postal_code.present? && city_code.present?
+        commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code }
+        if commune.present?
+          [commune[:departement_code], commune[:region_code]]
+        else
+          []
+        end
+      end
+
+      {
+        street_number: etablissement.numero_voie,
+        street_name: etablissement.nom_voie,
+        street_address: etablissement.nom_voie.present? ? [etablissement.numero_voie, etablissement.type_voie, etablissement.nom_voie].compact.join(' ') : nil,
+        postal_code: postal_code.presence || '',
+        city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback),
+        city_code: city_code.presence || '',
+        departement_code:,
+        departement_name: departement_name(departement_code),
+        region_code:,
+        region_name: region_name(region_code)
+      }
+    end
+
     def safely_normalize_city_name(department_code, city_code, fallback)
       return fallback if department_code.blank? || city_code.blank?
 

app/tasks/maintenance/populate_rna_json_value_task.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre :
+#     la normalisation des adresses des champs RNA/RNF/SIRET
+#     le fait de stocker ces données normalisées dans le champs.value_json (un jsonb)
+#     le backfill les anciens champs RNA/RNF/SIRET
+module Maintenance
+  class PopulateRNAJSONValueTask < MaintenanceTasks::Task
+    def collection
+      Champs::RNAChamp.where.not(value: nil)
+    end
+
+    def process(champ)
+      return if champ&.dossier&.procedure&.id.blank?
+      data = APIEntreprise::RNAAdapter.new(champ.value, champ&.dossier&.procedure&.id).to_params
+      return if data.blank?
+      champ.update(value_json: APIGeoService.parse_rna_address(data['adresse']))
+    end
+
+    def count
+      # not really interested in counting because it raises PG Statement timeout
+    end
+  end
+end

app/tasks/maintenance/populate_rnf_json_value_task.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre :
+#     la normalisation des adresses des champs RNA/RNF/SIRET
+#     le fait de stocker ces données normalisées dans le champs.value_json (un jsonb)
+#     le backfill les anciens champs RNA/RNF/SIRET
+module Maintenance
+  class PopulateRNFJSONValueTask < MaintenanceTasks::Task
+    include Dry::Monads[:result]
+
+    def collection
+      Champs::RNFChamp.where(value_json: nil)
+      # Collection to be iterated over
+      # Must be Active Record Relation or Array
+    end
+
+    def process(champ)
+      result = champ.fetch_external_data
+      case result
+      in Success(data)
+        begin
+          champ.update_with_external_data!(data:)
+        rescue ActiveRecord::RecordInvalid
+          # some champ might have dossier nil
+        end
+      else
+        # not found
+      end
+    end
+
+    def count
+      # not really interested in counting because it raises PG Statement timeout
+    end
+  end
+end

app/tasks/maintenance/populate_siret_value_json_task.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre :
+#     la normalisation des adresses des champs RNA/RNF/SIRET
+#     le fait de stocker ces données normalisées dans le champs.value_json (un jsonb)
+#     le backfill les anciens champs RNA/RNF/SIRET
+module Maintenance
+  class PopulateSiretValueJSONTask < MaintenanceTasks::Task
+    def collection
+      Champs::SiretChamp.where.not(value: nil)
+    end
+
+    def process(champ)
+      return if champ.etablissement.blank?
+      champ.update!(value_json: APIGeoService.parse_etablissement_address(champ.etablissement))
+    end
+
+    def count
+      # not really interested in counting because it raises PG Statement timeout
+    end
+  end
+end

spec/controllers/champs/rna_controller_spec.rb

@@ -117,6 +117,22 @@
           expect(champ.data["association_date_publication"]).to eq("2018-01-01")
           expect(champ.data["association_rna"]).to eq("W751080001")
         end
+        it 'populates the value_json and RNA on the model' do
+          champ.reload
+          expect(champ.value).to eq(rna)
+          expect(champ.value_json).to eq({
+            "city_code" => "75108",
+            "city_name" => "Paris",
+            "departement_code" => nil, # might seem broken lookup, but no, it's anonymized
+            "departement_name" => nil,
+            "postal_code" => "75009",
+            "region_code" => nil,
+            "region_name" => nil,
+            "street_address" => "33 rue de Modagor",
+            "street_name" => "de Modagor",
+            "street_number" => "33"
+          })
+        end
       end
     end
 

spec/models/champ_spec.rb

@@ -576,7 +576,7 @@
     end
 
     context "fetch_external_data_later" do
-      let(:data) { 'some other data' }
+      let(:data) { { address: { city: "some external data" } }.with_indifferent_access }
 
       it "fill data from external source" do
         expect_any_instance_of(Champs::RNFChamp).to receive(:fetch_external_data) { data }

spec/models/champs/rnf_champ_spec.rb

@@ -96,6 +96,25 @@
         expect(subject.failure.reason).to be_a(API::Client::HTTPError)
       }
     end
+
+    describe 'update_with_external_data!' do
+      it 'works' do
+        value_json = {
+          :street_number => "16",
+          :street_name => "Rue du Général de Boissieu",
+          :street_address => "16 Rue du Général de Boissieu",
+          :postal_code => "75015",
+          :city_name => "Paris 15e Arrondissement",
+          :city_code => "75115",
+          :departement_code => "75",
+          :departement_name => "Paris",
+          :region_code => "11",
+          :region_name => "Île-de-France"
+        }
+        expect(champ).to receive(:update!).with(data: anything, value_json:)
+        champ.update_with_external_data!(data: subject.value!)
+      end
+    end
   end
 
   describe 'for_export' do

spec/tasks/maintenance/populate_rna_json_value_task_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+module Maintenance
+  RSpec.describe PopulateRNAJSONValueTask do
+    describe "#process" do
+      let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :rna }]) }
+      let(:dossier) { create(:dossier, :with_populated_champs, procedure:) }
+      let(:element) { dossier.champs.first }
+      subject(:process) { described_class.process(element) }
+
+      let(:body) { File.read('spec/fixtures/files/api_entreprise/associations.json') }
+      let(:status) { 200 }
+
+      before do
+        stub_request(:get, /https:\/\/entreprise.api.gouv.fr\/v4\/djepva\/api-association\/associations\/open_data\/#{element.value}/)
+          .to_return(body: body, status: status)
+        allow_any_instance_of(APIEntrepriseToken).to receive(:expired?).and_return(false)
+      end
+      it 'updates value_json' do
+        expect { subject }.to change { element.reload.value_json }
+          .from(nil)
+          .to({
+            "street_number" => "33",
+            "street_name" => "de Modagor",
+            "street_address" => "33 rue de Modagor",
+            "postal_code" => "75009",
+            "city_name" => "Paris",
+            "city_code" => "75108",
+            "departement_code" => nil,
+            "departement_name" => nil,
+            "region_code" => nil,
+            "region_name" => nil
+          })
+      end
+    end
+  end
+end

spec/tasks/maintenance/populate_rnf_json_value_task_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+module Maintenance
+  RSpec.describe PopulateRNFJSONValueTask do
+    describe "#process" do
+      include Dry::Monads[:result]
+      let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :rnf }]) }
+      let(:dossier) { create(:dossier, :with_populated_champs, procedure:) }
+      let(:element) { dossier.champs.first }
+      let(:data) do
+        {
+          id: 3,
+          rnfId: '075-FDD-00003-01',
+          type: 'FDD',
+          department: '75',
+          title: 'Fondation SFR',
+          dissolvedAt: nil,
+          phone: '+33185060000',
+          email: '[email protected]',
+          addressId: 3,
+          createdAt: "2023-09-07T13:26:10.358Z",
+          updatedAt: "2023-09-07T13:26:10.358Z",
+          address: {
+            id: 3,
+            createdAt: "2023-09-07T13:26:10.358Z",
+            updatedAt: "2023-09-07T13:26:10.358Z",
+            label: "16 Rue du Général de Boissieu 75015 Paris",
+            type: "housenumber",
+            streetAddress: "16 Rue du Général de Boissieu",
+            streetNumber: "16",
+            streetName: "Rue du Général de Boissieu",
+            postalCode: "75015",
+            cityName: "Paris",
+            cityCode: "75115",
+            departmentName: "Paris",
+            departmentCode: "75",
+            regionName: "Île-de-France",
+            regionCode: "11"
+          },
+          status: nil,
+          persons: []
+        }
+      end
+
+      subject(:process) { described_class.process(element) }
+
+      before do
+        allow_any_instance_of(Champs::RNFChamp).to receive(:fetch_external_data).and_return(Success(data))
+      end
+
+      it 'updates value_json' do
+        expect { subject }.to change { element.reload.value_json }
+          .from(nil)
+          .to({
+            "street_number" => "16",
+            "street_name" => "Rue du Général de Boissieu",
+            "street_address" => "16 Rue du Général de Boissieu",
+            "postal_code" => "75015",
+            "city_name" => "Paris 15e Arrondissement",
+            "city_code" => "75115",
+            "departement_code" => "75",
+            "departement_name" => "Paris",
+            "region_code" => "11",
+            "region_name" => "Île-de-France"
+          })
+      end
+    end
+  end
+end