From d866309d45dfd003109c3adc7e0fe4b6f91e000f Mon Sep 17 00:00:00 2001 From: mfo Date: Wed, 24 Jul 2024 10:23:00 +0200 Subject: [PATCH 1/5] feat(rnf/rna/siret): normalize address in champs.value_json --- app/models/champs/rnf_champ.rb | 4 + ...rna_champ_association_fetchable_concern.rb | 2 +- ...t_champ_etablissement_fetchable_concern.rb | 3 +- app/services/api_geo_service.rb | 81 +++++++++++++++++++ .../controllers/champs/rna_controller_spec.rb | 16 ++++ spec/models/champ_spec.rb | 2 +- spec/models/champs/rnf_champ_spec.rb | 19 +++++ 7 files changed, 123 insertions(+), 4 deletions(-) diff --git a/app/models/champs/rnf_champ.rb b/app/models/champs/rnf_champ.rb index 05e7dede799..e4ac7ff2a2e 100644 --- a/app/models/champs/rnf_champ.rb +++ b/app/models/champs/rnf_champ.rb @@ -13,6 +13,10 @@ def fetch_external_data RNFService.new.(rnf_id:) end + def update_with_external_data!(data:) + update!(data:, value_json: APIGeoService.parse_rnf_address(data[:address])) + end + def fetch_external_data? true end diff --git a/app/models/concerns/rna_champ_association_fetchable_concern.rb b/app/models/concerns/rna_champ_association_fetchable_concern.rb index 1a17691dc86..0788d428d49 100644 --- a/app/models/concerns/rna_champ_association_fetchable_concern.rb +++ b/app/models/concerns/rna_champ_association_fetchable_concern.rb @@ -10,7 +10,7 @@ def fetch_association!(rna) return clear_association!(:invalid) unless valid_champ_value? return clear_association!(:not_found) if (data = APIEntreprise::RNAAdapter.new(rna, procedure_id).to_params).blank? - update!(data: data) + update!(data: data, value_json: APIGeoService.parse_rna_address(data['adresse'])) rescue APIEntreprise::API::Error => error error_key = :network_error if error.try(:network_error?) && !APIEntrepriseService.api_djepva_up? clear_association!(error_key) diff --git a/app/models/concerns/siret_champ_etablissement_fetchable_concern.rb b/app/models/concerns/siret_champ_etablissement_fetchable_concern.rb index 333d58ad8d2..bfff40977f8 100644 --- a/app/models/concerns/siret_champ_etablissement_fetchable_concern.rb +++ b/app/models/concerns/siret_champ_etablissement_fetchable_concern.rb @@ -9,10 +9,9 @@ def fetch_etablissement!(siret, user) return clear_etablissement!(:invalid_checksum) if invalid_because?(siret, :checksum) # i18n-tasks-use t('errors.messages.invalid_siret_checksum') return clear_etablissement!(:not_found) unless (etablissement = APIEntrepriseService.create_etablissement(self, siret, user&.id)) # i18n-tasks-use t('errors.messages.siret_not_found') - update!(etablissement: etablissement) + update!(etablissement: etablissement, value_json: APIGeoService.parse_etablissement_address(etablissement)) rescue => error if error.try(:network_error?) && !APIEntrepriseService.api_insee_up? - # TODO: notify ops update!( etablissement: APIEntrepriseService.create_etablissement_as_degraded_mode(self, siret, user.id) ) diff --git a/app/services/api_geo_service.rb b/app/services/api_geo_service.rb index f1ed8c22881..167bae36bee 100644 --- a/app/services/api_geo_service.rb +++ b/app/services/api_geo_service.rb @@ -122,6 +122,87 @@ def parse_ban_address(feature) }.merge(territory) end + def parse_rna_address(address) + postal_code = address[:code_postal] + city_name_fallback = address[:commune] + city_code = address[:code_insee] + departement_code, region_code = if postal_code.present? && city_code.present? + commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code } + if commune.present? + [commune[:departement_code], commune[:region_code]] + else + [] + end + end + + { + street_number: address[:numero_voie], + street_name: address[:libelle_voie], + street_address: address[:libelle_voie].present? ? [address[:numero_voie], address[:type_voie], address[:libelle_voie]].compact.join(' ') : nil, + postal_code: postal_code.presence || '', + city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback), + city_code: city_code.presence || '', + departement_code:, + departement_name: departement_name(departement_code), + region_code:, + region_name: region_name(region_code) + } + end + + def parse_rnf_address(address) + postal_code = address[:postalCode] + city_name_fallback = address[:cityName] + city_code = address[:cityCode] + departement_code, region_code = if postal_code.present? && city_code.present? + commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code } + if commune.present? + [commune[:departement_code], commune[:region_code]] + else + [] + end + end + + { + street_number: address[:streetNumber], + street_name: address[:streetName], + street_address: address[:streetAddress], + postal_code: postal_code.presence || '', + city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback), + city_code: city_code.presence || '', + departement_code:, + departement_name: departement_name(departement_code), + region_code:, + region_name: region_name(region_code) + } + end + + def parse_etablissement_address(etablissement) + postal_code = etablissement.code_postal + city_name_fallback = etablissement.localite.presence || '' + city_code = etablissement.code_insee_localite + departement_code, region_code = if postal_code.present? && city_code.present? + commune = communes_by_postal_code(postal_code).find { _1[:code] == city_code } + if commune.present? + [commune[:departement_code], commune[:region_code]] + else + [] + end + end + + { + street_number: etablissement.numero_voie, + street_name: etablissement.nom_voie, + street_address: etablissement.nom_voie.present? ? [etablissement.numero_voie, etablissement.type_voie, etablissement.nom_voie].compact.join(' ') : nil, + postal_code: postal_code.presence || '', + city_name: safely_normalize_city_name(departement_code, city_code, city_name_fallback), + city_code: city_code.presence || '', + departement_code:, + departement_name: departement_name(departement_code), + region_code:, + region_name: region_name(region_code) + } + end + def safely_normalize_city_name(department_code, city_code, fallback) return fallback if department_code.blank? || city_code.blank? diff --git a/spec/controllers/champs/rna_controller_spec.rb b/spec/controllers/champs/rna_controller_spec.rb index 449c34c6807..288ba48df0d 100644 --- a/spec/controllers/champs/rna_controller_spec.rb +++ b/spec/controllers/champs/rna_controller_spec.rb @@ -117,6 +117,22 @@ expect(champ.data["association_date_publication"]).to eq("2018-01-01") expect(champ.data["association_rna"]).to eq("W751080001") end + it 'populates the value_json and RNA on the model' do + champ.reload + expect(champ.value).to eq(rna) + expect(champ.value_json).to eq({ + "city_code" => "75108", + "city_name" => "Paris", + "departement_code" => nil, # might seem broken lookup, but no, it's anonymized + "departement_name" => nil, + "postal_code" => "75009", + "region_code" => nil, + "region_name" => nil, + "street_address" => "33 rue de Modagor", + "street_name" => "de Modagor", + "street_number" => "33" + }) + end end end diff --git a/spec/models/champ_spec.rb b/spec/models/champ_spec.rb index fe457faff94..22f64c5e706 100644 --- a/spec/models/champ_spec.rb +++ b/spec/models/champ_spec.rb @@ -576,7 +576,7 @@ end context "fetch_external_data_later" do - let(:data) { 'some other data' } + let(:data) { { data: { address: { city: "some external data" } } }.with_indifferent_access } it "fill data from external source" do expect_any_instance_of(Champs::RNFChamp).to receive(:fetch_external_data) { data } diff --git a/spec/models/champs/rnf_champ_spec.rb b/spec/models/champs/rnf_champ_spec.rb index bd22eff4d41..aa15b20e1e5 100644 --- a/spec/models/champs/rnf_champ_spec.rb +++ b/spec/models/champs/rnf_champ_spec.rb @@ -96,6 +96,25 @@ expect(subject.failure.reason).to be_a(API::Client::HTTPError) } end + + describe 'update_with_external_data!' do + it 'works' do + value_json = { + :street_number => "16", + :street_name => "Rue du Général de Boissieu", + :street_address => "16 Rue du Général de Boissieu", + :postal_code => "75015", + :city_name => "Paris 15e Arrondissement", + :city_code => "75115", + :departement_code => "75", + :departement_name => "Paris", + :region_code => "11", + :region_name => "Île-de-France" + } + expect(champ).to receive(:update!).with(data: anything, value_json:) + champ.update_with_external_data!(data: subject.value!) + end + end end describe 'for_export' do From 4bf5725d6db1f6c299631e71a9df72501174b8f8 Mon Sep 17 00:00:00 2001 From: mfo Date: Mon, 19 Aug 2024 17:07:43 +0200 Subject: [PATCH 2/5] =?UTF-8?q?data(backfill):=20Champs::RnfChamp.value=5F?= =?UTF-8?q?json=20[=C2=B1400=20occurences]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_rnf_json_value_task.rb | 31 ++++++++ spec/models/champ_spec.rb | 2 +- .../populate_rnf_json_value_task_spec.rb | 71 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 app/tasks/maintenance/populate_rnf_json_value_task.rb create mode 100644 spec/tasks/maintenance/populate_rnf_json_value_task_spec.rb diff --git a/app/tasks/maintenance/populate_rnf_json_value_task.rb b/app/tasks/maintenance/populate_rnf_json_value_task.rb new file mode 100644 index 00000000000..50bf87a8183 --- /dev/null +++ b/app/tasks/maintenance/populate_rnf_json_value_task.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Maintenance + class PopulateRNFJSONValueTask < MaintenanceTasks::Task + include Dry::Monads[:result] + + def collection + Champs::RNFChamp.where(value_json: nil) + # Collection to be iterated over + # Must be Active Record Relation or Array + end + + def process(champ) + result = champ.fetch_external_data + case result + in Success(data) + begin + champ.update_with_external_data!(data:) + rescue ActiveRecord::RecordInvalid + # some champ might have dossier nil + end + else + # not found + end + end + + def count + # not really interested in counting because it raises PG Statement timeout + end + end +end diff --git a/spec/models/champ_spec.rb b/spec/models/champ_spec.rb index 22f64c5e706..4ea63a54497 100644 --- a/spec/models/champ_spec.rb +++ b/spec/models/champ_spec.rb @@ -576,7 +576,7 @@ end context "fetch_external_data_later" do - let(:data) { { data: { address: { city: "some external data" } } }.with_indifferent_access } + let(:data) { { address: { city: "some external data" } }.with_indifferent_access } it "fill data from external source" do expect_any_instance_of(Champs::RNFChamp).to receive(:fetch_external_data) { data } diff --git a/spec/tasks/maintenance/populate_rnf_json_value_task_spec.rb b/spec/tasks/maintenance/populate_rnf_json_value_task_spec.rb new file mode 100644 index 00000000000..d3e739ab3a2 --- /dev/null +++ b/spec/tasks/maintenance/populate_rnf_json_value_task_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require "rails_helper" + +module Maintenance + RSpec.describe PopulateRNFJSONValueTask do + describe "#process" do + include Dry::Monads[:result] + let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :rnf }]) } + let(:dossier) { create(:dossier, :with_populated_champs, procedure:) } + let(:element) { dossier.champs.first } + let(:data) do + { + id: 3, + rnfId: '075-FDD-00003-01', + type: 'FDD', + department: '75', + title: 'Fondation SFR', + dissolvedAt: nil, + phone: '+33185060000', + email: 'fondation@sfr.fr', + addressId: 3, + createdAt: "2023-09-07T13:26:10.358Z", + updatedAt: "2023-09-07T13:26:10.358Z", + address: { + id: 3, + createdAt: "2023-09-07T13:26:10.358Z", + updatedAt: "2023-09-07T13:26:10.358Z", + label: "16 Rue du Général de Boissieu 75015 Paris", + type: "housenumber", + streetAddress: "16 Rue du Général de Boissieu", + streetNumber: "16", + streetName: "Rue du Général de Boissieu", + postalCode: "75015", + cityName: "Paris", + cityCode: "75115", + departmentName: "Paris", + departmentCode: "75", + regionName: "Île-de-France", + regionCode: "11" + }, + status: nil, + persons: [] + } + end + + subject(:process) { described_class.process(element) } + + before do + allow_any_instance_of(Champs::RNFChamp).to receive(:fetch_external_data).and_return(Success(data)) + end + + it 'updates value_json' do + expect { subject }.to change { element.reload.value_json } + .from(nil) + .to({ + "street_number" => "16", + "street_name" => "Rue du Général de Boissieu", + "street_address" => "16 Rue du Général de Boissieu", + "postal_code" => "75015", + "city_name" => "Paris 15e Arrondissement", + "city_code" => "75115", + "departement_code" => "75", + "departement_name" => "Paris", + "region_code" => "11", + "region_name" => "Île-de-France" + }) + end + end + end +end From 917f25dcfd9569564567caafc6a53b016d015322 Mon Sep 17 00:00:00 2001 From: mfo Date: Mon, 19 Aug 2024 17:43:55 +0200 Subject: [PATCH 3/5] =?UTF-8?q?data(backfill):=20Champs::RnaChamp.value=5F?= =?UTF-8?q?json=20[=C2=B17.5k=20occurences]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_rna_json_value_task.rb | 20 ++++++++++ .../populate_rna_json_value_task_spec.rb | 39 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 app/tasks/maintenance/populate_rna_json_value_task.rb create mode 100644 spec/tasks/maintenance/populate_rna_json_value_task_spec.rb diff --git a/app/tasks/maintenance/populate_rna_json_value_task.rb b/app/tasks/maintenance/populate_rna_json_value_task.rb new file mode 100644 index 00000000000..b809f9345ec --- /dev/null +++ b/app/tasks/maintenance/populate_rna_json_value_task.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Maintenance + class PopulateRNAJSONValueTask < MaintenanceTasks::Task + def collection + Champs::RNAChamp.where.not(value: nil) + end + + def process(champ) + return if champ&.dossier&.procedure&.id.blank? + data = APIEntreprise::RNAAdapter.new(champ.value, champ&.dossier&.procedure&.id).to_params + return if data.blank? + champ.update(value_json: APIGeoService.parse_rna_address(data['adresse'])) + end + + def count + # not really interested in counting because it raises PG Statement timeout + end + end +end diff --git a/spec/tasks/maintenance/populate_rna_json_value_task_spec.rb b/spec/tasks/maintenance/populate_rna_json_value_task_spec.rb new file mode 100644 index 00000000000..9103b9a95a7 --- /dev/null +++ b/spec/tasks/maintenance/populate_rna_json_value_task_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "rails_helper" + +module Maintenance + RSpec.describe PopulateRNAJSONValueTask do + describe "#process" do + let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :rna }]) } + let(:dossier) { create(:dossier, :with_populated_champs, procedure:) } + let(:element) { dossier.champs.first } + subject(:process) { described_class.process(element) } + + let(:body) { File.read('spec/fixtures/files/api_entreprise/associations.json') } + let(:status) { 200 } + + before do + stub_request(:get, /https:\/\/entreprise.api.gouv.fr\/v4\/djepva\/api-association\/associations\/open_data\/#{element.value}/) + .to_return(body: body, status: status) + allow_any_instance_of(APIEntrepriseToken).to receive(:expired?).and_return(false) + end + it 'updates value_json' do + expect { subject }.to change { element.reload.value_json } + .from(nil) + .to({ + "street_number" => "33", + "street_name" => "de Modagor", + "street_address" => "33 rue de Modagor", + "postal_code" => "75009", + "city_name" => "Paris", + "city_code" => "75108", + "departement_code" => nil, + "departement_name" => nil, + "region_code" => nil, + "region_name" => nil + }) + end + end + end +end From a66bd46082745ae26ee0bd1255d482deb807c577 Mon Sep 17 00:00:00 2001 From: mfo Date: Tue, 20 Aug 2024 10:27:27 +0200 Subject: [PATCH 4/5] =?UTF-8?q?data(backfill):=20Champs::SiretChamp.value?= =?UTF-8?q?=5Fjson=20[=C2=B11.4M=20occurences]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_siret_value_json_task.rb | 18 +++++++++++ .../populate_siret_value_json_task_spec.rb | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 app/tasks/maintenance/populate_siret_value_json_task.rb create mode 100644 spec/tasks/maintenance/populate_siret_value_json_task_spec.rb diff --git a/app/tasks/maintenance/populate_siret_value_json_task.rb b/app/tasks/maintenance/populate_siret_value_json_task.rb new file mode 100644 index 00000000000..b8531e70500 --- /dev/null +++ b/app/tasks/maintenance/populate_siret_value_json_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Maintenance + class PopulateSiretValueJSONTask < MaintenanceTasks::Task + def collection + Champs::SiretChamp.where.not(value: nil) + end + + def process(champ) + return if champ.etablissement.blank? + champ.update!(value_json: APIGeoService.parse_etablissement_address(champ.etablissement)) + end + + def count + # not really interested in counting because it raises PG Statement timeout + end + end +end diff --git a/spec/tasks/maintenance/populate_siret_value_json_task_spec.rb b/spec/tasks/maintenance/populate_siret_value_json_task_spec.rb new file mode 100644 index 00000000000..4fca8f9ced1 --- /dev/null +++ b/spec/tasks/maintenance/populate_siret_value_json_task_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "rails_helper" + +module Maintenance + RSpec.describe PopulateSiretValueJSONTask do + describe "#process" do + let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :siret }]) } + let(:dossier) { create(:dossier, :with_populated_champs, procedure:) } + let(:element) { dossier.champs.first } + subject(:process) { described_class.process(element) } + + it 'updates value_json' do + expect { subject }.to change { element.reload.value_json } + .from(nil) + .to({ + "city_code" => "92009", + "city_name" => "Bois-Colombes", + "postal_code" => "92270", + "region_code" => "11", + "region_name" => "Île-de-France", + "street_name" => "RAOUL NORDLING", + "street_number" => "6", + "street_address" => "6 RUE RAOUL NORDLING", + "departement_code" => "92", + "departement_name" => "Hauts-de-Seine" + }) + end + end + end +end From 92ffd69bf1a85155a8f30dbceeab57dfbbca6092 Mon Sep 17 00:00:00 2001 From: mfo Date: Tue, 20 Aug 2024 11:37:21 +0200 Subject: [PATCH 5/5] doc(instance): explicite quand/comment faire tourner ces taches --- app/tasks/maintenance/populate_rna_json_value_task.rb | 4 ++++ app/tasks/maintenance/populate_rnf_json_value_task.rb | 4 ++++ app/tasks/maintenance/populate_siret_value_json_task.rb | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/app/tasks/maintenance/populate_rna_json_value_task.rb b/app/tasks/maintenance/populate_rna_json_value_task.rb index b809f9345ec..1e20a5cc933 100644 --- a/app/tasks/maintenance/populate_rna_json_value_task.rb +++ b/app/tasks/maintenance/populate_rna_json_value_task.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre : +# la normalisation des adresses des champs RNA/RNF/SIRET +# le fait de stocker ces données normalisées dans le champs.value_json (un jsonb) +# le backfill les anciens champs RNA/RNF/SIRET module Maintenance class PopulateRNAJSONValueTask < MaintenanceTasks::Task def collection diff --git a/app/tasks/maintenance/populate_rnf_json_value_task.rb b/app/tasks/maintenance/populate_rnf_json_value_task.rb index 50bf87a8183..daf9a2cd4d0 100644 --- a/app/tasks/maintenance/populate_rnf_json_value_task.rb +++ b/app/tasks/maintenance/populate_rnf_json_value_task.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre : +# la normalisation des adresses des champs RNA/RNF/SIRET +# le fait de stocker ces données normalisées dans le champs.value_json (un jsonb) +# le backfill les anciens champs RNA/RNF/SIRET module Maintenance class PopulateRNFJSONValueTask < MaintenanceTasks::Task include Dry::Monads[:result] diff --git a/app/tasks/maintenance/populate_siret_value_json_task.rb b/app/tasks/maintenance/populate_siret_value_json_task.rb index b8531e70500..e3f6eacace6 100644 --- a/app/tasks/maintenance/populate_siret_value_json_task.rb +++ b/app/tasks/maintenance/populate_siret_value_json_task.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# Dans le cadre de la story pour pouvoir rechercher un dossier en fonction des valeurs des champs branchées sur une API, voici une première pièce qui cible les champs RNA/RNF/SIRET (notamment les adresses pour de la recherche). Cette PR intègre : +# la normalisation des adresses des champs RNA/RNF/SIRET +# le fait de stocker ces données normalisées dans le champs.value_json (un jsonb) +# le backfill les anciens champs RNA/RNF/SIRET module Maintenance class PopulateSiretValueJSONTask < MaintenanceTasks::Task def collection