From c82d0dd472d8e9f948106d6c138fdddf849b33d5 Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Wed, 18 Sep 2024 17:00:12 +0200 Subject: [PATCH 1/2] chore(pipeline) : Switch to the new endpoints for France Travail All the pole-emploi.xxx endpoints will be shut down on September 30th. --- .../dags/dag_utils/sources/france_travail.py | 2 +- .../stg_france_travail__agences.sql | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pipeline/dags/dag_utils/sources/france_travail.py b/pipeline/dags/dag_utils/sources/france_travail.py index 6b853384e..d50ed64eb 100644 --- a/pipeline/dags/dag_utils/sources/france_travail.py +++ b/pipeline/dags/dag_utils/sources/france_travail.py @@ -10,7 +10,7 @@ def extract(url, token, id=None): client_id, client_secret = token.split(":") token = requests.post( - url="https://entreprise.pole-emploi.fr/connexion/oauth2/access_token", + url="https://entreprise.francetravail.fr/connexion/oauth2/access_token", params={ "realm": "/partenaire", }, diff --git a/pipeline/dbt/models/staging/sources/france_travail/stg_france_travail__agences.sql b/pipeline/dbt/models/staging/sources/france_travail/stg_france_travail__agences.sql index 064a8d12b..5ab5eed21 100644 --- a/pipeline/dbt/models/staging/sources/france_travail/stg_france_travail__agences.sql +++ b/pipeline/dbt/models/staging/sources/france_travail/stg_france_travail__agences.sql @@ -4,22 +4,22 @@ WITH source AS ( final AS ( SELECT - _di_source_id AS "_di_source_id", - CURRENT_DATE AS "date_maj", - CASE WHEN data ->> 'dispositifADEDA' = 'true' THEN 'https://www.pole-emploi.fr/actualites/a-laffiche/2022/adeda-un-dispositif-pour-mieux-a.html' END AS "accessibilite", - CAST(data #>> '{adressePrincipale,gpsLat}' AS FLOAT) AS "latitude", - CAST(data #>> '{adressePrincipale,gpsLon}' AS FLOAT) AS "longitude", - data #>> '{adressePrincipale,ligne4}' AS "adresse", - data #>> '{adressePrincipale,ligne3}' AS "complement_adresse", - data #>> '{adressePrincipale,communeImplantation}' AS "code_insee", - data #>> '{adressePrincipale,bureauDistributeur}' AS "code_postal", - data #>> '{contact,email}' AS "courriel", - data ->> 'horaires_open_street_map' AS "horaires_open_street_map", - data #>> '{contact,telephonePublic}' AS "telephone", - data ->> 'code' AS "id", - data ->> 'libelleEtendu' AS "nom", - data ->> 'siret' AS "siret", - data ->> 'type' AS "typologie" + _di_source_id AS "_di_source_id", + CURRENT_DATE AS "date_maj", + CASE WHEN data ->> 'dispositifADEDA' = 'true' THEN 'https://www.francetravail.fr/actualites/a-laffiche/2022/adeda-un-dispositif-pour-mieux-a.html' END AS "accessibilite", + CAST(data #>> '{adressePrincipale,gpsLat}' AS FLOAT) AS "latitude", + CAST(data #>> '{adressePrincipale,gpsLon}' AS FLOAT) AS "longitude", + data #>> '{adressePrincipale,ligne4}' AS "adresse", + data #>> '{adressePrincipale,ligne3}' AS "complement_adresse", + data #>> '{adressePrincipale,communeImplantation}' AS "code_insee", + data #>> '{adressePrincipale,bureauDistributeur}' AS "code_postal", + data #>> '{contact,email}' AS "courriel", + data ->> 'horaires_open_street_map' AS "horaires_open_street_map", + data #>> '{contact,telephonePublic}' AS "telephone", + data ->> 'code' AS "id", + data ->> 'libelleEtendu' AS "nom", + data ->> 'siret' AS "siret", + data ->> 'type' AS "typologie" FROM source ) From f4b919e4d4fbab3af673ee20304dd20c7a0646dc Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 19 Sep 2024 09:14:21 +0200 Subject: [PATCH 2/2] chore(pipeline) : Convert 'PE' entities to 'FT' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some sources still transmit 'PE' types for their structures, which leads to those being discarded. Avant: data-inclusion=# select count(*) from public_dbt_test__audit.check_structure_int__union_structures__di_surrogate_id where value = 'PE'; -[ RECORD 1 ] count | 1146 Après: data-inclusion=# select count(*) from public_dbt_test__audit.check_structure_int__union_structures__di_surrogate_id where value = 'PE'; -[ RECORD 1 ] count | 0 --- .../int_emplois_de_linclusion__structures.sql | 5 ++++- .../int_mediation_numerique__structures.sql | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pipeline/dbt/models/intermediate/sources/emplois_de_linclusion/int_emplois_de_linclusion__structures.sql b/pipeline/dbt/models/intermediate/sources/emplois_de_linclusion/int_emplois_de_linclusion__structures.sql index 2163b5260..d848298a3 100644 --- a/pipeline/dbt/models/intermediate/sources/emplois_de_linclusion/int_emplois_de_linclusion__structures.sql +++ b/pipeline/dbt/models/intermediate/sources/emplois_de_linclusion/int_emplois_de_linclusion__structures.sql @@ -21,7 +21,10 @@ final AS ( labels_nationaux AS "labels_nationaux", labels_autres AS "labels_autres", thematiques AS "thematiques", - typologie AS "typologie", + CASE + WHEN typologie = 'PE' THEN 'FT' + ELSE typologie + END AS typologie, date_maj AS "date_maj", NULLIF(siret, '') AS "siret", NULLIF(nom, '') AS "nom", diff --git a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__structures.sql b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__structures.sql index 0f34491e6..4679adaf1 100644 --- a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__structures.sql +++ b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__structures.sql @@ -17,7 +17,10 @@ final AS ( accessibilite AS "accessibilite", labels_nationaux AS "labels_nationaux", thematiques AS "thematiques", - typologie AS "typologie", + CASE + WHEN typologie = 'PE' THEN 'FT' + ELSE typologie + END AS typologie, presentation_resume AS "presentation_resume", {{ truncate_text("presentation_detail") }} AS "presentation_detail", CAST(date_maj AS DATE) AS "date_maj",