diff --git a/api/requirements/dev-requirements.txt b/api/requirements/dev-requirements.txt index 2174a2f0..bd241634 100644 --- a/api/requirements/dev-requirements.txt +++ b/api/requirements/dev-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile setup.py --extra=dev --output-file=requirements/dev-requirements.txt alembic==1.13.1 + # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -21,6 +22,7 @@ cachetools==5.3.3 # via tox certifi==2024.2.2 # via + # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -40,6 +42,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # data-inclusion-api (setup.py) # click-plugins # cligj # fiona @@ -51,8 +54,11 @@ cligj==0.7.2 colorama==0.4.6 # via tox cryptography==42.0.5 - # via python-jose -data-inclusion-schema==0.14.0 + # via + # data-inclusion-api (setup.py) + # python-jose +data-inclusion-schema==0.15.0 + # via data-inclusion-api (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -62,13 +68,17 @@ ecdsa==0.19.0 email-validator==2.1.1 # via pydantic faker==24.11.0 + # via data-inclusion-api (setup.py) fastapi==0.110.2 # via + # data-inclusion-api (setup.py) # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 + # via data-inclusion-api (setup.py) fastapi-pagination==0.12.23 + # via data-inclusion-api (setup.py) filelock==3.13.4 # via # tox @@ -76,11 +86,15 @@ filelock==3.13.4 fiona==1.9.6 # via geopandas furl==2.1.3 + # via data-inclusion-api (setup.py) geoalchemy2==0.14.7 + # via data-inclusion-api (setup.py) geopandas==0.14.3 + # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 + # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -90,6 +104,7 @@ httpcore==1.0.5 httptools==0.6.1 # via uvicorn httpx==0.27.0 + # via data-inclusion-api (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -109,12 +124,14 @@ markupsafe==2.1.5 # jinja2 # mako minio==7.2.5 + # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr nodeenv==1.8.0 # via pre-commit numpy==1.26.4 # via + # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -128,7 +145,9 @@ packaging==24.0 # pyproject-api # tox pandas==2.2.2 - # via geopandas + # via + # data-inclusion-api (setup.py) + # geopandas platformdirs==4.2.0 # via # tox @@ -136,11 +155,15 @@ platformdirs==4.2.0 pluggy==1.5.0 # via tox pre-commit==3.7.0 + # via data-inclusion-api (setup.py) psutil==5.9.8 # via py7zr psycopg2==2.9.9 + # via data-inclusion-api (setup.py) py7zr==0.21.0 + # via data-inclusion-api (setup.py) pyarrow==16.0.0 + # via data-inclusion-api (setup.py) pyasn1==0.6.0 # via # python-jose @@ -155,6 +178,7 @@ pycryptodomex==3.20.0 # via py7zr pydantic==2.7.0 # via + # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-debug-toolbar @@ -166,7 +190,9 @@ pydantic-core==2.18.1 pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar pydantic-settings==2.2.1 - # via fastapi-debug-toolbar + # via + # data-inclusion-api (setup.py) + # fastapi-debug-toolbar pyinstrument==4.6.2 # via fastapi-debug-toolbar pyppmd==1.1.0 @@ -181,11 +207,15 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via + # data-inclusion-api (setup.py) # pydantic-settings # uvicorn python-jose==3.3.0 + # via data-inclusion-api (setup.py) pytz==2024.1 - # via pandas + # via + # data-inclusion-api (setup.py) + # pandas pyyaml==6.0.1 # via # pre-commit @@ -193,10 +223,13 @@ pyyaml==6.0.1 pyzstd==0.15.10 # via py7zr requests==2.31.0 + # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose ruff==0.4.1 + # via data-inclusion-api (setup.py) sentry-sdk==1.45.0 + # via data-inclusion-api (setup.py) setuptools==69.5.1 # via nodeenv shapely==2.0.4 @@ -214,6 +247,7 @@ sniffio==1.3.1 # httpx sqlalchemy==2.0.29 # via + # data-inclusion-api (setup.py) # alembic # geoalchemy2 sqlparse==0.5.0 @@ -223,7 +257,9 @@ starlette==0.37.2 texttable==1.7.0 # via py7zr tox==4.14.2 + # via data-inclusion-api (setup.py) tqdm==4.66.2 + # via data-inclusion-api (setup.py) typing-extensions==4.11.0 # via # alembic @@ -241,7 +277,9 @@ urllib3==2.2.1 # requests # sentry-sdk uv==0.1.35 + # via data-inclusion-api (setup.py) uvicorn==0.29.0 + # via data-inclusion-api (setup.py) uvloop==0.19.0 # via uvicorn virtualenv==20.25.3 diff --git a/api/requirements/requirements.txt b/api/requirements/requirements.txt index 25edd520..9f459700 100644 --- a/api/requirements/requirements.txt +++ b/api/requirements/requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile setup.py --output-file=requirements/requirements.txt alembic==1.13.1 + # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -18,6 +19,7 @@ brotli==1.1.0 # via py7zr certifi==2024.2.2 # via + # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -33,6 +35,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # data-inclusion-api (setup.py) # click-plugins # cligj # fiona @@ -42,8 +45,11 @@ click-plugins==1.1.1 cligj==0.7.2 # via fiona cryptography==42.0.5 - # via python-jose -data-inclusion-schema==0.14.0 + # via + # data-inclusion-api (setup.py) + # python-jose +data-inclusion-schema==0.15.0 + # via data-inclusion-api (setup.py) dnspython==2.6.1 # via email-validator ecdsa==0.19.0 @@ -51,19 +57,26 @@ ecdsa==0.19.0 email-validator==2.1.1 # via pydantic faker==24.11.0 + # via data-inclusion-api (setup.py) fastapi==0.110.2 # via + # data-inclusion-api (setup.py) # fastapi-pagination # sentry-sdk fastapi-pagination==0.12.23 + # via data-inclusion-api (setup.py) fiona==1.9.6 # via geopandas furl==2.1.3 + # via data-inclusion-api (setup.py) geoalchemy2==0.14.7 + # via data-inclusion-api (setup.py) geopandas==0.14.3 + # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 + # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -73,6 +86,7 @@ httpcore==1.0.5 httptools==0.6.1 # via uvicorn httpx==0.27.0 + # via data-inclusion-api (setup.py) idna==3.7 # via # anyio @@ -86,10 +100,12 @@ mako==1.3.3 markupsafe==2.1.5 # via mako minio==7.2.5 + # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr numpy==1.26.4 # via + # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -101,12 +117,17 @@ packaging==24.0 # geopandas # gunicorn pandas==2.2.2 - # via geopandas + # via + # data-inclusion-api (setup.py) + # geopandas psutil==5.9.8 # via py7zr psycopg2==2.9.9 + # via data-inclusion-api (setup.py) py7zr==0.21.0 + # via data-inclusion-api (setup.py) pyarrow==16.0.0 + # via data-inclusion-api (setup.py) pyasn1==0.6.0 # via # python-jose @@ -121,6 +142,7 @@ pycryptodomex==3.20.0 # via py7zr pydantic==2.7.0 # via + # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-pagination @@ -128,6 +150,7 @@ pydantic==2.7.0 pydantic-core==2.18.1 # via pydantic pydantic-settings==2.2.1 + # via data-inclusion-api (setup.py) pyppmd==1.1.0 # via py7zr pyproj==3.6.1 @@ -138,19 +161,25 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via + # data-inclusion-api (setup.py) # pydantic-settings # uvicorn python-jose==3.3.0 + # via data-inclusion-api (setup.py) pytz==2024.1 - # via pandas + # via + # data-inclusion-api (setup.py) + # pandas pyyaml==6.0.1 # via uvicorn pyzstd==0.15.10 # via py7zr requests==2.31.0 + # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose sentry-sdk==1.45.0 + # via data-inclusion-api (setup.py) shapely==2.0.4 # via geopandas six==1.16.0 @@ -166,6 +195,7 @@ sniffio==1.3.1 # httpx sqlalchemy==2.0.29 # via + # data-inclusion-api (setup.py) # alembic # geoalchemy2 starlette==0.37.2 @@ -173,6 +203,7 @@ starlette==0.37.2 texttable==1.7.0 # via py7zr tqdm==4.66.2 + # via data-inclusion-api (setup.py) typing-extensions==4.11.0 # via # alembic @@ -190,6 +221,7 @@ urllib3==2.2.1 # requests # sentry-sdk uvicorn==0.29.0 + # via data-inclusion-api (setup.py) uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/api/requirements/test-requirements.txt b/api/requirements/test-requirements.txt index 26894f89..26d3be83 100644 --- a/api/requirements/test-requirements.txt +++ b/api/requirements/test-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile setup.py --extra=test --output-file=requirements/test-requirements.txt alembic==1.13.1 + # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -19,6 +20,7 @@ brotli==1.1.0 # via py7zr certifi==2024.2.2 # via + # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -34,6 +36,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # data-inclusion-api (setup.py) # click-plugins # cligj # fiona @@ -43,8 +46,11 @@ click-plugins==1.1.1 cligj==0.7.2 # via fiona cryptography==42.0.5 - # via python-jose -data-inclusion-schema==0.14.0 + # via + # data-inclusion-api (setup.py) + # python-jose +data-inclusion-schema==0.15.0 + # via data-inclusion-api (setup.py) dnspython==2.6.1 # via email-validator ecdsa==0.19.0 @@ -52,23 +58,33 @@ ecdsa==0.19.0 email-validator==2.1.1 # via pydantic factory-boy==3.3.0 + # via data-inclusion-api (setup.py) faker==24.11.0 - # via factory-boy + # via + # data-inclusion-api (setup.py) + # factory-boy fastapi==0.110.2 # via + # data-inclusion-api (setup.py) # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 + # via data-inclusion-api (setup.py) fastapi-pagination==0.12.23 + # via data-inclusion-api (setup.py) fiona==1.9.6 # via geopandas furl==2.1.3 + # via data-inclusion-api (setup.py) geoalchemy2==0.14.7 + # via data-inclusion-api (setup.py) geopandas==0.14.3 + # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 + # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -78,6 +94,7 @@ httpcore==1.0.5 httptools==0.6.1 # via uvicorn httpx==0.27.0 + # via data-inclusion-api (setup.py) idna==3.7 # via # anyio @@ -97,10 +114,12 @@ markupsafe==2.1.5 # jinja2 # mako minio==7.2.5 + # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr numpy==1.26.4 # via + # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -113,14 +132,19 @@ packaging==24.0 # gunicorn # pytest pandas==2.2.2 - # via geopandas + # via + # data-inclusion-api (setup.py) + # geopandas pluggy==1.5.0 # via pytest psutil==5.9.8 # via py7zr psycopg2==2.9.9 + # via data-inclusion-api (setup.py) py7zr==0.21.0 + # via data-inclusion-api (setup.py) pyarrow==16.0.0 + # via data-inclusion-api (setup.py) pyasn1==0.6.0 # via # python-jose @@ -135,6 +159,7 @@ pycryptodomex==3.20.0 # via py7zr pydantic==2.7.0 # via + # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-debug-toolbar @@ -146,7 +171,9 @@ pydantic-core==2.18.1 pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar pydantic-settings==2.2.1 - # via fastapi-debug-toolbar + # via + # data-inclusion-api (setup.py) + # fastapi-debug-toolbar pyinstrument==4.6.2 # via fastapi-debug-toolbar pyppmd==1.1.0 @@ -155,29 +182,37 @@ pyproj==3.6.1 # via geopandas pytest==8.1.1 # via + # data-inclusion-api (setup.py) # pytest-dotenv # syrupy pytest-dotenv==0.5.2 + # via data-inclusion-api (setup.py) python-dateutil==2.9.0.post0 # via # faker # pandas python-dotenv==1.0.1 # via + # data-inclusion-api (setup.py) # pydantic-settings # pytest-dotenv # uvicorn python-jose==3.3.0 + # via data-inclusion-api (setup.py) pytz==2024.1 - # via pandas + # via + # data-inclusion-api (setup.py) + # pandas pyyaml==6.0.1 # via uvicorn pyzstd==0.15.10 # via py7zr requests==2.31.0 + # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose sentry-sdk==1.45.0 + # via data-inclusion-api (setup.py) shapely==2.0.4 # via geopandas six==1.16.0 @@ -193,6 +228,7 @@ sniffio==1.3.1 # httpx sqlalchemy==2.0.29 # via + # data-inclusion-api (setup.py) # alembic # geoalchemy2 sqlparse==0.5.0 @@ -200,9 +236,11 @@ sqlparse==0.5.0 starlette==0.37.2 # via fastapi syrupy==4.6.1 + # via data-inclusion-api (setup.py) texttable==1.7.0 # via py7zr tqdm==4.66.2 + # via data-inclusion-api (setup.py) typing-extensions==4.11.0 # via # alembic @@ -220,6 +258,7 @@ urllib3==2.2.1 # requests # sentry-sdk uvicorn==0.29.0 + # via data-inclusion-api (setup.py) uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/api/setup.py b/api/setup.py index db5f734c..63a15686 100644 --- a/api/setup.py +++ b/api/setup.py @@ -38,7 +38,7 @@ "sqlalchemy", "tqdm", "uvicorn[standard]", - "data-inclusion-schema==0.14.0", + "data-inclusion-schema==0.15.0", ], extras_require={ "test": [ diff --git a/api/src/alembic/versions/20240610_092847_170af30febde_add_service_page_web.py b/api/src/alembic/versions/20240610_092847_170af30febde_add_service_page_web.py new file mode 100644 index 00000000..bc226f65 --- /dev/null +++ b/api/src/alembic/versions/20240610_092847_170af30febde_add_service_page_web.py @@ -0,0 +1,24 @@ +"""Add page_web to service model + +Revision ID: 170af30febde +Revises: 14252f787143 +Create Date: 2024-06-10 09:28:47.160120 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "170af30febde" +down_revision = "14252f787143" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("api__services", sa.Column("page_web", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("api__services", "page_web") diff --git a/api/src/data_inclusion/api/core/db.py b/api/src/data_inclusion/api/core/db.py index 56ecf2d0..936b058a 100644 --- a/api/src/data_inclusion/api/core/db.py +++ b/api/src/data_inclusion/api/core/db.py @@ -3,6 +3,7 @@ from typing import Annotated import sqlalchemy as sqla +import sqlalchemy.types as types from sqlalchemy import orm from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.orm import mapped_column @@ -21,6 +22,16 @@ ] +class SortedTextArray(types.TypeDecorator): + """Automatically sorts a list of strings when inserting into the database.""" + + impl = ARRAY(sqla.Text) + cache_ok = True + + def process_bind_param(self, value, dialect): + return sorted(value) if value is not None else None + + class Base(orm.DeclarativeBase): metadata = sqla.MetaData( naming_convention={ @@ -32,7 +43,7 @@ class Base(orm.DeclarativeBase): } ) type_annotation_map = { - list[str]: ARRAY(sqla.Text), + list[str]: SortedTextArray, dict: JSONB, } diff --git a/api/src/data_inclusion/api/inclusion_data/models.py b/api/src/data_inclusion/api/inclusion_data/models.py index 9a0466e8..58f94e32 100644 --- a/api/src/data_inclusion/api/inclusion_data/models.py +++ b/api/src/data_inclusion/api/inclusion_data/models.py @@ -17,32 +17,32 @@ class Structure(Base): _di_geocodage_score: Mapped[float | None] # structure data - source: Mapped[str] - id: Mapped[str] - siret: Mapped[str | None] - rna: Mapped[str | None] - nom: Mapped[str] - commune: Mapped[str | None] - code_postal: Mapped[str | None] - code_insee: Mapped[str | None] + accessibilite: Mapped[str | None] adresse: Mapped[str | None] + antenne: Mapped[bool | None] = mapped_column(default=False) + code_insee: Mapped[str | None] + code_postal: Mapped[str | None] + commune: Mapped[str | None] complement_adresse: Mapped[str | None] - longitude: Mapped[float | None] - latitude: Mapped[float | None] - typologie: Mapped[str | None] - telephone: Mapped[str | None] courriel: Mapped[str | None] - site_web: Mapped[str | None] - presentation_resume: Mapped[str | None] - presentation_detail: Mapped[str | None] date_maj: Mapped[date | None] - antenne: Mapped[bool | None] = mapped_column(default=False) - lien_source: Mapped[str | None] horaires_ouverture: Mapped[str | None] - accessibilite: Mapped[str | None] - labels_nationaux: Mapped[list[str] | None] + id: Mapped[str] labels_autres: Mapped[list[str] | None] + labels_nationaux: Mapped[list[str] | None] + latitude: Mapped[float | None] + lien_source: Mapped[str | None] + longitude: Mapped[float | None] + nom: Mapped[str] + presentation_detail: Mapped[str | None] + presentation_resume: Mapped[str | None] + rna: Mapped[str | None] + siret: Mapped[str | None] + site_web: Mapped[str | None] + source: Mapped[str] + telephone: Mapped[str | None] thematiques: Mapped[list[str] | None] + typologie: Mapped[str | None] services: Mapped[list["Service"]] = relationship(back_populates="structure") @@ -63,46 +63,47 @@ class Service(Base): structure: Mapped[Structure] = relationship(back_populates="services") # service data - source: Mapped[str] - id: Mapped[str] - structure_id: Mapped[str] - nom: Mapped[str] - presentation_resume: Mapped[str | None] - presentation_detail: Mapped[str | None] - types: Mapped[list[str] | None] - thematiques: Mapped[list[str] | None] - prise_rdv: Mapped[str | None] - frais: Mapped[list[str] | None] - frais_autres: Mapped[str | None] - profils: Mapped[list[str] | None] - pre_requis: Mapped[list[str] | None] - cumulable: Mapped[bool | None] = mapped_column(default=False) - justificatifs: Mapped[list[str] | None] - formulaire_en_ligne: Mapped[str | None] - commune: Mapped[str | None] - code_postal: Mapped[str | None] - code_insee: Mapped[str | None] = mapped_column(sqla.ForeignKey(Commune.code)) adresse: Mapped[str | None] + code_insee: Mapped[str | None] = mapped_column(sqla.ForeignKey(Commune.code)) + code_postal: Mapped[str | None] + commune: Mapped[str | None] complement_adresse: Mapped[str | None] - longitude: Mapped[float | None] - latitude: Mapped[float | None] - recurrence: Mapped[str | None] + contact_nom_prenom: Mapped[str | None] + contact_public: Mapped[bool | None] = mapped_column(default=False) + courriel: Mapped[str | None] + cumulable: Mapped[bool | None] = mapped_column(default=False) date_creation: Mapped[date | None] + date_maj: Mapped[date | None] date_suspension: Mapped[date | None] + formulaire_en_ligne: Mapped[str | None] + frais_autres: Mapped[str | None] + frais: Mapped[list[str] | None] + id: Mapped[str] + justificatifs: Mapped[list[str] | None] + latitude: Mapped[float | None] lien_source: Mapped[str | None] - telephone: Mapped[str | None] - courriel: Mapped[str | None] - contact_public: Mapped[bool | None] = mapped_column(default=False) - contact_nom_prenom: Mapped[str | None] - date_maj: Mapped[date | None] + longitude: Mapped[float | None] modes_accueil: Mapped[list[str] | None] - modes_orientation_accompagnateur: Mapped[list[str] | None] modes_orientation_accompagnateur_autres: Mapped[str | None] - modes_orientation_beneficiaire: Mapped[list[str] | None] + modes_orientation_accompagnateur: Mapped[list[str] | None] modes_orientation_beneficiaire_autres: Mapped[str | None] - zone_diffusion_type: Mapped[str | None] + modes_orientation_beneficiaire: Mapped[list[str] | None] + nom: Mapped[str] + page_web: Mapped[str | None] + pre_requis: Mapped[list[str] | None] + presentation_detail: Mapped[str | None] + presentation_resume: Mapped[str | None] + prise_rdv: Mapped[str | None] + profils: Mapped[list[str] | None] + recurrence: Mapped[str | None] + source: Mapped[str] + structure_id: Mapped[str] + telephone: Mapped[str | None] + thematiques: Mapped[list[str] | None] + types: Mapped[list[str] | None] zone_diffusion_code: Mapped[str | None] zone_diffusion_nom: Mapped[str | None] + zone_diffusion_type: Mapped[str | None] commune_: Mapped[Commune] = relationship(back_populates="services") diff --git a/api/src/data_inclusion/api/inclusion_data/sources.json b/api/src/data_inclusion/api/inclusion_data/sources.json index f22b7d08..e7ead309 100644 --- a/api/src/data_inclusion/api/inclusion_data/sources.json +++ b/api/src/data_inclusion/api/inclusion_data/sources.json @@ -1,4 +1,9 @@ [ + { + "slug": "action-logement", + "nom": "Action Logement", + "description": "Aides et solutions pour accompagner les salariés dans leur parcours logement. Contient les services proposés par les structures d’Action Logement sur tout le territoire." + }, { "slug": "agefiph", "nom": "AGEFIPH", diff --git a/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr b/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr index 9238e85a..fe066101 100644 --- a/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr +++ b/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr @@ -1313,7 +1313,8 @@ "items": { "$ref": "#/components/schemas/TypologieService" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1327,7 +1328,8 @@ "items": { "$ref": "#/components/schemas/Thematique" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1352,7 +1354,8 @@ "items": { "$ref": "#/components/schemas/Frais" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1377,7 +1380,8 @@ "items": { "$ref": "#/components/schemas/Profil" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1391,7 +1395,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1416,7 +1421,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1620,7 +1626,8 @@ "items": { "$ref": "#/components/schemas/ModeAccueil" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1692,13 +1699,32 @@ ], "title": "Contact Nom Prenom" }, + "page_web": { + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "Page Web", + "description": "Lien vers une page web dédiée au service sur le site web de la structure. Ce champ n'est pas destiné à recevoir un lien vers le site d'un producteur de donnée.", + "examples": [ + "https://insersol.fr/biclou-atelier-reparation-participatif-solidaire/" + ] + }, "modes_orientation_beneficiaire": { "anyOf": [ { "items": { "$ref": "#/components/schemas/ModeOrientationBeneficiaire" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -1723,7 +1749,8 @@ "items": { "$ref": "#/components/schemas/ModeOrientationAccompagnateur" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2038,7 +2065,8 @@ "items": { "$ref": "#/components/schemas/LabelNational" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2052,7 +2080,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2066,7 +2095,8 @@ "items": { "$ref": "#/components/schemas/Thematique" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2299,21 +2329,27 @@ "Profil": { "type": "string", "enum": [ - "seniors-65", - "familles-enfants", "adultes", - "jeunes-16-26", - "public-langues-etrangeres", + "alternants", + "beneficiaires-rsa", "deficience-visuelle", - "surdite", - "handicaps-psychiques", - "handicaps-mentaux", + "demandeurs-demploi", + "familles-enfants", "femmes", - "personnes-handicapees", + "handicaps-mentaux", + "handicaps-psychiques", + "jeunes", + "jeunes-16-26", + "locataires", "personnes-en-situation-de-handicap", "personnes-en-situation-illettrisme", - "beneficiaires-rsa", - "demandeurs-demploi" + "personnes-handicapees", + "proprietaires", + "public-langues-etrangeres", + "retraites", + "salaries", + "seniors-65", + "surdite" ], "title": "Profil" }, @@ -2364,7 +2400,8 @@ "items": { "$ref": "#/components/schemas/TypologieService" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2378,7 +2415,8 @@ "items": { "$ref": "#/components/schemas/Thematique" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2403,7 +2441,8 @@ "items": { "$ref": "#/components/schemas/Frais" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2428,7 +2467,8 @@ "items": { "$ref": "#/components/schemas/Profil" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2442,7 +2482,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2467,7 +2508,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2671,7 +2713,8 @@ "items": { "$ref": "#/components/schemas/ModeAccueil" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2743,13 +2786,32 @@ ], "title": "Contact Nom Prenom" }, + "page_web": { + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], + "title": "Page Web", + "description": "Lien vers une page web dédiée au service sur le site web de la structure. Ce champ n'est pas destiné à recevoir un lien vers le site d'un producteur de donnée.", + "examples": [ + "https://insersol.fr/biclou-atelier-reparation-participatif-solidaire/" + ] + }, "modes_orientation_beneficiaire": { "anyOf": [ { "items": { "$ref": "#/components/schemas/ModeOrientationBeneficiaire" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -2774,7 +2836,8 @@ "items": { "$ref": "#/components/schemas/ModeOrientationAccompagnateur" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -3138,7 +3201,8 @@ "items": { "$ref": "#/components/schemas/LabelNational" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -3152,7 +3216,8 @@ "items": { "type": "string" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -3166,7 +3231,8 @@ "items": { "$ref": "#/components/schemas/Thematique" }, - "type": "array" + "type": "array", + "uniqueItems": true }, { "type": "null" @@ -3220,8 +3286,8 @@ "acces-aux-droits-et-citoyennete--aide-aux-victimes", "acces-aux-droits-et-citoyennete--connaitre-ses-droits", "acces-aux-droits-et-citoyennete--demandeurs-dasile-et-naturalisation", - "acces-aux-droits-et-citoyennete--faciliter-laction-citoyenne", "acces-aux-droits-et-citoyennete--developpement-durable", + "acces-aux-droits-et-citoyennete--faciliter-laction-citoyenne", "accompagnement-social-et-professionnel-personnalise", "accompagnement-social-et-professionnel-personnalise--decrochage-scolaire", "accompagnement-social-et-professionnel-personnalise--definition-du-projet-professionnel", @@ -3231,16 +3297,16 @@ "apprendre-francais--communiquer-vie-tous-les-jours", "apprendre-francais--suivre-formation", "choisir-un-metier", - "choisir-un-metier--identifier-ses-points-forts-et-ses-competences", + "choisir-un-metier--confirmer-son-choix-de-metier", "choisir-un-metier--connaitre-les-opportunites-demploi", "choisir-un-metier--decouvrir-un-metier-ou-un-secteur-dactivite", - "choisir-un-metier--confirmer-son-choix-de-metier", + "choisir-un-metier--identifier-ses-points-forts-et-ses-competences", "creation-activite", "creation-activite--definir-son-projet-de-creation-dentreprise", "creation-activite--developper-son-entreprise", - "creation-activite--structurer-son-projet-de-creation-dentreprise", - "creation-activite--reseautage-pour-createurs-dentreprise", "creation-activite--financer-son-projet", + "creation-activite--reseautage-pour-createurs-dentreprise", + "creation-activite--structurer-son-projet-de-creation-dentreprise", "equipement-et-alimentation", "equipement-et-alimentation--acces-a-du-materiel-informatique", "equipement-et-alimentation--acces-a-un-telephone-et-un-abonnement", @@ -3248,21 +3314,21 @@ "equipement-et-alimentation--electromenager", "equipement-et-alimentation--habillement", "famille", - "famille--information-et-accompagnement-des-parents", + "famille--accompagnement-femme-enceinte-bebe-jeune-enfant", "famille--garde-denfants", + "famille--information-et-accompagnement-des-parents", "famille--jeunes-sans-soutien-familial", + "famille--soutien-a-la-parentalite", "famille--soutien-aux-familles", "famille--violences-intrafamiliales", - "famille--accompagnement-femme-enceinte-bebe-jeune-enfant", - "famille--soutien-a-la-parentalite", "gestion-financiere", "gestion-financiere--accompagnement-aux-personnes-en-difficultes-financieres", + "gestion-financiere--acces-au-micro-credit", "gestion-financiere--apprendre-a-gerer-son-budget", - "gestion-financiere--prevention-et-gestion-du-surendettement", + "gestion-financiere--beneficier-daides-financieres", "gestion-financiere--creation-et-utilisation-dun-compte-bancaire", - "gestion-financiere--acces-au-micro-credit", "gestion-financiere--obtenir-une-aide-alimentaire", - "gestion-financiere--beneficier-daides-financieres", + "gestion-financiere--prevention-et-gestion-du-surendettement", "handicap", "handicap--accompagnement-par-une-structure-specialisee", "handicap--adaptation-au-poste-de-travail", @@ -3270,8 +3336,8 @@ "handicap--connaissance-des-droits-des-travailleurs", "handicap--faire-reconnaitre-un-handicap", "handicap--favoriser-le-retour-et-le-maintien-dans-lemploi", - "handicap--mobilite-des-personnes-en-situation-de-handicap", "handicap--gerer-le-depart-a-la-retraite-des-personnes-en-situation-de-handicap", + "handicap--mobilite-des-personnes-en-situation-de-handicap", "illettrisme", "illettrisme--accompagner-scolarite", "illettrisme--ameliorer-vocabulaire", @@ -3284,79 +3350,83 @@ "illettrisme--utiliser-numerique", "illettrisme--valider-certification-clea", "logement-hebergement", - "logement-hebergement--etre-accompagne-pour-se-loger", + "logement-hebergement--aides-financieres-investissement-locatif", "logement-hebergement--besoin-dadapter-mon-logement", + "logement-hebergement--connaissance-de-ses-droits-et-interlocuteurs", "logement-hebergement--demenagement", + "logement-hebergement--etre-accompagne-dans-son-projet-accession", + "logement-hebergement--etre-accompagne-en cas-de-difficultes-financieres", + "logement-hebergement--etre-accompagne-pour-se-loger", + "logement-hebergement--financer-son-projet-travaux", + "logement-hebergement--gerer-son-budget", "logement-hebergement--mal-loges-sans-logis", "logement-hebergement--probleme-avec-son-logement", "logement-hebergement--reprendre-un-emploi-ou-une-formation", - "logement-hebergement--connaissance-de-ses-droits-et-interlocuteurs", - "logement-hebergement--gerer-son-budget", "mobilite", - "mobilite--etre-accompagne-dans-son-parcours-mobilite", - "mobilite--entretenir-reparer-son-vehicule", - "mobilite--louer-un-vehicule", "mobilite--acheter-un-vehicule-motorise", - "mobilite--preparer-son-permis-de-conduire-se-reentrainer-a-la-conduite", + "mobilite--acheter-un-velo", "mobilite--aides-a-la-reprise-demploi-ou-a-la-formation", - "mobilite--comprendre-et-utiliser-les-transports-en-commun", "mobilite--apprendre-a-utiliser-un-deux-roues", + "mobilite--comprendre-et-utiliser-les-transports-en-commun", + "mobilite--entretenir-reparer-son-vehicule", + "mobilite--etre-accompagne-dans-son-parcours-mobilite", + "mobilite--louer-un-vehicule", "mobilite--financer-mon-projet-mobilite", - "mobilite--acheter-un-velo", + "mobilite--preparer-son-permis-de-conduire-se-reentrainer-a-la-conduite", "numerique", - "numerique--devenir-autonome-dans-les-demarches-administratives", - "numerique--realiser-des-demarches-administratives-avec-un-accompagnement", - "numerique--prendre-en-main-un-smartphone-ou-une-tablette", - "numerique--prendre-en-main-un-ordinateur", - "numerique--utiliser-le-numerique-au-quotidien", - "numerique--approfondir-ma-culture-numerique", - "numerique--favoriser-mon-insertion-professionnelle", - "numerique--acceder-a-une-connexion-internet", "numerique--acceder-a-du-materiel", - "numerique--s-equiper-en-materiel-informatique", - "numerique--creer-et-developper-mon-entreprise", - "numerique--creer-avec-le-numerique", + "numerique--acceder-a-une-connexion-internet", "numerique--accompagner-les-demarches-de-sante", + "numerique--approfondir-ma-culture-numerique", + "numerique--creer-avec-le-numerique", + "numerique--creer-et-developper-mon-entreprise", + "numerique--devenir-autonome-dans-les-demarches-administratives", + "numerique--favoriser-mon-insertion-professionnelle", + "numerique--prendre-en-main-un-ordinateur", + "numerique--prendre-en-main-un-smartphone-ou-une-tablette", "numerique--promouvoir-la-citoyennete-numerique", + "numerique--realiser-des-demarches-administratives-avec-un-accompagnement", + "numerique--s-equiper-en-materiel-informatique", "numerique--soutenir-la-parentalite-et-l-education-avec-le-numerique", + "numerique--utiliser-le-numerique-au-quotidien", "preparer-sa-candidature", - "preparer-sa-candidature--valoriser-ses-competences", - "preparer-sa-candidature--realiser-un-cv-et-ou-une-lettre-de-motivation", "preparer-sa-candidature--developper-son-reseau", "preparer-sa-candidature--organiser-ses-demarches-de-recherche-demploi", + "preparer-sa-candidature--realiser-un-cv-et-ou-une-lettre-de-motivation", + "preparer-sa-candidature--valoriser-ses-competences", "remobilisation", "remobilisation--bien-etre", + "remobilisation--decouvrir-son-potentiel-via-le-sport-et-la-culture", + "remobilisation--discrimination", "remobilisation--identifier-ses-competences-et-aptitudes", "remobilisation--lien-social", - "remobilisation--restaurer-sa-confiance-son-image-de-soi", - "remobilisation--pression-sociale", - "remobilisation--discrimination", - "remobilisation--decouvrir-son-potentiel-via-le-sport-et-la-culture", "remobilisation--participer-a-des-actions-solidaires-ou-de-benevolat", + "remobilisation--pression-sociale", + "remobilisation--restaurer-sa-confiance-son-image-de-soi", "sante", - "sante--faire-face-a-une-situation-daddiction", + "sante--acces-aux-soins", + "sante--accompagnement-de-la-femme-enceinte-du-bebe-et-du-jeune-enfant", + "sante--accompagner-les-traumatismes", "sante--bien-etre-psychologique", + "sante--diagnostic-et-accompagnement-a-lemployabilite", + "sante--faire-face-a-une-situation-daddiction", "sante--obtenir-la-prise-en-charge-de-frais-medicaux", - "sante--se-soigner-et-prevenir-la-maladie", - "sante--accompagnement-de-la-femme-enceinte-du-bebe-et-du-jeune-enfant", "sante--prevention-et-acces-aux-soins", + "sante--se-soigner-et-prevenir-la-maladie", "sante--vie-relationnelle-et-affective", - "sante--accompagner-les-traumatismes", - "sante--acces-aux-soins", - "sante--diagnostic-et-accompagnement-a-lemployabilite", "se-former", - "se-former--trouver-sa-formation", "se-former--monter-son-dossier-de-formation", + "se-former--trouver-sa-formation", "se-former--utiliser-le-numerique", "souvrir-a-linternational", "souvrir-a-linternational--connaitre-les-opportunites-demploi-a-letranger", "souvrir-a-linternational--sinformer-sur-les-aides-pour-travailler-a-letranger", "souvrir-a-linternational--sorganiser-suite-a-son-retour-en-france", "trouver-un-emploi", - "trouver-un-emploi--repondre-a-des-offres-demploi", + "trouver-un-emploi--convaincre-un-recruteur-en-entretien", "trouver-un-emploi--faire-des-candidatures-spontanees", - "trouver-un-emploi--suivre-ses-candidatures-et-relancer-les-employeurs", - "trouver-un-emploi--convaincre-un-recruteur-en-entretien" + "trouver-un-emploi--repondre-a-des-offres-demploi", + "trouver-un-emploi--suivre-ses-candidatures-et-relancer-les-employeurs" ], "title": "Thematique" }, @@ -3382,14 +3452,20 @@ "CCAS", "CCONS", "CD", + "CDAS", + "CFP", "CHRS", "CHU", "CIAS", "CIDFF", "CITMET", + "CMP", + "CMS", + "CPAM", "CPH", "CS", "CSAPA", + "CSC", "DEETS", "DEPT", "DIPLP", @@ -3398,14 +3474,21 @@ "EATT", "EI", "EITI", + "ENM", "EPCI", + "EPI", "EPIDE", + "EPN", + "ES", "ESS", "ETTI", + "EVS", + "FABLAB", "FAIS", "FT", "GEIQ", "HUDA", + "LA_POSTE", "MDE", "MDEF", "MDPH", @@ -3414,6 +3497,7 @@ "ML", "MQ", "MSA", + "MSAP", "MUNI", "OACAS", "ODC", @@ -3423,6 +3507,7 @@ "PAD", "PE", "PENSION", + "PI", "PIJ_BIJ", "PIMMS", "PJJ", @@ -3430,6 +3515,7 @@ "PREF", "PREVENTION", "REG", + "RESSOURCERIE", "RFS", "RS_FJT", "SCP", diff --git a/api/tests/e2e/api/test_inclusion_data.py b/api/tests/e2e/api/test_inclusion_data.py index ee1366fe..1ab0b80f 100644 --- a/api/tests/e2e/api/test_inclusion_data.py +++ b/api/tests/e2e/api/test_inclusion_data.py @@ -70,8 +70,8 @@ def test_list_structures_all(api_client): "horaires_ouverture": 'Mo-Fr 10:00-20:00 "sur rendez-vous"; PH off', "accessibilite": "https://acceslibre.beta.gouv.fr/app/prince-point-monde/", "labels_nationaux": [], - "labels_autres": ["SudLabs", "Nièvre médiation numérique"], - "thematiques": ["choisir-un-metier", "creation-activite"], + "labels_autres": ["Nièvre médiation numérique"], + "thematiques": ["choisir-un-metier"], } ], "total": 1, @@ -122,9 +122,9 @@ def assert_structure_data(structure, data): assert structure.lien_source == data["lien_source"] assert structure.horaires_ouverture == data["horaires_ouverture"] assert structure.accessibilite == data["accessibilite"] - assert structure.labels_nationaux == data["labels_nationaux"] - assert structure.labels_autres == data["labels_autres"] - assert structure.thematiques == data["thematiques"] + assert sorted(structure.labels_nationaux) == sorted(data["labels_nationaux"]) + assert sorted(structure.labels_autres) == sorted(data["labels_autres"]) + assert sorted(structure.thematiques) == sorted(data["thematiques"]) assert structure._di_geocodage_code_insee == data["_di_geocodage_code_insee"] assert structure._di_geocodage_score == data["_di_geocodage_score"] @@ -365,45 +365,46 @@ def test_list_services_all(api_client): assert resp_data == { "items": [ { - "_di_geocodage_code_insee": LILLE["code_insee"], - "_di_geocodage_score": 0.5, - "adresse": "5, rue Guichard", - "code_insee": LILLE["code_insee"], - "code_postal": "25454", - "commune": "Sainte Jacquelineboeuf", + "_di_geocodage_code_insee": "59350", + "_di_geocodage_score": 0.89, + "adresse": "20, rue Lambert", + "code_insee": "59350", + "code_postal": "22846", + "commune": "Guichard", "complement_adresse": None, - "contact_nom_prenom": "Alphonse Baudry-Couturier", + "contact_nom_prenom": "Thibaut de Michaud", "contact_public": False, - "courriel": "gbonnet@example.org", + "courriel": "michelgerard@example.net", "cumulable": False, "date_creation": "2022-01-01", "date_maj": "2023-01-01", "date_suspension": "2054-01-01", "formulaire_en_ligne": None, - "frais_autres": "Camarade il.", - "frais": ["gratuit", "gratuit-sous-conditions"], - "id": "cacher-violent", + "frais_autres": "Point saint source.", + "frais": ["gratuit"], + "id": "rassurer-vaincre", "justificatifs": [], - "latitude": -0.7355065, - "lien_source": "https://dora.fr/cacher-violent", - "longitude": -88.473296, + "latitude": 80.2434875, + "lien_source": "https://dora.fr/rassurer-vaincre", + "longitude": 128.091981, "modes_accueil": ["a-distance"], "modes_orientation_accompagnateur_autres": None, "modes_orientation_accompagnateur": ["telephoner"], "modes_orientation_beneficiaire_autres": None, "modes_orientation_beneficiaire": ["telephoner"], - "nom": "Munoz", + "nom": "Barbe", + "page_web": "http://www.foucher.com/", "pre_requis": [], - "presentation_detail": "Épaule élever un.", - "presentation_resume": "Puissant fine.", - "prise_rdv": "https://teixeira.fr/", - "profils": ["femmes", "jeunes-16-26"], + "presentation_detail": "Noir roi fin parmi.", + "presentation_resume": "Épaule élever un.", + "prise_rdv": "https://www.raymond.com/", + "profils": ["femmes"], "recurrence": None, "source": "dora", - "structure_id": "libre-rouge-empire", + "structure_id": "grace-plaindre", "telephone": "0102030405", - "thematiques": ["choisir-un-metier", "creation-activite"], - "types": ["formation", "numerique"], + "thematiques": ["choisir-un-metier"], + "types": ["formation"], "zone_diffusion_code": None, "zone_diffusion_nom": None, "zone_diffusion_type": None, diff --git a/api/tests/factories.py b/api/tests/factories.py index 9d53a9cb..96583141 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -1,5 +1,4 @@ from datetime import date -from itertools import tee import factory import faker @@ -11,12 +10,6 @@ fake = faker.Faker("fr_FR") -def pairwise(iterable): - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - class RequestFactory(factory.alchemy.SQLAlchemyModelFactory): class Meta: model = Request @@ -52,7 +45,7 @@ class Meta: di_schema.Typologie.MUNI, di_schema.Typologie.PE, ], - getter=lambda t: t.value, + getter=lambda v: v.value, ) telephone = "0102030405" courriel = factory.Faker("email", locale="fr_FR") @@ -67,16 +60,14 @@ class Meta: lambda o: f"https://acceslibre.beta.gouv.fr/app/{o.id}/" ) labels_nationaux = [] - labels_autres = ["SudLabs", "Nièvre médiation numérique"] + labels_autres = ["Nièvre médiation numérique"] thematiques = factory.Iterator( - pairwise( - [ - di_schema.Thematique.CHOISIR_UN_METIER, - di_schema.Thematique.CREATION_ACTIVITE, - di_schema.Thematique.MOBILITE, - ] - ), - getter=lambda lst: list(map(lambda t: t.value, lst)), + [ + di_schema.Thematique.CHOISIR_UN_METIER, + di_schema.Thematique.CREATION_ACTIVITE, + di_schema.Thematique.MOBILITE, + ], + getter=lambda v: [v.value], ) @@ -86,6 +77,7 @@ class Meta: sqlalchemy_session_persistence = "commit" _di_surrogate_id = factory.Faker("uuid4") + _di_structure_surrogate_id = factory.SelfAttribute("structure._di_surrogate_id") _di_geocodage_code_insee = factory.SelfAttribute("code_insee") _di_geocodage_score = factory.Faker( "pyfloat", right_digits=2, positive=True, max_value=1 @@ -99,39 +91,36 @@ class Meta: presentation_resume = factory.Faker("text", max_nb_chars=20, locale="fr_FR") presentation_detail = factory.Faker("text", max_nb_chars=20, locale="fr_FR") types = factory.Iterator( - pairwise( - [ - di_schema.TypologieService.FORMATION, - di_schema.TypologieService.NUMÉRIQUE, - ] - ), - getter=lambda lst: list(map(lambda t: t.value, lst)), + [ + di_schema.TypologieService.FORMATION, + di_schema.TypologieService.NUMÉRIQUE, + ], + getter=lambda v: [v.value], ) thematiques = factory.Iterator( - pairwise( - [ - di_schema.Thematique.CHOISIR_UN_METIER, - di_schema.Thematique.CREATION_ACTIVITE, - di_schema.Thematique.MOBILITE, - ] - ), - getter=lambda lst: list(map(lambda t: t.value, lst)), + [ + di_schema.Thematique.CHOISIR_UN_METIER, + di_schema.Thematique.CREATION_ACTIVITE, + di_schema.Thematique.MOBILITE, + ], + getter=lambda v: [v.value], ) prise_rdv = factory.Faker("url", locale="fr_FR") frais = factory.Iterator( - pairwise(di_schema.Frais), - getter=lambda lst: list(map(lambda t: t.value, lst)), + [ + di_schema.Frais.GRATUIT, + ], + getter=lambda v: [v.value], ) frais_autres = factory.Faker("text", max_nb_chars=20, locale="fr_FR") + page_web = factory.Faker("url", locale="fr_FR") profils = factory.Iterator( - pairwise( - [ - di_schema.Profil.FEMMES, - di_schema.Profil.JEUNES_16_26, - di_schema.Profil.SENIORS_65, - ] - ), - getter=lambda lst: list(map(lambda t: t.value, lst)), + [ + di_schema.Profil.FEMMES, + di_schema.Profil.JEUNES_16_26, + di_schema.Profil.SENIORS_65, + ], + getter=lambda v: [v.value], ) pre_requis = [] cumulable = False @@ -155,26 +144,26 @@ class Meta: date_maj = factory.LazyFunction(lambda: date(2023, 1, 1)) modes_accueil = factory.Iterator( [ - [di_schema.ModeAccueil.A_DISTANCE.value], - [di_schema.ModeAccueil.EN_PRESENTIEL.value], - [ - di_schema.ModeAccueil.A_DISTANCE.value, - di_schema.ModeAccueil.EN_PRESENTIEL.value, - ], - ] + di_schema.ModeAccueil.A_DISTANCE, + di_schema.ModeAccueil.EN_PRESENTIEL, + di_schema.ModeAccueil.EN_PRESENTIEL, + ], + getter=lambda v: [v.value], ) modes_orientation_accompagnateur = factory.Iterator( [ - [di_schema.ModeOrientationAccompagnateur.TELEPHONER.value], - [di_schema.ModeOrientationAccompagnateur.ENVOYER_UN_MAIL.value], - ] + di_schema.ModeOrientationAccompagnateur.TELEPHONER, + di_schema.ModeOrientationAccompagnateur.ENVOYER_UN_MAIL, + ], + getter=lambda v: [v.value], ) modes_orientation_accompagnateur_autres = None modes_orientation_beneficiaire = factory.Iterator( [ - [di_schema.ModeOrientationBeneficiaire.TELEPHONER.value], - [di_schema.ModeOrientationBeneficiaire.SE_PRESENTER.value], - ] + di_schema.ModeOrientationBeneficiaire.TELEPHONER, + di_schema.ModeOrientationBeneficiaire.SE_PRESENTER, + ], + getter=lambda v: [v.value], ) modes_orientation_beneficiaire_autres = None zone_diffusion_type = None diff --git a/pipeline/dags/dag_utils/sources/__init__.py b/pipeline/dags/dag_utils/sources/__init__.py index 8f01b00e..f75667dd 100644 --- a/pipeline/dags/dag_utils/sources/__init__.py +++ b/pipeline/dags/dag_utils/sources/__init__.py @@ -253,6 +253,21 @@ }, }, }, + "action-logement": { + "schedule": "@daily", + "snapshot": True, + "reader": lambda path: utils.read_csv(path, sep=","), + "streams": { + "services": { + "filename": "services.csv", + "url": Variable.get("ACTION_LOGEMENT_SERVICES_URL", None), + }, + "structures": { + "filename": "structures.csv", + "url": Variable.get("ACTION_LOGEMENT_STRUCTURES_URL", None), + }, + }, + }, } diff --git a/pipeline/dbt/macros/domain/checks/check_service.sql b/pipeline/dbt/macros/domain/checks/check_service.sql index d4b2d697..db2e3553 100644 --- a/pipeline/dbt/macros/domain/checks/check_service.sql +++ b/pipeline/dbt/macros/domain/checks/check_service.sql @@ -19,6 +19,7 @@ CREATE OR REPLACE FUNCTION LIST_SERVICE_ERRORS( modes_orientation_beneficiaire TEXT[], modes_orientation_beneficiaire_autres TEXT, nom TEXT, + page_web TEXT, presentation_detail TEXT, presentation_resume TEXT, prise_rdv TEXT, @@ -105,6 +106,7 @@ WITH final AS ( modes_orientation_beneficiaire, modes_orientation_beneficiaire_autres, nom, + page_web, presentation_detail, presentation_resume, prise_rdv, diff --git a/pipeline/dbt/models/_sources.yml b/pipeline/dbt/models/_sources.yml index 1ddfa0c8..0c7b1039 100644 --- a/pipeline/dbt/models/_sources.yml +++ b/pipeline/dbt/models/_sources.yml @@ -141,3 +141,9 @@ sources: schema: fredo tables: - name: structures + + - name: action_logement + schema: action_logement + tables: + - name: structures + - name: services diff --git a/pipeline/dbt/models/intermediate/int__union_adresses.sql b/pipeline/dbt/models/intermediate/int__union_adresses.sql index 21fde4d2..6615aebc 100644 --- a/pipeline/dbt/models/intermediate/int__union_adresses.sql +++ b/pipeline/dbt/models/intermediate/int__union_adresses.sql @@ -2,6 +2,7 @@ WITH adresses AS ( {{ dbt_utils.union_relations( relations=[ + ref('int_action_logement__adresses'), ref('int_agefiph__adresses'), ref('int_data_inclusion__adresses'), ref('int_dora__adresses'), diff --git a/pipeline/dbt/models/intermediate/int__union_services.sql b/pipeline/dbt/models/intermediate/int__union_services.sql index 76b67ba2..582bd196 100644 --- a/pipeline/dbt/models/intermediate/int__union_services.sql +++ b/pipeline/dbt/models/intermediate/int__union_services.sql @@ -2,6 +2,7 @@ WITH services AS ( {{ dbt_utils.union_relations( relations=[ + ref('int_action_logement__services'), ref('int_agefiph__services'), ref('int_cd72__services'), ref('int_data_inclusion__services'), diff --git a/pipeline/dbt/models/intermediate/int__union_services__enhanced.sql b/pipeline/dbt/models/intermediate/int__union_services__enhanced.sql index 74c0c4c4..07672221 100644 --- a/pipeline/dbt/models/intermediate/int__union_services__enhanced.sql +++ b/pipeline/dbt/models/intermediate/int__union_services__enhanced.sql @@ -20,12 +20,12 @@ services_with_zone_diffusion AS ( {{ dbt_utils.star(from=ref('int__union_services'), relation_alias='services', except=["zone_diffusion_code", "zone_diffusion_nom"]) }}, CASE WHEN services.source = ANY(ARRAY['monenfant', 'soliguide']) THEN adresses.result_citycode - WHEN services.source = 'reseau-alpha' THEN LEFT(adresses.result_citycode, 2) + WHEN services.source = ANY(ARRAY['reseau-alpha', 'action-logement']) THEN LEFT(adresses.result_citycode, 2) ELSE services.zone_diffusion_code END AS "zone_diffusion_code", CASE WHEN services.source = ANY(ARRAY['monenfant', 'soliguide']) THEN adresses.commune - WHEN services.source = 'reseau-alpha' THEN (SELECT departements."LIBELLE" FROM departements WHERE departements."DEP" = LEFT(adresses.result_citycode, 2)) + WHEN services.source = ANY(ARRAY['reseau-alpha', 'action-logement']) THEN (SELECT departements."LIBELLE" FROM departements WHERE departements."DEP" = LEFT(adresses.result_citycode, 2)) WHEN services.source = 'mediation-numerique' THEN (SELECT departements."LIBELLE" FROM departements WHERE departements."DEP" = services.zone_diffusion_code) ELSE services.zone_diffusion_nom END AS "zone_diffusion_nom" @@ -63,6 +63,7 @@ valid_services AS ( modes_orientation_beneficiaire, modes_orientation_beneficiaire_autres, nom, + page_web, presentation_detail, presentation_resume, prise_rdv, diff --git a/pipeline/dbt/models/intermediate/int__union_structures.sql b/pipeline/dbt/models/intermediate/int__union_structures.sql index 63e4d4f6..0f7a63bd 100644 --- a/pipeline/dbt/models/intermediate/int__union_structures.sql +++ b/pipeline/dbt/models/intermediate/int__union_structures.sql @@ -2,6 +2,7 @@ WITH structures AS ( {{ dbt_utils.union_relations( relations=[ + ref('int_action_logement__structures'), ref('int_agefiph__structures'), ref('int_data_inclusion__structures'), ref('int_dora__structures'), diff --git a/pipeline/dbt/models/intermediate/sources/action_logement/_action_logement__models.yml b/pipeline/dbt/models/intermediate/sources/action_logement/_action_logement__models.yml new file mode 100644 index 00000000..0c09a776 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/action_logement/_action_logement__models.yml @@ -0,0 +1,56 @@ +version: 2 + +models: + - name: int_action_logement__adresses + tests: + - check_adresse: + config: + severity: warn + columns: + - name: id + tests: + - unique + - not_null + - dbt_utils.not_empty_string + + - name: int_action_logement__services + tests: + - check_service: + config: + severity: warn + columns: + - name: id + tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: structure_id + tests: + - not_null + - relationships: + to: ref('int_action_logement__structures') + field: id + - name: adresse_id + tests: + - not_null + - relationships: + to: ref('int_action_logement__adresses') + field: id + + - name: int_action_logement__structures + tests: + - check_structure: + config: + severity: warn + columns: + - name: id + tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: adresse_id + tests: + - not_null + - relationships: + to: ref('int_action_logement__adresses') + field: id diff --git a/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__adresses.sql b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__adresses.sql new file mode 100644 index 00000000..594b86a2 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__adresses.sql @@ -0,0 +1,19 @@ +WITH structures AS ( + SELECT * FROM {{ ref('stg_action_logement__structures') }} +), + +final AS ( + SELECT + id AS "id", + longitude AS "longitude", + latitude AS "latitude", + _di_source_id AS "source", + complement_adresse AS "complement_adresse", + commune AS "commune", + adresse AS "adresse", + code_postal AS "code_postal", + code_insee AS "code_insee" + FROM structures +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__services.sql b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__services.sql new file mode 100644 index 00000000..f3e42831 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__services.sql @@ -0,0 +1,50 @@ +WITH services AS ( + SELECT * FROM {{ ref('stg_action_logement__services') }} +), + +structures AS ( + SELECT * FROM {{ ref('int_action_logement__structures') }} +), + +final AS ( + SELECT + services._di_source_id AS "source", + structures.id AS "structure_id", + structures.id AS "adresse_id", + structures.courriel AS "courriel", + services.cumulable AS "cumulable", + TRUE AS "contact_public", + NULL AS "contact_nom_prenom", + structures.date_maj AS "date_maj", + structures.date_maj AS "date_creation", + services.formulaire_en_ligne AS "formulaire_en_ligne", + services.frais_autres AS "frais_autres", + services.justificatifs AS "justificatifs", + NULL AS "lien_source", + services.modes_accueil AS "modes_accueil", + services.modes_orientation_accompagnateur AS "modes_orientation_accompagnateur", + services.modes_orientation_accompagnateur_autres AS "modes_orientation_accompagnateur_autres", + services.modes_orientation_beneficiaire AS "modes_orientation_beneficiaire", + services.modes_orientation_beneficiaire_autres AS "modes_orientation_beneficiaire_autres", + services.nom AS "nom", + services.page_web AS "page_web", + services.presentation_detail AS "presentation_detail", + services.presentation_resume AS "presentation_resume", + services.prise_rdv AS "prise_rdv", + services.profils AS "profils", + services.pre_requis AS "pre_requis", + services.recurrence AS "recurrence", + services.thematiques AS "thematiques", + services.types AS "types", + structures.telephone AS "telephone", + services.frais AS "frais", + 'departement' AS "zone_diffusion_type", + NULL AS "zone_diffusion_code", + NULL AS "zone_diffusion_nom", + CAST(NULL AS DATE) AS "date_suspension", + structures.id || '-' || services.id AS "id" + FROM services + CROSS JOIN structures +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__structures.sql b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__structures.sql new file mode 100644 index 00000000..3798a5fe --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__structures.sql @@ -0,0 +1,30 @@ +WITH structures AS ( + SELECT * FROM {{ ref('stg_action_logement__structures') }} +), + +final AS ( + SELECT + accessibilite AS "accessibilite", + id AS "adresse_id", + antenne AS "antenne", + courriel AS "courriel", + horaires_ouverture AS "horaires_ouverture", + id AS "id", + labels_autres AS "labels_autres", + labels_nationaux AS "labels_nationaux", + NULL AS "lien_source", + nom AS "nom", + presentation_detail AS "presentation_detail", + presentation_resume AS "presentation_resume", + rna AS "rna", + siret AS "siret", + site_web AS "site_web", + _di_source_id AS "source", + telephone AS "telephone", + typologie AS "typologie", + CAST(date_maj AS DATE) AS "date_maj", + CAST(NULL AS TEXT []) AS "thematiques" + FROM structures +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql b/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql index 3483784c..c3eaaa77 100644 --- a/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql +++ b/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql @@ -70,6 +70,7 @@ final AS ( NULL AS "zone_diffusion_nom", 'pays' AS "zone_diffusion_type", services.id AS "id", + NULL AS "page_web", 'https://www.agefiph.fr' || services.attributes__path__alias AS "modes_orientation_accompagnateur_autres", 'https://www.agefiph.fr' || services.attributes__path__alias AS "modes_orientation_beneficiaire_autres", CAST(NULL AS TEXT []) AS "justificatifs", diff --git a/pipeline/dbt/models/intermediate/sources/cd72/int_cd72__services.sql b/pipeline/dbt/models/intermediate/sources/cd72/int_cd72__services.sql index f5903214..38e03855 100644 --- a/pipeline/dbt/models/intermediate/sources/cd72/int_cd72__services.sql +++ b/pipeline/dbt/models/intermediate/sources/cd72/int_cd72__services.sql @@ -30,6 +30,7 @@ final AS ( zone_diffusion_code AS "zone_diffusion_code", NULL AS "zone_diffusion_nom", zone_diffusion_type AS "zone_diffusion_type", + NULL AS "page_web", CAST(NULL AS BOOLEAN) AS "cumulable", CAST(NULL AS TEXT []) AS "justificatifs", CAST(NULL AS TEXT []) AS "modes_accueil", diff --git a/pipeline/dbt/models/intermediate/sources/data_inclusion/int_data_inclusion__services.sql b/pipeline/dbt/models/intermediate/sources/data_inclusion/int_data_inclusion__services.sql index 5893bb3c..52050675 100644 --- a/pipeline/dbt/models/intermediate/sources/data_inclusion/int_data_inclusion__services.sql +++ b/pipeline/dbt/models/intermediate/sources/data_inclusion/int_data_inclusion__services.sql @@ -53,7 +53,8 @@ final AS ( zone_diffusion_nom AS "zone_diffusion_nom", zone_diffusion_type AS "zone_diffusion_type", pre_requis AS "pre_requis", - frais AS "frais" + frais AS "frais", + NULL AS "page_web" FROM services ) diff --git a/pipeline/dbt/models/intermediate/sources/dora/int_dora__services.sql b/pipeline/dbt/models/intermediate/sources/dora/int_dora__services.sql index 5a5ac480..9abe20f8 100644 --- a/pipeline/dbt/models/intermediate/sources/dora/int_dora__services.sql +++ b/pipeline/dbt/models/intermediate/sources/dora/int_dora__services.sql @@ -65,6 +65,7 @@ final AS ( services.zone_diffusion_nom AS "zone_diffusion_nom", services.zone_diffusion_type AS "zone_diffusion_type", services.pre_requis AS "pre_requis", + NULL AS "page_web", CASE WHEN blocked_services_uids.id IS NULL THEN services.contact_nom_prenom diff --git a/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql b/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql index 4a908726..5bef02ad 100644 --- a/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql +++ b/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql @@ -53,6 +53,7 @@ final AS ( 'commune' AS "zone_diffusion_type", structures.code_insee AS "zone_diffusion_code", structures.commune AS "zone_diffusion_nom", + NULL AS "page_web", CAST(NULL AS DATE) AS "date_suspension", structures.id || '-' || services.id AS "id" FROM services diff --git a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql index 86529f87..d5fc65be 100644 --- a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql +++ b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql @@ -64,7 +64,8 @@ final AS ( CASE WHEN CARDINALITY(services.types) > 0 THEN services.types ELSE ARRAY['accompagnement'] END AS "types", ARRAY['en-presentiel'] AS "modes_accueil", {{ truncate_text(presentation) }} AS "presentation_resume", - {{ presentation }} AS "presentation_detail" + {{ presentation }} AS "presentation_detail", + NULL AS "page_web" FROM services LEFT JOIN structures ON services.structure_id = structures.id ) diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql index d67eec6b..bf2d7336 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql @@ -92,7 +92,8 @@ final AS ( NULL AS "contact_nom_prenom", garages.email AS "courriel", garages.telephone AS "telephone", - CAST(NULL AS TEXT []) AS "frais" + CAST(NULL AS TEXT []) AS "frais", + NULL AS "page_web" FROM garages LEFT JOIN service_types_by_garage ON garages.id = service_types_by_garage.id WHERE diff --git a/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql b/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql index f0db3f5b..ed777f7b 100644 --- a/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql +++ b/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql @@ -63,7 +63,8 @@ final AS ( CASE WHEN details__modalite__modalites_inscription IS NOT NULL THEN '## Les modalités d''inscription :' || E'\n\n' || details__modalite__modalites_inscription END ], E'\n\n' - ) AS "presentation_detail" + ) AS "presentation_detail", + NULL AS "page_web" FROM creches ) diff --git a/pipeline/dbt/models/intermediate/sources/odspep/int_odspep__services.sql b/pipeline/dbt/models/intermediate/sources/odspep/int_odspep__services.sql index eed98ae4..0483b25e 100644 --- a/pipeline/dbt/models/intermediate/sources/odspep/int_odspep__services.sql +++ b/pipeline/dbt/models/intermediate/sources/odspep/int_odspep__services.sql @@ -60,7 +60,8 @@ final AS ( CASE WHEN LENGTH(service_description) <= 280 THEN NULL ELSE service_description - END AS "presentation_detail" + END AS "presentation_detail", + NULL AS "page_web" FROM ressources_partenariales ORDER BY 1 diff --git a/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql b/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql index e94a2b76..68e0712b 100644 --- a/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql +++ b/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql @@ -28,6 +28,7 @@ final AS ( formations.content__date_maj AS "date_maj", NULL AS "modes_orientation_accompagnateur_autres", NULL AS "modes_orientation_beneficiaire_autres", + NULL AS "page_web", CASE WHEN LENGTH(formations.content__contenu_et_objectifs__titre) <= 280 THEN formations.content__contenu_et_objectifs__titre diff --git a/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql b/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql index 4a6de0ac..2ddf8841 100644 --- a/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql +++ b/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql @@ -186,7 +186,8 @@ final AS ( CASE WHEN open_services.modalities__inscription__checked THEN '## Sur inscription :' || E'\n' || open_services.modalities__inscription__precisions END ], E'\n\n' - ) AS "modes_orientation_beneficiaire_autres" + ) AS "modes_orientation_beneficiaire_autres", + NULL AS "page_web" FROM open_services LEFT JOIN lieux ON open_services.lieu_id = lieux.id LEFT JOIN categories ON open_services.category = categories.code diff --git a/pipeline/dbt/models/marts/inclusion/_inclusion_models.yml b/pipeline/dbt/models/marts/inclusion/_inclusion_models.yml index 381baa39..acf5c4af 100644 --- a/pipeline/dbt/models/marts/inclusion/_inclusion_models.yml +++ b/pipeline/dbt/models/marts/inclusion/_inclusion_models.yml @@ -163,6 +163,8 @@ models: data_type: text constraints: - type: not_null + - name: page_web + data_type: text - name: presentation_resume data_type: text - name: presentation_detail diff --git a/pipeline/dbt/models/staging/sources/action_logement/_action_logement__models.yml b/pipeline/dbt/models/staging/sources/action_logement/_action_logement__models.yml new file mode 100644 index 00000000..c06853be --- /dev/null +++ b/pipeline/dbt/models/staging/sources/action_logement/_action_logement__models.yml @@ -0,0 +1,18 @@ +version: 2 + +models: + - name: stg_action_logement__services + columns: + - name: id + tests: + - unique + - not_null + - dbt_utils.not_empty_string + + - name: stg_action_logement__structures + columns: + - name: id + tests: + - unique + - not_null + - dbt_utils.not_empty_string diff --git a/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__services.sql b/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__services.sql new file mode 100644 index 00000000..7b1decb8 --- /dev/null +++ b/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__services.sql @@ -0,0 +1,39 @@ +WITH source AS ( + {{ stg_source_header('action_logement', 'services') }} +), + +final AS ( + SELECT + _di_source_id AS "_di_source_id", + TRUE AS "cumulable", + data ->> 'id' AS "id", + CURRENT_DATE AT TIME ZONE 'Europe/Paris' AS "date_maj", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'modes_accueil.\d+'), NULL) AS "modes_accueil", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'profils.\d+'), NULL) AS "profils", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'thematiques.\d+'), NULL) AS "thematiques", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'types.\d+'), NULL) AS "types", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'justificatifs.\d+'), NULL) AS "justificatifs", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'pre_requis.\d+'), NULL) AS "pre_requis", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'modes_orientation_accompagnateur.\d+'), NULL) AS "modes_orientation_accompagnateur", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'modes_orientation_beneficiaire.\d+'), NULL) AS "modes_orientation_beneficiaire", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'frais.\d+'), NULL) AS "frais", + data ->> 'modes_orientation_accompagnateur_autres' AS "modes_orientation_accompagnateur_autres", + data ->> 'modes_orientation_beneficiaire_autres' AS "modes_orientation_beneficiaire_autres", + data ->> 'formulaire_en_ligne' AS "formulaire_en_ligne", + data ->> 'frais_autres' AS "frais_autres", + data ->> 'nom' AS "nom", + data ->> 'page_web' AS "page_web", + data ->> 'presentation_resume' AS "presentation_resume", + data ->> 'presentation_detail' AS "presentation_detail", + data ->> 'prise_rdv' AS "prise_rdv", + data ->> 'recurrence' AS "recurrence", + NULLIF(TRIM(data ->> 'zone_diffusion_code'), '') AS "zone_diffusion_code", + NULLIF(TRIM(data ->> 'zone_diffusion_nom'), '') AS "zone_diffusion_nom", + data ->> 'zone_diffusion_type' AS "zone_diffusion_type", + data ->> 'lien_source' AS "lien_source" + FROM source + WHERE + NOT COALESCE(CAST(data ->> '__ignore__' AS BOOLEAN), FALSE) +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__structures.sql b/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__structures.sql new file mode 100644 index 00000000..470c9f45 --- /dev/null +++ b/pipeline/dbt/models/staging/sources/action_logement/stg_action_logement__structures.sql @@ -0,0 +1,38 @@ +WITH source AS ( + {{ stg_source_header('action_logement', 'structures') }} +), + +final AS ( + SELECT + _di_source_id AS "_di_source_id", + CAST(data ->> 'antenne' AS BOOLEAN) AS "antenne", + TO_DATE(data ->> 'date_maj', 'DD/MM/YYYY') AS "date_maj", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'labels_autres.\d+'), NULL) AS "labels_autres", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'labels_nationaux.\d+'), NULL) AS "labels_nationaux", + CAST(data ->> 'latitude' AS FLOAT) AS "latitude", + CAST(data ->> 'longitude' AS FLOAT) AS "longitude", + ARRAY_REMOVE(ARRAY(SELECT value FROM JSONB_EACH_TEXT(data) WHERE key ~* 'thematiques.\d+'), NULL) AS "thematiques", + data ->> 'accessibilite' AS "accessibilite", + data ->> 'adresse' AS "adresse", + data ->> 'code_insee' AS "code_insee", + data ->> 'code_postal' AS "code_postal", + data ->> 'commune' AS "commune", + data ->> 'complement_adresse' AS "complement_adresse", + data ->> 'courriel' AS "courriel", + data ->> 'horaires_ouverture' AS "horaires_ouverture", + data ->> 'id' AS "id", + data ->> 'nom' AS "nom", + data ->> 'presentation_detail' AS "presentation_detail", + data ->> 'presentation_resume' AS "presentation_resume", + data ->> 'rna' AS "rna", + data ->> 'siret' AS "siret", + data ->> 'site_web' AS "site_web", + data ->> 'source' AS "source", + data ->> 'telephone' AS "telephone", + data ->> 'typologie' AS "typologie" + FROM source + WHERE + NOT COALESCE(CAST(data ->> '__ignore__' AS BOOLEAN), FALSE) +) + +SELECT * FROM final diff --git a/pipeline/defaults.env b/pipeline/defaults.env index c2e7d458..fd48c81c 100644 --- a/pipeline/defaults.env +++ b/pipeline/defaults.env @@ -6,6 +6,8 @@ AIRFLOW_VAR_BAN_API_URL=https://api-adresse.data.gouv.fr AIRFLOW_VAR_CD35_API_URL=https://services1.arcgis.com/jGLANYlFVVx3nuxa/arcgis/rest/services/annuaire_social_CD35/FeatureServer/0/query AIRFLOW_VAR_CD72_STRUCTURES_FILE_URL=https://grist.incubateur.net/o/datainclusion/api/docs/dFpXXzs2fug9Kb7zZhyWyn/download/csv?tableId=Structures AIRFLOW_VAR_CD72_SERVICES_FILE_URL=https://grist.incubateur.net/o/datainclusion/api/docs/dFpXXzs2fug9Kb7zZhyWyn/download/csv?tableId=Services +AIRFLOW_VAR_ACTION_LOGEMENT_SERVICES_URL=https://docs.google.com/spreadsheets/d/1aiZrTeBKaOwjeeJft-gsEerAOQ9miZB-MUCE_unjCGM/export?format=csv&gid=636818394 +AIRFLOW_VAR_ACTION_LOGEMENT_STRUCTURES_URL=https://docs.google.com/spreadsheets/d/1aiZrTeBKaOwjeeJft-gsEerAOQ9miZB-MUCE_unjCGM/export?format=csv&gid=1318485024 AIRFLOW_VAR_DI_EXTRA_SERVICES_FILE_URL=https://docs.google.com/spreadsheets/d/1vAuD_XTMOEmtH9li3hL1iydyp3UVLVb_DaR5nmE5kFY/export?format=csv&gid=0 AIRFLOW_VAR_DI_EXTRA_STRUCTURES_FILE_URL=https://docs.google.com/spreadsheets/d/1vAuD_XTMOEmtH9li3hL1iydyp3UVLVb_DaR5nmE5kFY/export?format=csv&gid=943548723 AIRFLOW_VAR_DORA_API_URL=https://api.dora.inclusion.beta.gouv.fr/api/v2/