remove-totally-siao

gip-inclusion · Nov 6, 2024 · 1fdec5b · 1fdec5b
1 parent 8ec0807
commit 1fdec5b
Show file tree

Hide file tree

Showing 9 changed files with 0 additions and 51 deletions.
diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml
@@ -37,7 +37,6 @@ x-airflow-common:
     AIRFLOW_VAR_FREDO_API_TOKEN: ${AIRFLOW_VAR_FREDO_API_TOKEN}
     AIRFLOW_VAR_FT_API_TOKEN: ${AIRFLOW_VAR_FT_API_TOKEN}
     AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY: ${AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY}
-    AIRFLOW_VAR_SIAO_FILE_URL: ${AIRFLOW_VAR_SIAO_FILE_URL}
     AIRFLOW_VAR_SOLIGUIDE_API_TOKEN: ${AIRFLOW_VAR_SOLIGUIDE_API_TOKEN}
     AIRFLOW_VAR_TWOCAPTCHA_API_KEY: ${AIRFLOW_VAR_TWOCAPTCHA_API_KEY}
 

diff --git a/deployment/main.tf b/deployment/main.tf
@@ -248,7 +248,6 @@ resource "null_resource" "up" {
     AIRFLOW_VAR_FREDO_API_TOKEN='${var.fredo_api_token}'
     AIRFLOW_VAR_FT_API_TOKEN='${var.ft_api_token}'
     AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY='${var.mes_aides_airtable_key}'
-    AIRFLOW_VAR_SIAO_FILE_URL='${var.siao_file_url}'
     AIRFLOW_VAR_SOLIGUIDE_API_TOKEN='${var.soliguide_api_token}'
     AIRFLOW_VAR_TWOCAPTCHA_API_KEY='${var.twocaptcha_api_key}'
     EOT

diff --git a/deployment/template.terraform.tfvars.json b/deployment/template.terraform.tfvars.json
@@ -27,7 +27,6 @@
     "scaleway_application_id": null,
     "scaleway_project_id": null,
     "scaleway_secret_key": null,
-    "siao_file_url": null,
     "soliguide_api_token": null,
     "ssh_private_key": null,
     "stack_version": null

diff --git a/deployment/variables.tf b/deployment/variables.tf
@@ -196,12 +196,6 @@ variable "airflow_conn_mattermost" {
   default     = ""
 }
 
-variable "siao_file_url" {
-  description = "Public URL to the siao export on our s3 bucket"
-  type        = string
-  default     = ""
-}
-
 variable "twocaptcha_api_key" {
   description = "Used in extraction tasks orchestrated by airflow"
   type        = string

diff --git a/pipeline/dags/dag_utils/sources/__init__.py b/pipeline/dags/dag_utils/sources/__init__.py
@@ -56,17 +56,6 @@
             },
         },
     },
-    "siao": {
-        "schedule": "@once",
-        "snapshot": False,
-        "reader": utils.read_excel,
-        "streams": {
-            "etablissements": {
-                "filename": "etablissements.xlsx",
-                "url": Variable.get("SIAO_FILE_URL", None),
-            },
-        },
-    },
     "finess": {
         "schedule": "@daily",
         "snapshot": True,

diff --git a/pipeline/dbt/models/_sources.yml b/pipeline/dbt/models/_sources.yml
@@ -39,11 +39,6 @@ sources:
     tables:
       - name: etablissements
 
-  - name: siao
-    schema: siao
-    tables:
-      - name: etablissements
-
   - name: un_jeune_une_solution
     schema: un_jeune_une_solution
     tables:

diff --git a/pipeline/dbt/models/staging/sources/siao/_siao__models.yml b/pipeline/dbt/models/staging/sources/siao/_siao__models.yml
diff --git a/pipeline/dbt/models/staging/sources/siao/stg_siao__etablissements.sql b/pipeline/dbt/models/staging/sources/siao/stg_siao__etablissements.sql
diff --git a/pipeline/defaults.env b/pipeline/defaults.env
@@ -21,7 +21,6 @@ AIRFLOW_VAR_MES_AIDES_PERMIS_VELO_URL=https://airtable.com/appRga7C9USklxYiV/tbl
 AIRFLOW_VAR_MONENFANT_BASE_URL=https://monenfant.fr
 AIRFLOW_VAR_ODSPEP_S3_KEY_PREFIX=sources/odspep/2023-01-23/denormalized/Exports/
 AIRFLOW_VAR_RESEAU_ALPHA_URL=https://www.reseau-alpha.org
-AIRFLOW_VAR_SIAO_FILE_URL=
 AIRFLOW_VAR_SIRENE_STOCK_ETAB_GEOCODE_FILE_URL=https://data.cquest.org/geo_sirene/v2019/last/StockEtablissementActif_utf8_geo.csv.gz
 AIRFLOW_VAR_SIRENE_STOCK_ETAB_HIST_FILE_URL=https://www.data.gouv.fr/fr/datasets/r/88fbb6b4-0320-443e-b739-b4376a012c32
 AIRFLOW_VAR_SIRENE_STOCK_ETAB_LIENS_SUCCESSION_URL=https://www.data.gouv.fr/fr/datasets/r/9c4d5d9c-4bbb-4b9c-837a-6155cb589e26