fix: Adding push to internal test tables secdond push.

GoogleCloudPlatform · Nov 30, 2023 · cabc823 · cabc823
1 parent 3702c39
commit cabc823
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/.../cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py b/.../cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py
@@ -138,7 +138,7 @@
         image_pull_policy="Always",
         image="{{ var.json.cloud_storage_geo_index.container_registry.run_csv_transform_kub }}",
         env_vars={
-            "PIPELINE_NAME": "Cloud Storage GEO Index - Sentinel 2 Index",
+            "PIPELINE_NAME": "Cloud Storage GEO Index - Sentinel 2 Index [Int_Test]",
             "SOURCE_URL": "https://storage.googleapis.com/gcp-public-data-sentinel-2/index.csv.gz",
             "SOURCE_ZIPFILE": "files/cloud_storage_geo_index-sentinel_2-data.csv.gz",
             "SOURCE_FILE": "files/cloud_storage_geo_index-sentinel_2-data.csv",

diff --git a/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml b/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml
@@ -298,15 +298,15 @@ dag:
         env_vars:
           PIPELINE_NAME: "Cloud Storage GEO Index - Sentinel 2 Index [Int_Test]"
           SOURCE_URL: "https://storage.googleapis.com/gcp-public-data-sentinel-2/index.csv.gz"
-          SOURCE_ZIPFILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data.csv.gz"
-          SOURCE_FILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data.csv"
-          TARGET_FILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data_output.csv"
+          SOURCE_ZIPFILE: "files/cloud_storage_geo_index-sentinel_2-data.csv.gz"
+          SOURCE_FILE: "files/cloud_storage_geo_index-sentinel_2-data.csv"
+          TARGET_FILE: "files/cloud_storage_geo_index-sentinel_2-data_output.csv"
           CHUNKSIZE: "1000000"
           PROJECT_ID: "{{ var.value.gcp_project }}"
           DATASET_ID: "s2_internal_test"
           TABLE_ID: "sentinel_2_index"
           TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
-          TARGET_GCS_PATH: "data/cloud_storage_geo_index/sentinel_2_index/int_test_data_output.csv"
+          TARGET_GCS_PATH: "data/cloud_storage_geo_index/sentinel_2_index/data_output.csv"
           SCHEMA_PATH: "data/cloud_storage_geo_index/schema/cloud_storage_geo_index_sentinel_2_schema.json"
           DROP_DEST_TABLE: "N"
           INPUT_FIELD_DELIMITER: ","