From cabc8230c5f24ea22514bda8c0aaa6c60aac93e6 Mon Sep 17 00:00:00 2001 From: nlarge-google Date: Thu, 30 Nov 2023 22:29:34 +0000 Subject: [PATCH] fix: Adding push to internal test tables secdond push. --- .../cloud_storage_geo_index_dag.py | 2 +- .../pipelines/cloud_storage_geo_index/pipeline.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py b/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py index 3d4654133..75c8b5499 100644 --- a/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py +++ b/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/cloud_storage_geo_index_dag.py @@ -138,7 +138,7 @@ image_pull_policy="Always", image="{{ var.json.cloud_storage_geo_index.container_registry.run_csv_transform_kub }}", env_vars={ - "PIPELINE_NAME": "Cloud Storage GEO Index - Sentinel 2 Index", + "PIPELINE_NAME": "Cloud Storage GEO Index - Sentinel 2 Index [Int_Test]", "SOURCE_URL": "https://storage.googleapis.com/gcp-public-data-sentinel-2/index.csv.gz", "SOURCE_ZIPFILE": "files/cloud_storage_geo_index-sentinel_2-data.csv.gz", "SOURCE_FILE": "files/cloud_storage_geo_index-sentinel_2-data.csv", diff --git a/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml b/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml index 59605183d..f85a3a579 100644 --- a/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml +++ b/datasets/cloud_storage_geo_index/pipelines/cloud_storage_geo_index/pipeline.yaml @@ -298,15 +298,15 @@ dag: env_vars: PIPELINE_NAME: "Cloud Storage GEO Index - Sentinel 2 Index [Int_Test]" SOURCE_URL: "https://storage.googleapis.com/gcp-public-data-sentinel-2/index.csv.gz" - SOURCE_ZIPFILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data.csv.gz" - SOURCE_FILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data.csv" - TARGET_FILE: "files/cloud_storage_geo_index-sentinel_2_int_test-data_output.csv" + SOURCE_ZIPFILE: "files/cloud_storage_geo_index-sentinel_2-data.csv.gz" + SOURCE_FILE: "files/cloud_storage_geo_index-sentinel_2-data.csv" + TARGET_FILE: "files/cloud_storage_geo_index-sentinel_2-data_output.csv" CHUNKSIZE: "1000000" PROJECT_ID: "{{ var.value.gcp_project }}" DATASET_ID: "s2_internal_test" TABLE_ID: "sentinel_2_index" TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" - TARGET_GCS_PATH: "data/cloud_storage_geo_index/sentinel_2_index/int_test_data_output.csv" + TARGET_GCS_PATH: "data/cloud_storage_geo_index/sentinel_2_index/data_output.csv" SCHEMA_PATH: "data/cloud_storage_geo_index/schema/cloud_storage_geo_index_sentinel_2_schema.json" DROP_DEST_TABLE: "N" INPUT_FIELD_DELIMITER: ","