From 1a029ee642a5dc83ef8c43613ecc5cdc44eefdbd Mon Sep 17 00:00:00 2001 From: RuslanBergenov Date: Thu, 15 Feb 2024 17:15:16 -0700 Subject: [PATCH 1/5] fix: logging and requirement errors --- cf/requirements.txt | 3 ++- cfintraday/main.py | 11 +++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cf/requirements.txt b/cf/requirements.txt index 46aa9c2..374cffb 100644 --- a/cf/requirements.txt +++ b/cf/requirements.txt @@ -4,4 +4,5 @@ google-cloud-bigquery~=3.13.0 google-cloud-pubsub~=2.2.0 google-cloud-bigquery-storage~=2.10.1 pytz~=2022.7.1 -db_dtypes~=1.1.1 \ No newline at end of file +db_dtypes~=1.1.1 +flask<3.0,>=1.0 \ No newline at end of file diff --git a/cfintraday/main.py b/cfintraday/main.py index c55ab2d..03092aa 100644 --- a/cfintraday/main.py +++ b/cfintraday/main.py @@ -186,12 +186,11 @@ def manage_intraday_schedule(event, context="context"): # Construct the request body. cron_schedule = input_event.intraday_schedule() - MESSAGE_DATA = {"protoPayload": { - "serviceData": {"jobCompletedEvent": {"job": {"jobConfiguration": {"load": {"destinationTable": { - "datasetId": input_event.dataset - , "projectId": input_event.gcp_project - , "tableId": f"events_intraday_{input_event.table_date_shard}" - }}}}}}}} + MESSAGE_DATA = { + "protoPayload": { + "resourceName": f"projects/{input_event.gcp_project}/datasets/{input_event.dataset}/tables/events_intraday_{input_event.table_date_shard}" + } + } topic_name = os.environ["TOPIC_NAME"] From 6e0e6293f500f669eda6f7efef8051b0d992e5d5 Mon Sep 17 00:00:00 2001 From: RuslanBergenov Date: Thu, 15 Feb 2024 17:16:38 -0700 Subject: [PATCH 2/5] docs: API enabled --- .github/workflows/python-package.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c4ba264..9f01a89 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,7 +10,7 @@ env: on: push: - branches: [ master ] + branches: [ master, development ] pull_request: branches: [ master ] diff --git a/README.md b/README.md index 08850f7..cb4835d 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ The GCP resources for the solutions are installed via Deployment Manager. * Cloud Build API * Cloud Deployment Manager V2 API * Cloud Functions API + * Artifact Registry API * Identity and Access Management (IAM) API * Cloud Scheduler API (if you need to flatten intraday tables) From 8eeb43a03d4f2634a45778c5eafed20b95df3b6c Mon Sep 17 00:00:00 2001 From: RuslanBergenov Date: Thu, 15 Feb 2024 17:29:05 -0700 Subject: [PATCH 3/5] build: CI/CD --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9f01a89..17bcf1a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,9 +10,9 @@ env: on: push: - branches: [ master, development ] + branches: [ staging ] pull_request: - branches: [ master ] + branches: [ staging ] jobs: build: From b4207d2ecee538f3319e86778d935d941001aa03 Mon Sep 17 00:00:00 2001 From: RuslanBergenov Date: Thu, 15 Feb 2024 17:30:08 -0700 Subject: [PATCH 4/5] Revert "build: CI/CD" This reverts commit 8eeb43a03d4f2634a45778c5eafed20b95df3b6c. --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 17bcf1a..9f01a89 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,9 +10,9 @@ env: on: push: - branches: [ staging ] + branches: [ master, development ] pull_request: - branches: [ staging ] + branches: [ master ] jobs: build: From e38cdc4d3a2eb709d282d350280d241b6ec95692 Mon Sep 17 00:00:00 2001 From: RuslanBergenov Date: Thu, 15 Feb 2024 17:47:26 -0700 Subject: [PATCH 5/5] test: logging fix --- .github/workflows/python-package.yml | 2 +- tests/test_manage_intraday_schedule.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9f01a89..9120315 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,7 +10,7 @@ env: on: push: - branches: [ master, development ] + branches: [ staging ] pull_request: branches: [ master ] diff --git a/tests/test_manage_intraday_schedule.py b/tests/test_manage_intraday_schedule.py index d2a8336..f854330 100644 --- a/tests/test_manage_intraday_schedule.py +++ b/tests/test_manage_intraday_schedule.py @@ -200,14 +200,9 @@ def test_create_intraday_flattening_schedule_minutes(self, logcapture): }) self.assertEqual(job_id_full_path, response_get_job.name) self.assertEqual('*/30 * * * *', response_get_job.schedule) - # I refactored %s to be an f-string. In case with a stringified dictionary, f-strings actully became less readable and maintainable - # While using a dict with an f-string, we need to duplicate { for this to appear in the string, we also have to escape the quotes - # https://realpython.com/python-f-strings/#python-f-strings-the-pesky-details - # For now, I decided to keep both % and f-string - expected_percent_string = 'b\'{"protoPayload": {"serviceData": {"jobCompletedEvent": {"job": {"jobConfiguration": {"load": {"destinationTable": {"datasetId": "%s", "projectId": "%s", "tableId": "events_intraday_%s"}}}}}}}}\'' % (self.dataset_id, self.project_id, self.date_shard) - expected_f_string = f"b\'{{\"protoPayload\": {{\"serviceData\": {{\"jobCompletedEvent\": {{\"job\": {{\"jobConfiguration\": {{\"load\": {{\"destinationTable\": {{\"datasetId\": \"{self.dataset_id}\", \"projectId\": \"{self.project_id}\", \"tableId\": \"events_intraday_{self.date_shard}\"}}}}}}}}}}}}}}}}\'" + expected = f"b\'{{\"protoPayload\": {{\"resourceName\": \"projects/{self.project_id}/datasets/{self.dataset_id}/tables/events_intraday_{self.date_shard}\"}}}}\'" actual = str(response_get_job.pubsub_target.data) - assert expected_percent_string == expected_f_string == actual + assert expected == actual # check log expected_log = ('root', 'INFO',