Skip to content

Commit

Permalink
Always write to BQ from global window
Browse files Browse the repository at this point in the history
  • Loading branch information
damccorm committed Sep 27, 2024
1 parent c2c640f commit d674a8e
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 4 deletions.
2 changes: 1 addition & 1 deletion .github/trigger_files/beam_PostCommit_Python.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run.",
"modification": 2
"modification": 3
}

Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run"
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
3 changes: 3 additions & 0 deletions sdks/python/apache_beam/io/gcp/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -1864,6 +1864,9 @@ def _restore_table_ref(sharded_table_ref_elems_kv):
return (
tagged_data
| 'FromHashableTableRef' >> beam.Map(_restore_table_ref)
# Use global window for writes since we're outputting back into the
# global window.
| 'Window into Global Window' >> beam.WindowInto(GlobalWindows())
| 'StreamInsertRows' >> ParDo(
bigquery_write_fn, *self.schema_side_inputs).with_outputs(
BigQueryWriteFn.FAILED_ROWS,
Expand Down
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/io/gcp/bigquery_write_it_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ def test_big_query_write_insert_non_transient_api_call_error(self):
input_data = [{
'number': 1,
'str': 'some_string',
}]
}]*500

table_schema = {
"fields": [{
Expand All @@ -483,7 +483,7 @@ def test_big_query_write_insert_non_transient_api_call_error(self):
bq_result_errors = [({
'number': 1,
'str': 'some_string',
}, "Not Found")]
}, "Not Found")]*500

args = self.test_pipeline.get_full_options_as_args()

Expand Down

0 comments on commit d674a8e

Please sign in to comment.