remove explicit -1 test; default to -1 in schematransform

apache · Sep 23, 2023 · 951ba03 · 951ba03
1 parent 1539537
commit 951ba03
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 25 deletions.
diff --git a/...c/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java b/...c/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteSchemaTransformProvider.java
@@ -179,12 +179,13 @@ public KV<ByteString, Iterable<Mutation>> apply(Row row) {
                     .setColumnQualifier(
                         ByteString.copyFrom(ofNullable(mutation.get("column_qualifier")).get()))
                     .setFamilyNameBytes(
-                        ByteString.copyFrom(ofNullable(mutation.get("family_name")).get()));
-            if (mutation.containsKey("timestamp_micros")) {
-              setMutation =
-                  setMutation.setTimestampMicros(
-                      Longs.fromByteArray(ofNullable(mutation.get("timestamp_micros")).get()));
-            }
+                        ByteString.copyFrom(ofNullable(mutation.get("family_name")).get()))
+                    // Use timestamp if provided, else default to -1 (current Bigtable server time)
+                    .setTimestampMicros(
+                        mutation.containsKey("timestamp_micros")
+                            ? Longs.fromByteArray(
+                                ofNullable(mutation.get("timestamp_micros")).get())
+                            : -1);
             bigtableMutation = Mutation.newBuilder().setSetCell(setMutation.build()).build();
             break;
           case "DeleteFromColumn":

diff --git a/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py b/sdks/python/apache_beam/io/gcp/bigtableio_it_test.py
@@ -226,7 +226,6 @@ def test_set_mutation(self):
     # When setting this cell, we won't set a timestamp. We expect the timestamp
     # to default to -1, and Bigtable will set it to system time at insertion.
     row2_col1_no_timestamp = Cell(b'val2-2-notimestamp', time.time())
-    row2_col1_neg1_timestamp = Cell(b'val2-2-neg1-timestamp', time.time())
     # rows sent to write transform
     row1.set_cell(
         'col_fam', b'col-1', row1_col1_cell.value, row1_col1_cell.timestamp)
@@ -238,8 +237,6 @@ def test_set_mutation(self):
         'col_fam', b'col-2', row2_col2_cell.value, row2_col2_cell.timestamp)
     # don't set a timestamp here. it should default to -1
     row2.set_cell('col_fam', b'col-no-timestamp', row2_col1_no_timestamp.value)
-    row2.set_cell(
-        'col_fam', b'col-neg1-timestamp', row2_col1_no_timestamp.value, -1)
 
     self.run_pipeline([row1, row2])
 
@@ -257,7 +254,7 @@ def test_set_mutation(self):
     self.assertEqual(
         row2_col2_cell, actual_row2.find_cells('col_fam', b'col-2')[0])
 
-    # check cell that doesn't have a timestamp set is handled properly:
+    # check mutation that doesn't have a timestamp set is handled properly:
     self.assertEqual(
         row2_col1_no_timestamp.value,
         actual_row2.find_cells('col_fam', b'col-no-timestamp')[0].value)
@@ -266,23 +263,9 @@ def test_set_mutation(self):
     cell_timestamp = actual_row2.find_cells('col_fam',
                                             b'col-no-timestamp')[0].timestamp
     self.assertTrue(
-        row2_col1_no_timestamp.timestamp < actual_row2.find_cells(
-            'col_fam', b'col-no-timestamp')[0].timestamp,
+        row2_col1_no_timestamp.timestamp < cell_timestamp,
         msg="Expected cell with unset timestamp to have ingestion time "
         f"attached, but was {cell_timestamp}")
-    # check cell that has timestamp of `-1` is handled properly:
-    self.assertEqual(
-        row2_col1_neg1_timestamp.value,
-        actual_row2.find_cells('col_fam', b'col-neg1-timestamp')[0].value)
-    # Bigtable sets -1 timestamp as insertion time, which is later than the
-    # time.time() we set when creating this test case
-    cell_timestamp = actual_row2.find_cells('col_fam',
-                                            b'col-neg1-timestamp')[0].timestamp
-    self.assertTrue(
-        row2_col1_neg1_timestamp.timestamp < actual_row2.find_cells(
-            'col_fam', b'col-neg1-timestamp')[0].timestamp,
-        msg="Expected cell with `-1` timestamp to have ingestion time "
-        f"attached, but was {cell_timestamp}")
 
   def test_delete_cells_mutation(self):
     col_fam = self.table.column_family('col_fam')