From 8ae7215d748c907d17aca33d3a8f3102a0276e72 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Sun, 3 Sep 2023 18:47:37 -0400 Subject: [PATCH 01/12] Create HealthcareUtils file with shared resources --- .../beam/sdk/io/gcp/healthcare/FhirIO.java | 22 +++--------- .../io/gcp/healthcare/HealthcareUtils.java | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 17 deletions(-) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java index 4fdd3513c806..80ecff0f7818 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java @@ -51,6 +51,7 @@ import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; +import org.apache.beam.sdk.io.gcp.healthcare.HealthcareUtils; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.MatchResult.Status; @@ -1761,19 +1762,6 @@ public PCollection expand(PBegin input) { /** A function that schedules a deidentify operation and monitors the status. */ public static class DeidentifyFn extends DoFn { - private static final Counter DEIDENTIFY_OPERATION_SUCCESS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_success_count"); - private static final Counter DEIDENTIFY_OPERATION_ERRORS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_failure_count"); - private static final Counter RESOURCES_DEIDENTIFIED_SUCCESS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_success_count"); - private static final Counter RESOURCES_DEIDENTIFIED_ERRORS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_failure_count"); - private HealthcareApiClient client; private final ValueProvider destinationFhirStore; private static final Gson gson = new Gson(); @@ -1800,10 +1788,10 @@ public void deidentify(ProcessContext context) throws IOException, InterruptedEx operation = client.pollOperation(operation, 15000L); incrementLroCounters( operation, - DEIDENTIFY_OPERATION_SUCCESS, - DEIDENTIFY_OPERATION_ERRORS, - RESOURCES_DEIDENTIFIED_SUCCESS, - RESOURCES_DEIDENTIFIED_ERRORS); + HealthcareUtils.deidentifyLroCounters.DEIDENTIFY_OPERATION_SUCCESS, + HealthcareUtils.deidentifyLroCounters.DEIDENTIFY_OPERATION_ERRORS, + HealthcareUtils.deidentifyLroCounters.RESOURCES_DEIDENTIFIED_SUCCESS, + HealthcareUtils.deidentifyLroCounters.RESOURCES_DEIDENTIFIED_ERRORS); if (operation.getError() != null) { throw new IOException( String.format("DeidentifyFhirStore operation (%s) failed.", operation.getName())); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java new file mode 100644 index 000000000000..3305c3baeb25 --- /dev/null +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + import org.apache.beam.sdk.metrics.Counter; + + public class HealthcareUtils{ + public static class deidentifyLroCounters { + private static final Counter DEIDENTIFY_OPERATION_SUCCESS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_success_count"); + private static final Counter DEIDENTIFY_OPERATION_ERRORS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_failure_count"); + private static final Counter RESOURCES_DEIDENTIFIED_SUCCESS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_success_count"); + private static final Counter RESOURCES_DEIDENTIFIED_ERRORS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_failure_count"); + } + } \ No newline at end of file From ef2786e9151e37a1da3a4543ea0bebc760078902 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Sun, 3 Sep 2023 18:56:16 -0400 Subject: [PATCH 02/12] revert --- .../beam/sdk/io/gcp/healthcare/FhirIO.java | 22 +++++++++--- .../io/gcp/healthcare/HealthcareUtils.java | 36 ------------------- 2 files changed, 17 insertions(+), 41 deletions(-) delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java index 80ecff0f7818..4fdd3513c806 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIO.java @@ -51,7 +51,6 @@ import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.FileSystems; import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; -import org.apache.beam.sdk.io.gcp.healthcare.HealthcareUtils; import org.apache.beam.sdk.io.fs.MatchResult; import org.apache.beam.sdk.io.fs.MatchResult.Metadata; import org.apache.beam.sdk.io.fs.MatchResult.Status; @@ -1762,6 +1761,19 @@ public PCollection expand(PBegin input) { /** A function that schedules a deidentify operation and monitors the status. */ public static class DeidentifyFn extends DoFn { + private static final Counter DEIDENTIFY_OPERATION_SUCCESS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_success_count"); + private static final Counter DEIDENTIFY_OPERATION_ERRORS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_failure_count"); + private static final Counter RESOURCES_DEIDENTIFIED_SUCCESS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_success_count"); + private static final Counter RESOURCES_DEIDENTIFIED_ERRORS = + Metrics.counter( + DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_failure_count"); + private HealthcareApiClient client; private final ValueProvider destinationFhirStore; private static final Gson gson = new Gson(); @@ -1788,10 +1800,10 @@ public void deidentify(ProcessContext context) throws IOException, InterruptedEx operation = client.pollOperation(operation, 15000L); incrementLroCounters( operation, - HealthcareUtils.deidentifyLroCounters.DEIDENTIFY_OPERATION_SUCCESS, - HealthcareUtils.deidentifyLroCounters.DEIDENTIFY_OPERATION_ERRORS, - HealthcareUtils.deidentifyLroCounters.RESOURCES_DEIDENTIFIED_SUCCESS, - HealthcareUtils.deidentifyLroCounters.RESOURCES_DEIDENTIFIED_ERRORS); + DEIDENTIFY_OPERATION_SUCCESS, + DEIDENTIFY_OPERATION_ERRORS, + RESOURCES_DEIDENTIFIED_SUCCESS, + RESOURCES_DEIDENTIFIED_ERRORS); if (operation.getError() != null) { throw new IOException( String.format("DeidentifyFhirStore operation (%s) failed.", operation.getName())); diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java deleted file mode 100644 index 3305c3baeb25..000000000000 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/HealthcareUtils.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - import org.apache.beam.sdk.metrics.Counter; - - public class HealthcareUtils{ - public static class deidentifyLroCounters { - private static final Counter DEIDENTIFY_OPERATION_SUCCESS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_success_count"); - private static final Counter DEIDENTIFY_OPERATION_ERRORS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "deidentify_operation_failure_count"); - private static final Counter RESOURCES_DEIDENTIFIED_SUCCESS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_success_count"); - private static final Counter RESOURCES_DEIDENTIFIED_ERRORS = - Metrics.counter( - DeidentifyFn.class, BASE_METRIC_PREFIX + "resources_deidentified_failure_count"); - } - } \ No newline at end of file From f6498477df84642247d6e414f95983a2ecd1b4b0 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 11:03:33 -0400 Subject: [PATCH 03/12] Clear GC --- .../org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index e2422f9e172c..04fd8a851125 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -141,6 +141,7 @@ public void testFhirIOSearch() { // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); // Verify that none of the result resource sets are empty sets, using both getResources methods. + System.gc(); PCollection> keyedResources = result.getKeyedResources(); PAssert.that(keyedResources) .satisfies( @@ -173,6 +174,7 @@ public void testFhirIOSearchWithGenericParameters() { // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); // Verify that none of the result resource sets are empty sets, using both getResources methods. + System.gc(); PCollection resources = result.getResources(); PAssert.that(resources) .satisfies( From 7e90e037a3a4bb2244d106f25abe678d1195e134 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 12:16:01 -0400 Subject: [PATCH 04/12] try in loop --- .../org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 04fd8a851125..9a14bcd87dfe 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -141,7 +141,6 @@ public void testFhirIOSearch() { // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); // Verify that none of the result resource sets are empty sets, using both getResources methods. - System.gc(); PCollection> keyedResources = result.getKeyedResources(); PAssert.that(keyedResources) .satisfies( @@ -149,6 +148,7 @@ public void testFhirIOSearch() { for (KV resource : input) { assertEquals(KEY, resource.getKey()); assertNotEquals(0, resource.getValue().size()); + System.gc(); } return null; }); @@ -174,13 +174,13 @@ public void testFhirIOSearchWithGenericParameters() { // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); // Verify that none of the result resource sets are empty sets, using both getResources methods. - System.gc(); PCollection resources = result.getResources(); PAssert.that(resources) .satisfies( input -> { for (JsonArray resource : input) { assertNotEquals(0, resource.size()); + System.gc(); } return null; }); From bb978c8967438d8ba73a314c074831a5adc521a5 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 13:35:18 -0400 Subject: [PATCH 05/12] check filter on jenkins --- .../apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 9a14bcd87dfe..89c056e92dab 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -133,7 +133,7 @@ public void testFhirIOSearch() { // Search using the resource type of each written resource and empty search parameters. PCollection> searchConfigs = pipeline.apply( - Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))); + Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))).apply(Sample.any(10)); FhirIO.Search.Result result = searchConfigs.apply( FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)); @@ -148,7 +148,6 @@ public void testFhirIOSearch() { for (KV resource : input) { assertEquals(KEY, resource.getKey()); assertNotEquals(0, resource.getValue().size()); - System.gc(); } return null; }); @@ -164,7 +163,7 @@ public void testFhirIOSearchWithGenericParameters() { PCollection>> searchConfigs = pipeline.apply( Create.of(genericParametersInput) - .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))); + .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))).apply(Sample.any(10)); FhirIO.Search.Result result = searchConfigs.apply( (FhirIO.Search>) From 8b03314f9d72dc80109680fcc43eda6769081cca Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 13:42:09 -0400 Subject: [PATCH 06/12] try with filtering --- .../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 89c056e92dab..38c04f8f7c70 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -35,6 +35,7 @@ import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.io.aws2.schemas.Sample; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -133,10 +134,10 @@ public void testFhirIOSearch() { // Search using the resource type of each written resource and empty search parameters. PCollection> searchConfigs = pipeline.apply( - Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))).apply(Sample.any(10)); + Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))); FhirIO.Search.Result result = searchConfigs.apply( - FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)); + FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)).Sample.any(10); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); @@ -163,12 +164,12 @@ public void testFhirIOSearchWithGenericParameters() { PCollection>> searchConfigs = pipeline.apply( Create.of(genericParametersInput) - .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))).apply(Sample.any(10)); + .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))); FhirIO.Search.Result result = searchConfigs.apply( (FhirIO.Search>) FhirIO.searchResourcesWithGenericParameters( - healthcareDataset + "/fhirStores/" + fhirStoreId)); + healthcareDataset + "/fhirStores/" + fhirStoreId)).Sample.any(10); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); @@ -179,7 +180,6 @@ public void testFhirIOSearchWithGenericParameters() { input -> { for (JsonArray resource : input) { assertNotEquals(0, resource.size()); - System.gc(); } return null; }); From 2b0ad6e2fa24f4054ba8e82b87b78bc0ceaa9e4c Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 13:48:19 -0400 Subject: [PATCH 07/12] spotless --- .../sdk/io/gcp/healthcare/FhirIOSearchIT.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 38c04f8f7c70..a24406f912d3 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -35,7 +35,6 @@ import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.sdk.io.aws2.schemas.Sample; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -136,8 +135,10 @@ public void testFhirIOSearch() { pipeline.apply( Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))); FhirIO.Search.Result result = - searchConfigs.apply( - FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)).Sample.any(10); + searchConfigs + .apply(FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)) + .Sample + .any(10); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); @@ -166,10 +167,13 @@ public void testFhirIOSearchWithGenericParameters() { Create.of(genericParametersInput) .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))); FhirIO.Search.Result result = - searchConfigs.apply( - (FhirIO.Search>) - FhirIO.searchResourcesWithGenericParameters( - healthcareDataset + "/fhirStores/" + fhirStoreId)).Sample.any(10); + searchConfigs + .apply( + (FhirIO.Search>) + FhirIO.searchResourcesWithGenericParameters( + healthcareDataset + "/fhirStores/" + fhirStoreId)) + .Sample + .any(10); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); From d7ccc1da98ec0c3858646d88903d9369531398a9 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 13:56:01 -0400 Subject: [PATCH 08/12] filter --- .../apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index a24406f912d3..b032a9dd7566 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -35,6 +35,7 @@ import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.io.aws2.schemas.Sample; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -137,8 +138,7 @@ public void testFhirIOSearch() { FhirIO.Search.Result result = searchConfigs .apply(FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)) - .Sample - .any(10); + .apply(Sample.any(10)); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); @@ -172,8 +172,7 @@ public void testFhirIOSearchWithGenericParameters() { (FhirIO.Search>) FhirIO.searchResourcesWithGenericParameters( healthcareDataset + "/fhirStores/" + fhirStoreId)) - .Sample - .any(10); + .apply(Sample.any(10)); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); From 6f5c464150cb9d6811e5dafa7ef7a29e59c95707 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 14:24:41 -0400 Subject: [PATCH 09/12] temp disable --- .../apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index b032a9dd7566..db11cc729706 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -126,7 +126,7 @@ public void teardown() throws IOException { client.deleteFhirStore(healthcareDataset + "/fhirStores/" + BASE_STORE_ID + version); } } - + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearch() { pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); @@ -156,7 +156,7 @@ public void testFhirIOSearch() { pipeline.run().waitUntilFinish(); } - + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearchWithGenericParameters() { pipeline.getOptions().as(DirectOptions.class).setBlockOnRun(false); @@ -171,8 +171,7 @@ public void testFhirIOSearchWithGenericParameters() { .apply( (FhirIO.Search>) FhirIO.searchResourcesWithGenericParameters( - healthcareDataset + "/fhirStores/" + fhirStoreId)) - .apply(Sample.any(10)); + healthcareDataset + "/fhirStores/" + fhirStoreId)); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); From 400e2c43bbe1dce3eefd997415fe36577fcc7614 Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 14:25:30 -0400 Subject: [PATCH 10/12] temp disable --- .../org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index db11cc729706..594d45898bda 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -126,6 +126,7 @@ public void teardown() throws IOException { client.deleteFhirStore(healthcareDataset + "/fhirStores/" + BASE_STORE_ID + version); } } + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearch() { @@ -156,6 +157,7 @@ public void testFhirIOSearch() { pipeline.run().waitUntilFinish(); } + @Ignore("https://github.com/apache/beam/issues/28505") @Test public void testFhirIOSearchWithGenericParameters() { From 1f26d41502942356abdae3cef736118f4d857a2d Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 14:31:28 -0400 Subject: [PATCH 11/12] format --- .../sdk/io/gcp/healthcare/FhirIOSearchIT.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 594d45898bda..13761bd723a6 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -35,7 +35,6 @@ import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.coders.VarIntCoder; -import org.apache.beam.sdk.io.aws2.schemas.Sample; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; @@ -137,9 +136,8 @@ public void testFhirIOSearch() { pipeline.apply( Create.of(input).withCoder(FhirSearchParameterCoder.of(StringUtf8Coder.of()))); FhirIO.Search.Result result = - searchConfigs - .apply(FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)) - .apply(Sample.any(10)); + searchConfigs.apply( + FhirIO.searchResources(healthcareDataset + "/fhirStores/" + fhirStoreId)); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); @@ -169,11 +167,10 @@ public void testFhirIOSearchWithGenericParameters() { Create.of(genericParametersInput) .withCoder(FhirSearchParameterCoder.of(ListCoder.of(VarIntCoder.of())))); FhirIO.Search.Result result = - searchConfigs - .apply( - (FhirIO.Search>) - FhirIO.searchResourcesWithGenericParameters( - healthcareDataset + "/fhirStores/" + fhirStoreId)); + searchConfigs.apply( + (FhirIO.Search>) + FhirIO.searchResourcesWithGenericParameters( + healthcareDataset + "/fhirStores/" + fhirStoreId)); // Verify that there are no failures. PAssert.that(result.getFailedSearches()).empty(); From 726ed0d0179a0d2188feae16257c32225c33710c Mon Sep 17 00:00:00 2001 From: Svetak Sundhar Date: Thu, 28 Sep 2023 14:38:02 -0400 Subject: [PATCH 12/12] import --- .../org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java index 13761bd723a6..fc92a568ba4d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java @@ -43,6 +43,7 @@ import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith;