diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
index 91405af5f..13dafd06e 100644
--- a/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
+++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
@@ -767,6 +767,7 @@ private static Failure newTimeoutFailure(TimeoutType timeoutType, @Nullable Fail
   private static boolean isRetryPolicyNotSet(
       PollActivityTaskQueueResponseOrBuilder pollActivityTask) {
     return !pollActivityTask.hasScheduleToCloseTimeout()
+        && !pollActivityTask.hasStartToCloseTimeout()
         && (!pollActivityTask.hasRetryPolicy()
             || pollActivityTask.getRetryPolicy().getMaximumAttempts() <= 0);
   }