more resiliancy (#4395)

DataBiosphere · Dec 1, 2023 · d906619 · d906619
1 parent 03b42a6
commit d906619
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/src/toil/fileStores/nonCachingFileStore.py b/src/toil/fileStores/nonCachingFileStore.py
@@ -126,7 +126,11 @@ def open(self, job: Job) -> Generator[None, None, None]:
             os.chdir(startingDir)
             # Finally delete the job from the worker
             self.check_for_state_corruption()
-            os.remove(self.jobStateFile)
+            try:
+                os.remove(self.jobStateFile)
+            except FileNotFoundError:
+                logger.exception('Job state file %s has gone missing unexpectedly; some cleanup for failed jobs may be getting skipped!', self.jobStateFile)
+                pass
 
     def writeGlobalFile(self, localFileName: str, cleanup: bool=False) -> FileID:
         absLocalFileName = self._resolveAbsoluteLocalPath(localFileName)

diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py
@@ -148,6 +148,8 @@ def callback(fileHandle: Union[IO[bytes], IO[str]]) -> None:
             if not isinstance(statsStr, str):
                 statsStr = statsStr.decode()
             stats = json.loads(statsStr, object_hook=Expando)
+            if not stats:
+                return
             try:
                 logs = stats.workers.logsToMaster
             except AttributeError: