From 71906d02944d8d9c9df54332501e667239e59901 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Mon, 27 Nov 2023 21:05:06 -0800
Subject: [PATCH] Save report.json after 10 second intervals. (#256)

---
 simple/stats/reporter.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/simple/stats/reporter.py b/simple/stats/reporter.py
index 358e0fa4..1b90546c 100644
--- a/simple/stats/reporter.py
+++ b/simple/stats/reporter.py
@@ -17,9 +17,14 @@
 from enum import Enum
 from functools import wraps
 import json
+import time
 
 from util.filehandler import FileHandler
 
+# Minimum interval before a report should be saved to disk or cloud.
+# This keeps it from reporting too frequently and running into GCS rate limit issues.
+_REPORT_SAVE_INTERVAL_SECONDS = 10.0
+
 
 class Status(Enum):
   NOT_STARTED = auto()
@@ -28,6 +33,10 @@ class Status(Enum):
   FAILURE = auto()
 
 
+def _is_done_status(status: Status) -> bool:
+  return status == Status.SUCCESS or status == Status.FAILURE
+
+
 class ImportReporter:
   """Generates a report on every reported change.
 
@@ -38,6 +47,7 @@ def __init__(self, report_fh: FileHandler) -> None:
     self.status = Status.NOT_STARTED
     self.start_time = None
     self.last_update = datetime.now()
+    self.last_reported: float | None = None
     self.report_fh = report_fh
     self.data = {}
     self.import_files: dict[str, FileImportReporter] = {}
@@ -113,7 +123,12 @@ def _maybe_report(field: str, func=None):
 
   def save(self) -> None:
     self.last_update = datetime.now()
-    self.report_fh.write_string(json.dumps(self.json(), indent=2))
+    should_report = not self.last_reported or time.time(
+    ) - self.last_reported >= _REPORT_SAVE_INTERVAL_SECONDS or _is_done_status(
+        self.status)
+    if should_report:
+      self.last_reported = time.time()
+      self.report_fh.write_string(json.dumps(self.json(), indent=2))
 
 
 class FileImportReporter: