Stream instead of buffer JSON in read_json/write_json (#96)

* Stream instead of buffer JSON in read_json/write_json

* Add changelog entry

* Mollify mypy

* Fix potential encoding issue.

.changes/unreleased/Under the Hood-20240314-161737.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Stream JSON on read/write instead of holding it in memory
+time: 2024-03-14T16:17:37.570328-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "96"

dbt_common/clients/system.py

@@ -292,11 +292,28 @@ def write_file(path: str, contents: str = "") -> bool:
 
 
 def read_json(path: str) -> Dict[str, Any]:
-    return json.loads(load_file_contents(path))
+    path = convert_path(path)
+    with open(path, "r") as f:
+        return json.load(f)
 
 
 def write_json(path: str, data: Dict[str, Any]) -> bool:
-    return write_file(path, json.dumps(data, cls=dbt_common.utils.encoding.JSONEncoder))
+    path = convert_path(path)
+    try:
+        make_directory(os.path.dirname(path))
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, cls=dbt_common.utils.encoding.JSONEncoder)
+    except Exception as exc:
+        # See write_file() for an explanation of this error handling.
+        if os.name == "nt":
+            if getattr(exc, "winerror", 0) == 3:
+                reason = "Path was too long"
+            else:
+                reason = "Path was possibly too long"
+            fire_event(SystemCouldNotWrite(path=path, reason=reason, exc=str(exc)))
+        else:
+            raise
+    return True
 
 
 def _windows_rmdir_readonly(func: Callable[[str], Any], path: str, exc: Tuple[Any, OSError, Any]):

dbt_common/utils/executor.py

@@ -74,6 +74,6 @@ def executor(config: HasThreadingConfig) -> ConnectingExecutor:
     else:
         return MultiThreadedExecutor(
             max_workers=config.threads,
-            initializer=_thread_initializer,
-            initargs=(get_invocation_context(),),
+            initializer=_thread_initializer,  # type: ignore
+            initargs=(get_invocation_context(),),  # type: ignore
         )