From 7c9c3e50958834004a20fb9bc2bd2ce1fd1ecc5e Mon Sep 17 00:00:00 2001
From: Gguidini <giovanni@codecov.io>
Date: Tue, 17 Oct 2023 09:38:51 -0300
Subject: [PATCH] fix: Make static analysis handle a large number of uploads

This is more a fix around error handling and updating information to users than actual performance.

**Error Handling**

In terms of error handling, consider that the number of uploads is too big.
We "start" all of them at the same time, but the number of connections is limited.

For uploads you stringify the data and try to grab a connection from the pool to upload.
Then you need to _actually_ upload (e.g. write data to the pipe)

Some timeouts can exist in this process:
* `PoolTimeout` - waiting too long for a connection to become available
* `ReadTimeout` - waiting too long for a response back
(these are the most important ones)

Given that we _will_ have thousands of files to upload it _will_ take a long time,
but that's alright. So we can simply ignore those timeouts.

Then we need to balance the number of connections available.
Through some empirical testing 100 is too much. If you have many big files
trying to upload at the same time the COU time of each is too small, so it takes
too long for the file to finish upload. This throws a 500 error from GCS.

So it's important that we don't try to upload too many files at the same time.
With 20 connections I was able to upload ~6600 files with no errors.

**Update user info**

The progress bar was broken.
It was updating when the single upload was started, not when it finished.
To fix that I just made it update when it finishes each upload.

Now we can properly see that it takes forever :D
---
 .../services/staticanalysis/__init__.py       | 43 ++++++++++++-------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/codecov_cli/services/staticanalysis/__init__.py b/codecov_cli/services/staticanalysis/__init__.py
index ecd347a8..c1c9c1f1 100644
--- a/codecov_cli/services/staticanalysis/__init__.py
+++ b/codecov_cli/services/staticanalysis/__init__.py
@@ -9,7 +9,6 @@
 import click
 import httpx
 import requests
-import yaml
 
 from codecov_cli.helpers.config import CODECOV_API_URL
 from codecov_cli.services.staticanalysis.analyzers import get_best_analyzer
@@ -47,8 +46,16 @@ async def run_analysis_entrypoint(
     all_data = processing_results["all_data"]
     try:
         json_output = {"commit": commit, "filepaths": file_metadata}
+        logger.info(
+            "Sending files fingerprints to Codecov",
+            extra=dict(
+                extra_log_attributes=dict(
+                    files_effectively_analyzed=len(json_output["filepaths"])
+                )
+            ),
+        )
         logger.debug(
-            "Sending data for server",
+            "Data sent to Codecov",
             extra=dict(extra_log_attributes=dict(json_payload=json_output)),
         )
         upload_url = enterprise_url or CODECOV_API_URL
@@ -96,36 +103,42 @@ async def run_analysis_entrypoint(
         for el in response_json["filepaths"]
         if (el["state"].lower() == "created" or should_force)
     ]
+
     if files_that_need_upload:
         uploaded_files = []
-        failed_upload = []
+        failed_uploads = []
         with click.progressbar(
             length=len(files_that_need_upload),
-            label="Uploading files",
+            label=f"Upload info to storage",
         ) as bar:
-            async with httpx.AsyncClient() as client:
+            # It's better to have less files competing over CPU time when uploading
+            # Especially if we might have large files
+            limits = httpx.Limits(max_connections=20)
+            # Because there might be too many files to upload we will ignore most timeouts
+            timeout = httpx.Timeout(read=None, pool=None, connect=None, write=10.0)
+            async with httpx.AsyncClient(timeout=timeout, limits=limits) as client:
                 all_tasks = []
                 for el in files_that_need_upload:
                     all_tasks.append(send_single_upload_put(client, all_data, el))
-                    bar.update(1, all_data[el["filepath"]])
                 try:
-                    resps = await asyncio.gather(*all_tasks)
+                    for task in asyncio.as_completed(all_tasks):
+                        resp = await task
+                        bar.update(1, el["filepath"])
+                        if resp["succeeded"]:
+                            uploaded_files.append(resp["filepath"])
+                        else:
+                            failed_uploads.append(resp["filepath"])
                 except asyncio.CancelledError:
                     message = (
                         "Unknown error cancelled the upload tasks.\n"
                         + f"Uploaded {len(uploaded_files)}/{len(files_that_need_upload)} files successfully."
                     )
                     raise click.ClickException(message)
-            for resp in resps:
-                if resp["succeeded"]:
-                    uploaded_files.append(resp["filepath"])
-                else:
-                    failed_upload.append(resp["filepath"])
-        if failed_upload:
-            logger.warning(f"{len(failed_upload)} files failed to upload")
+        if failed_uploads:
+            logger.warning(f"{len(failed_uploads)} files failed to upload")
             logger.debug(
                 "Failed files",
-                extra=dict(extra_log_attributes=dict(filenames=failed_upload)),
+                extra=dict(extra_log_attributes=dict(filenames=failed_uploads)),
             )
         logger.info(
             f"Uploaded {len(uploaded_files)} files",