Allow for download through our caching proxy at sec-certs.org

Recently, CCportal changed infra and enabled a rate limit. Our
usual download procedure hits this rate limit and is able to download
only a small subset of the artifacts.

Thus a proxy is running at sec-certs.org that caches the artifacts.
This commit makes it possible to use the proxy in the tool.

By querying the proxy on the background, the server makes sure that
the rate limit is not hit and the artifacts are available.

src/sec_certs/configuration.py

@@ -106,6 +106,8 @@ class Configuration(BaseSettings):
         ge=0,
         le=100,
     )
+    cc_use_proxy: bool = Field(False, description="Download CC artifacts through the sec-certs.org proxy.")
+    fips_use_proxy: bool = Field(False, description="Download FIPS artifacts through the sec-certs.org proxy.")
     enable_progress_bars: bool = Field(
         True, description="If true, progress bars will be printed to stdout during computation."
     )

src/sec_certs/sample/cc.py

@@ -18,6 +18,7 @@
 import sec_certs.utils.sanitization
 from sec_certs import constants
 from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan
+from sec_certs.configuration import config
 from sec_certs.sample.cc_certificate_id import canonicalize, schemes
 from sec_certs.sample.certificate import Certificate, References, logger
 from sec_certs.sample.certificate import Heuristics as BaseHeuristics
@@ -42,8 +43,7 @@ class CCCertificate(
     the certificate can handle itself. `CCDataset` class then instrument this functionality.
     """
 
-    cc_url = "http://www.commoncriteriaportal.org"
-    empty_st_url = "http://www.commoncriteriaportal.org/files/epfiles/"
+    cc_url = "https://www.commoncriteriaportal.org"
 
     @dataclass(eq=True, frozen=True)
     class MaintenanceReport(ComplexSerializableType):
@@ -753,7 +753,7 @@ def download_pdf_report(cert: CCCertificate) -> CCCertificate:
         if not cert.report_link:
             exit_code = "No link"
         else:
-            exit_code = helpers.download_file(cert.report_link, cert.state.report.pdf_path)
+            exit_code = helpers.download_file(cert.report_link, cert.state.report.pdf_path, proxy=config.cc_use_proxy)
         if exit_code != requests.codes.ok:
             error_msg = f"failed to download report from {cert.report_link}, code: {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
@@ -773,7 +773,9 @@ def download_pdf_st(cert: CCCertificate) -> CCCertificate:
         :return CCCertificate: returns the modified certificate with updated state
         """
         exit_code: str | int = (
-            helpers.download_file(cert.st_link, cert.state.st.pdf_path) if cert.st_link else "No link"
+            helpers.download_file(cert.st_link, cert.state.st.pdf_path, proxy=config.cc_use_proxy)
+            if cert.st_link
+            else "No link"
         )
 
         if exit_code != requests.codes.ok:
@@ -795,7 +797,9 @@ def download_pdf_cert(cert: CCCertificate) -> CCCertificate:
         :return CCCertificate: returns the modified certificate with updated state
         """
         exit_code: str | int = (
-            helpers.download_file(cert.cert_link, cert.state.cert.pdf_path) if cert.cert_link else "No link"
+            helpers.download_file(cert.cert_link, cert.state.cert.pdf_path, proxy=config.cc_use_proxy)
+            if cert.cert_link
+            else "No link"
         )
 
         if exit_code != requests.codes.ok:

src/sec_certs/sample/fips.py

@@ -558,7 +558,11 @@ def parse_html_module(cert: FIPSCertificate) -> FIPSCertificate:
 
     @staticmethod
     def download_module(cert: FIPSCertificate) -> FIPSCertificate:
-        if (exit_code := helpers.download_file(cert.module_html_url, cert.state.module_html_path)) != requests.codes.ok:
+        if (
+            exit_code := helpers.download_file(
+                cert.module_html_url, cert.state.module_html_path, proxy=config.fips_use_proxy
+            )
+        ) != requests.codes.ok:
             error_msg = f"failed to download html module from {cert.module_html_url}, code {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
             cert.state.module_download_ok = False
@@ -568,7 +572,11 @@ def download_module(cert: FIPSCertificate) -> FIPSCertificate:
 
     @staticmethod
     def download_policy(cert: FIPSCertificate) -> FIPSCertificate:
-        if (exit_code := helpers.download_file(cert.policy_pdf_url, cert.state.policy_pdf_path)) != requests.codes.ok:
+        if (
+            exit_code := helpers.download_file(
+                cert.policy_pdf_url, cert.state.policy_pdf_path, proxy=config.fips_use_proxy
+            )
+        ) != requests.codes.ok:
             error_msg = f"failed to download pdf policy from {cert.policy_pdf_url}, code {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
             cert.state.policy_download_ok = False

src/sec_certs/utils/helpers.py

@@ -22,11 +22,27 @@
 logger = logging.getLogger(__name__)
 
 
+_PROXIES = {
+    "https://www.commoncriteriaportal.org/": "https://sec-certs.org/proxy/cc/",
+    "https://csrc.nist.gov/": "https://sec-certs.org/proxy/fips/",
+}
+
+
 def download_file(
-    url: str, output: Path, delay: float = 0, show_progress_bar: bool = False, progress_bar_desc: str | None = None
+    url: str,
+    output: Path,
+    delay: float = 0,
+    show_progress_bar: bool = False,
+    progress_bar_desc: str | None = None,
+    proxy: bool = False,
 ) -> str | int:
     try:
         time.sleep(delay)
+        if proxy:
+            for upstream in _PROXIES:
+                if upstream in url:
+                    url = url.replace(upstream, _PROXIES[upstream])
+                    break
         # See https://github.com/psf/requests/issues/3953 for header justification
         r = requests.get(
             url,