Merge pull request #96 from ossf/main

Resolving build issues with powershell and M1 Apple Chip capability

omega/analyzer/worker/tools/postprocess.py

@@ -754,8 +754,14 @@ def process_oss_defog(self, filename):
 
             for finding in findings:
                 parts = finding.split(':', 1)
-                filename = parts[0].split('/', 3)[-1]
-                snippet = parts[1]
+                if len(parts) == 2:
+                    filename = parts[0].split('/', 3)[-1]
+                    snippet = parts[1]
+                elif len(parts) == 1:
+                    filename = parts[0].split('/', 3)[-1]
+                    snippet = '(No snippet available)'
+                else:
+                    continue
 
                 self.add_result(**{
                     'tool_name': 'oss-defog',

omega/analyzer/worker/tools/runtools.sh

@@ -92,6 +92,8 @@ else
     NC=''
 fi
 
+VERY_SHORT_ANALYZER_TIMEOUT="30s"
+SHORT_ANALYZER_TIMEOUT="10m"
 LONG_ANALYZER_TIMEOUT="60m"
 
 BUILD_SCRIPT_ROOT="/opt/buildscripts"
@@ -143,7 +145,7 @@ fi
 
 PACKAGE_OVERRIDE_PREVIOUS_VERSION="$2"
 
-ANALYZER_VERSION="0.8.4"
+ANALYZER_VERSION="0.8.5"
 ANALYSIS_DATE=$(date)
 
 # ASCII Art generated using http://patorjk.com/software/taag/#p=display&h=0&v=0&c=echo&f=THIS&t=Toolshed
@@ -328,16 +330,19 @@ event stop tool-oss-detect-cryptography
 # Backdoor Detection
 printf "${RED}Detecting backdoors...${NC}\n"
 event start tool-oss-detect-backdoor
-oss-detect-backdoor -o /opt/results/tool-oss-detect-backdoor.sarif -f sarifv2 "$CUR_ROOT" 2>&1 | tail +11 >/opt/result/tool-oss-detect-backdoor.error
+oss-detect-backdoor -o /opt/result/tool-oss-detect-backdoor.sarif -f sarifv2 "$CUR_ROOT" 2>&1 | tail +11 >/opt/result/tool-oss-detect-backdoor.error
 event stop tool-oss-detect-backdoor
 
 # Defogger
 printf "${RED}Detecting obfuscated code...${NC}\n"
 event start tool-oss-defog
 if [[ "${PACKAGE_PURL_LOCAL_SOURCE}" == true ]]; then
-    oss-defog "$CUR_ROOT/src" >/dev/null 2>/opt/result/tool-oss-defog.txt
+    timeout $SHORT_ANALYZER_TIMEOUT oss-defog "$CUR_ROOT/src" >/dev/null 2>/opt/result/tool-oss-defog.txt
 else
-    oss-defog "$PACKAGE_PURL_OSSGADGET" >/dev/null 2>/opt/result/tool-oss-defog.txt
+    timeout $SHORT_ANALYZER_TIMEOUT oss-defog "$PACKAGE_PURL_OSSGADGET" >/dev/null 2>/opt/result/tool-oss-defog.txt
+fi
+if [ $? -eq 124 ]; then
+    echo "oss-defog timed out after $SHORT_ANALYZER_TIMEOUT." >>/opt/result/tool-oss-defog.txt
 fi
 event stop tool-oss-defog
 
@@ -428,7 +433,10 @@ if [[ "$FILE_TYPES" =~ ( )(c|h|hpp|c\+\+|cpp)( ) ]]; then
     cd "$CUR_ROOT/src"
     printf "${RED}Running CppCheck...${NC}\n"
     event start tool-cppcheck
-    cppcheck --enable=all --quiet --template='{file}~!~{line}~!~{severity}~!~{message}~!~{code}~!~{id}~!~{cwe}' "$CUR_ROOT/src" >/opt/result/tool-cppcheck.json 2>/opt/result/tool-cppcheck.error
+    timeout $SHORT_ANALYZER_TIMEOUT cppcheck --addon=threadsafety --addon=y2038 --template='{file}~!~{line}~!~{severity}~!~{message}~!~{code}~!~{id}~!~{cwe}' "$CUR_ROOT/src" >/opt/result/tool-cppcheck.json 2>/opt/result/tool-cppcheck.error
+    if [ $? -eq 124 ]; then
+        echo "CppCheck timed out after $SHORT_ANALYZER_TIMEOUT." >>/opt/result/tool-cppcheck.error
+    fi
     event stop tool-cppcheck
 fi
 
@@ -636,7 +644,7 @@ fi
 # Manalyze - https://github.com/JusticeRage/Manalyze
 printf "${RED}Checking Manalyze...${NC}\n"
 event start tool-manalyze
-find "$CUR_ROOT/src" -type f -print0 | xargs -0 file | grep "PE32" | cut -d: -f1 | xargs -I{} -n1 bash -c 'F="{}"; FN=$(echo $F | shasum -a256 | cut -d" " -f1); manalyze -d all --plugins=compilers,peid,strings,findcrypt,packer,imports,resources,mitigation,overlay,authenticode -o json --pe "$F" >"/opt/result/tool-manalyze.$FN.json" 2>>/opt/result/tool-manalyze.log'
+find "$CUR_ROOT/src" -type f -print0 | xargs -0 file | grep "PE32" | cut -d: -f1 | xargs -I{} -n1 bash -c 'F="{}"; FN=$(echo $F | shasum -a256 | cut -d" " -f1); timeout $VERY_SHORT_ANALYZER_TIMEOUT manalyze -d all --plugins=compilers,peid,strings,findcrypt,packer,imports,resources,mitigation,overlay,authenticode -o json --pe "$F" >"/opt/result/tool-manalyze.$FN.json" 2>>/opt/result/tool-manalyze.log'
 if [ -n "$(ls -A /opt/result/tool-manalyze.*.json 2>/dev/null)" ]; then
     cat /opt/result/tool-manalyze.*.json | jq -s > /opt/result/tool-manalyze.json
     rm /opt/result/tool-manalyze.*.json
@@ -649,9 +657,9 @@ if [ -z "$SNYK_TOKEN" ]; then
 else
     printf "${RED}Checking Snyk Code...${NC}\n"
     event start tool-snyk-code
-    snyk code test --sarif-file-output=/opt/result/tool-snyk-code.sarif --severity-threshold=low "${CUR_ROOT}/src" >/opt/result/tool-snyk-code.log 2>&1
+    timeout $SHORT_ANALYZER_TIMEOUT snyk code test --sarif-file-output=/opt/result/tool-snyk-code.sarif --severity-threshold=low "${CUR_ROOT}/src" >/opt/result/tool-snyk-code.log 2>&1
     SNYK_ERR=$?
-    if [[ $SNYK_ERR == 2 || $SNYK_ERR == 3 ]]; then
+    if [[ $SNYK_ERR == 2 || $SNYK_ERR == 3 || $SNYK_ERR == 124 ]]; then
         echo "Snyk Code failed to run, error code: $SNYK_ERR" >/opt/result/tool-snyk-code.error
     fi
     event stop tool-snyk-code
@@ -688,4 +696,4 @@ cp /tmp/events.txt "$EXPORT_DIR/summary-telemetry-events.txt"
 # Fix permissions to match the export directory
 chown -R nonroot:1000 $DESTINATION_DIR/*
 
-exit 0
+exit 0

omega/fuzzing/compression/fuzz-compressor

@@ -68,7 +68,7 @@ do
   radamsa "$WORK_DIR/sample.gz" > "${WORK_DIR}/fuzzed.gz"
   ${DECOMPRESSOR} ${DECOMPRESSOR_ARGS} "${WORK_DIR}/fuzzed.gz" > /dev/null 2>&1
   if (( $? > 127 )); then
-    NEW_FILENAME="${CRASHES_DIR}/${COMPRESSOR}-fuzzed-$(uuidgen-r).gz"
+    NEW_FILENAME="${CRASHES_DIR}/${COMPRESSOR}-fuzzed-$(uuidgen -r).gz"
     mv "$WORK_DIR/fuzzed.gz" "${NEW_FILENAME}"
     echo "!!! Found a crash, saving to: ${NEW_FILENAME}"
   else

omega/oaf/omega/analyze.py

@@ -8,7 +8,6 @@
 import shlex
 import subprocess  # nosec: B404
 import tempfile
-import uuid
 from datetime import datetime, timedelta
 
 from dotenv import dotenv_values
@@ -23,7 +22,7 @@ class AnalysisRunner:
     Executes analysis and creates assertions.
     """
 
-    def __init__(self, package_url: str, docker_container: str, repository: str, signer: str):
+    def __init__(self, package_url: str, docker_container: str, repository: str, signer: str, work_directory: str | None):
         """Initialize a new Analysis Runner."""
         required_commands = [
             ["python", "-V"],
@@ -49,20 +48,27 @@ def __init__(self, package_url: str, docker_container: str, repository: str, sig
         self.repository = repository
         self.signer = signer
 
-        _uuid = str(uuid.uuid4())
-        self.work_directory = os.path.join(tempfile.gettempdir(), f"omega-{_uuid}")  # ADD UUID
-
-        self.work_directory = tempfile.TemporaryDirectory(  # pylint: disable=consider-using-with
-            prefix="omega-", ignore_cleanup_errors=True
-        )
-        logging.debug("Output directory: %s", self.work_directory.name)
+        # Set up the work directory (default: temporary, or provided by the user)
+        if work_directory:
+            self.work_directory = work_directory
+            self.work_directory_name = self.work_directory
+        else:
+            self.work_directory = tempfile.TemporaryDirectory(  # pylint: disable=consider-using-with
+                prefix="omega-", ignore_cleanup_errors=True
+            )
+            self.work_directory_name = self.work_directory.name
+        logging.debug("Output (work) directory: %s", self.work_directory_name)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        """Cleans up after ourselves."""
-        logging.warning("We did not clean up the directory: %s", self.work_directory)
+        """Clean up after ourselves."""
+        if isinstance(self.work_directory, tempfile.TemporaryDirectory):
+            try:
+                self.work_directory.cleanup()
+            except Exception:  # pylint: disable=broad-except
+                logging.warning("We were unable to clean up the directory: %s", self.work_directory_name)
 
     def execute_docker_container(self):
         """Runs the Omega docker container with specific arguments."""
@@ -73,7 +79,7 @@ def execute_docker_container(self):
             "--rm",
             "-t",
             "-v",
-            f"{self.work_directory.name}:/opt/export",
+            f"{self.work_directory_name}:/opt/export",
             "--env-file",
             ".env",
             self.docker_container,
@@ -82,24 +88,24 @@ def execute_docker_container(self):
 
         # Write the command to a file so we can capture it later
         self.docker_cmdline = shlex.join(cmd)
-        with open(f"{self.work_directory.name}/top-execute-cmd.txt", "w", encoding="utf-8") as f:
+        with open(f"{self.work_directory_name}/top-execute-cmd.txt", "w", encoding="utf-8") as f:
             f.write(self.docker_cmdline)
 
         logging.debug("Running command: %s", cmd)
-        res = subprocess.Popen(  # nosec B603
+        with subprocess.Popen(  # nosec B603
             cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             encoding="utf-8",
             universal_newlines=True
-        )
-        for line in iter(res.stdout.readline, ""):
-            logging.debug(line.rstrip())
+        ) as res:
+            for line in iter(res.stdout.readline, ""):
+                logging.debug(line.rstrip())
 
-        res.stdout.close()
+            res.stdout.close()
 
-        if res.wait() != 0:
-            raise RuntimeError(f"Error running docker container: {res.stderr}")
+            if res.wait() != 0:
+                raise RuntimeError(f"Error running docker container: {res.stderr}")
 
     def _execute_assertion_noexcept(self, **kwargs):
         try:
@@ -119,7 +125,8 @@ def _execute_assertion(self, **kwargs):
             kwargs["expiration"] = datetime.strftime(
                 datetime.now() + timedelta(days=2 * 365), "%Y-%m-%dT%H:%M:%S.%fZ"
             )
-        cmd.append(f"--expiration={kwargs['expiration']}")
+        else:
+            cmd.append(f"--expiration={kwargs['expiration']}")
 
         if kwargs.get('signer'):
             cmd.append(f"--signer={kwargs['signer']}")
@@ -146,7 +153,7 @@ def _execute_assertion(self, **kwargs):
 
     def find_output_file(self, filename: str) -> str:
         """Finds a file in the output directory."""
-        for root, _, files in os.walk(self.work_directory.name):
+        for root, _, files in os.walk(self.work_directory_name):
             if filename in files:
                 return os.path.join(root, filename)
         return None
@@ -235,6 +242,7 @@ def execute_assertions(self):
     parser.add_argument(
         "--toolchain-container", required=False, default="openssf/omega-toolshed:latest"
     )
+    parser.add_argument('--work-directory', required=False, help='Use a specific working directory instead of a temporary one.')
     parser.add_argument(
         "--repository", required=True
     )
@@ -244,6 +252,6 @@ def execute_assertions(self):
     args = parser.parse_args()
 
     logging.info("Starting analysis runner")
-    runner = AnalysisRunner(args.package_url, args.toolchain_container, args.repository, args.signer)
+    runner = AnalysisRunner(args.package_url, args.toolchain_container, args.repository, args.signer, args.work_directory)
     runner.execute_docker_container()
     runner.execute_assertions()

omega/oaf/omega/assertion/utils.py

@@ -70,16 +70,22 @@ def find_repository(package_url: PackageURL | str) -> str | None:
             raise EnvironmentError("Invalid PackageURL provided.")
 
     if package_url.type == "github":
-        return purl2url(package_url)
+        try:
+            return purl2url(str(package_url))
+        except Exception:
+            logging.warning("Unable to parse PackageURL to GitHub repository: %s", str(package_url))
 
     if not is_command_available(["oss-find-source"]):
         raise EnvironmentError("oss-find-source is not available.")
 
-    cmd = ["oss-find-source", "-S", str(package_url)]
-    res = subprocess.run(cmd, check=False, capture_output=True, encoding="utf-8")  # nosec B603
-    if res.returncode == 0:
-        repository = res.stdout.strip()
-        return repository or None
+    try:
+        cmd = ["oss-find-source", "-S", str(package_url)]
+        res = subprocess.run(cmd, check=False, capture_output=True, encoding="utf-8")  # nosec B603
+        if res.returncode == 0:
+            repository = res.stdout.strip()
+            return repository or None
+    except Exception:
+        logging.warning("Failed to find repository for %s", str(package_url))
 
     return None