Add support for exclude patterns

README.md

@@ -80,6 +80,7 @@ The distances can also be used to produce a dendrogram, showing the result of hi
 - jsonschema
 - Matplotlib
 - NumPy
+- pathspec
 - Python 3
 - PyYAML
 - SciPy

codebasin.py

@@ -109,6 +109,16 @@ def guess_project_name(config_path):
         default=False,
         help="Set batch mode (additional output for bulk operation.)",
     )
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        dest="excludes",
+        metavar="<pattern>",
+        action="append",
+        default=[],
+        help="Exclude files matching this pattern from the code base. "
+        + "May be specified multiple times.",
+    )
     args = parser.parse_args()
 
     stdout_log = logging.StreamHandler(sys.stdout)
@@ -129,7 +139,11 @@ def guess_project_name(config_path):
             "Configuration file does not have YAML file extension.",
         )
         sys.exit(1)
-    codebase, configuration = config.load(config_file, rootdir)
+    codebase, configuration = config.load(
+        config_file,
+        rootdir,
+        exclude_patterns=args.excludes,
+    )
 
     # Parse the source tree, and determine source line associations.
     # The trees and associations are housed in state.

codebasin/config.py

@@ -12,6 +12,7 @@
 import os
 import re
 import shlex
+import warnings
 
 import yaml
 
@@ -102,7 +103,7 @@ def flatten(nested_list):
     return flattened
 
 
-def load_codebase(config, rootdir):
+def load_codebase(config, rootdir, *, exclude_patterns=None):
     """
     Load the code base definition into a Python object.
     Return a dict of files and platform names.
@@ -131,9 +132,20 @@ def load_codebase(config, rootdir):
                 ),
             ),
         )
+        warnings.warn(
+            "'exclude_files' is deprecated. Use 'exclude_pattern' instead.",
+        )
     else:
         codebase["exclude_files"] = frozenset([])
 
+    if "exclude_patterns" in cfg_codebase:
+        codebase["exclude_patterns"] = cfg_codebase["exclude_patterns"]
+    else:
+        codebase["exclude_patterns"] = []
+
+    if exclude_patterns:
+        codebase["exclude_patterns"] += exclude_patterns
+
     if cfg_codebase["files"]:
         codebase["files"] = list(
             it.chain(
@@ -559,7 +571,7 @@ def load_platform(config, rootdir, platform_name):
     return configuration
 
 
-def load(config_file, rootdir):
+def load(config_file, rootdir, *, exclude_patterns=None):
     """
     Load the configuration file into Python objects.
     Return a (codebase, platform configuration) tuple of dicts.
@@ -575,7 +587,11 @@ def load(config_file, rootdir):
 
     # Read codebase definition
     if "codebase" in config:
-        codebase = load_codebase(config, rootdir)
+        codebase = load_codebase(
+            config,
+            rootdir,
+            exclude_patterns=exclude_patterns,
+        )
     else:
         raise RuntimeError("Missing 'codebase' section in config file!")
 

codebasin/schema/config.schema

@@ -25,6 +25,12 @@
           "items": {
             "type": "string"
           }
+        },
+        "exclude_patterns": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
         }
       },
       "required": [

codebasin/walkers/platform_mapper.py

@@ -4,23 +4,38 @@
 import logging
 import os
 
+import pathspec
+
 from codebasin.preprocessor import CodeNode, FileNode
 from codebasin.walkers.tree_mapper import TreeMapper
 
 log = logging.getLogger("codebasin")
 
 
 def exclude(filename, cb):
+    # Always exclude files that were explicitly listed as excluded.
     if filename in cb["exclude_files"]:
         log.info(f"Excluding {filename}; matches 'exclude_files'.")
         return True
-    elif filename in cb["files"]:
-        return False
-    else:
-        path = os.path.realpath(filename)
-        if not path.startswith(cb["rootdir"]):
-            log.info(f"Excluding {filename}; outside of root directory.")
-            return True
+
+    # Only exclude files outside of the root directory if they weren't
+    # explicitly listed as part of the codebase.
+    path = os.path.realpath(filename)
+    if not path.startswith(cb["rootdir"]):
+        if filename in cb["files"]:
+            return False
+        log.info(f"Excluding {filename}; outside of root directory.")
+        return True
+
+    # Exclude files matching an exclude pattern.
+    #
+    # Use GitIgnoreSpec to match git behavior in weird corner cases.
+    # Convert relative paths to match .gitignore subdirectory behavior.
+    spec = pathspec.GitIgnoreSpec.from_lines(cb["exclude_patterns"])
+    rel = os.path.relpath(path, cb["rootdir"])
+    if spec.match_file(rel):
+        log.info(f"Excluding {filename}; matches exclude pattern.")
+        return True
 
     return False
 

setup.py

@@ -26,6 +26,7 @@
     install_requires=[
         "numpy",
         "matplotlib",
+        "pathspec",
         "pyyaml",
         "scipy>=1.11.1",
         "jsonschema",

tests/exclude/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) 2019-2024 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause

tests/exclude/commands.json

@@ -0,0 +1,18 @@
+[
+  {
+    "command": "/usr/bin/g++ included.cpp",
+    "file": "src/included.cpp"
+  },
+  {
+    "command": "/usr/bin/g++ excluded_name.cpp",
+    "file": "src/excluded_name.cpp"
+  },
+  {
+    "command": "/usr/bin/gfortran excluded_extension.f90",
+    "file": "src/excluded_extension.f90"
+  },
+  {
+    "command": "/usr/bin/g++ library.cpp",
+    "file": "src/thirdparty/library.cpp"
+  }
+]

tests/exclude/exclude.yaml

@@ -0,0 +1,6 @@
+codebase:
+    files: [ ]
+    platforms: [ test ]
+
+test:
+    commands: "commands.json"

tests/exclude/src/excluded_extension.f90

@@ -0,0 +1 @@
+#define EXCLUDED_EXTENSION

tests/exclude/src/excluded_name.cpp

@@ -0,0 +1 @@
+#define EXCLUDED_NAME

tests/exclude/src/included.cpp

@@ -0,0 +1 @@
+#define INCLUDED

tests/exclude/src/thirdparty/library.cpp

@@ -0,0 +1 @@
+#define THIRDPARTY_LIBRARY

tests/exclude/test_exclude.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2019-2024 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+
+import logging
+import unittest
+
+from codebasin import config, finder
+from codebasin.walkers.platform_mapper import PlatformMapper
+
+
+class TestExclude(unittest.TestCase):
+    """
+    Simple test of ability to exclude files using patterns.
+    """
+
+    def setUp(self):
+        self.rootdir = "./tests/exclude/"
+        logging.getLogger("codebasin").disabled = True
+
+    def _get_setmap(self, excludes):
+        codebase, configuration = config.load(
+            "./tests/exclude/exclude.yaml",
+            self.rootdir,
+            exclude_patterns=excludes,
+        )
+        state = finder.find(self.rootdir, codebase, configuration)
+        mapper = PlatformMapper(codebase)
+        setmap = mapper.walk(state)
+        return setmap
+
+    def test_exclude_nothing(self):
+        """exclude/nothing"""
+        excludes = []
+        setmap = self._get_setmap(excludes)
+        expected_setmap = {frozenset(["test"]): 4}
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+    def test_exclude_extension(self):
+        """exclude/extension"""
+        excludes = ["*.f90"]
+        setmap = self._get_setmap(excludes)
+        expected_setmap = {frozenset(["test"]): 3}
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+    def test_exclude_name(self):
+        """exclude/name"""
+        excludes = ["src/excluded_name.cpp"]
+        setmap = self._get_setmap(excludes)
+        expected_setmap = {frozenset(["test"]): 3}
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+    def test_excluded_directory(self):
+        excludes = ["thirdparty/"]
+        setmap = self._get_setmap(excludes)
+        expected_setmap = {frozenset(["test"]): 3}
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+    def test_excludes(self):
+        excludes = ["*.f90", "src/excluded_name.cpp", "thirdparty/"]
+        setmap = self._get_setmap(excludes)
+        expected_setmap = {frozenset(["test"]): 1}
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+
+if __name__ == "__main__":
+    unittest.main()