From 6241ff05d9d85a988bf02fc030d161fcea75e336 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Thu, 16 May 2024 13:31:18 +0200 Subject: [PATCH 1/6] Add flag --used-rules --- build-scripts/profile_tool.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 043d2a3e413..51ae7f237d7 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -314,6 +314,12 @@ def parse_most_used_components(subparsers): choices=get_available_products_with_components_root(), default=get_available_products_with_components_root(), ) + parser_most_used_components.add_argument( + "--used-rules", + default=False, + action="store_true", + help="Show used rules of component.", + ) def get_available_products_with_components_root(): From 3f3f2beb928a8fbe5a5e194c0804a795c353009d Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Thu, 16 May 2024 13:39:15 +0200 Subject: [PATCH 2/6] Add used rules of component --- utils/profile_tool/common.py | 16 +++++++++ utils/profile_tool/most_used_components.py | 41 ++++++++++++++++++---- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/utils/profile_tool/common.py b/utils/profile_tool/common.py index ae5729451f0..8eea054491f 100644 --- a/utils/profile_tool/common.py +++ b/utils/profile_tool/common.py @@ -13,3 +13,19 @@ def generate_output(dict_, format, csv_header): for rule_id, rule_count in dict_.items(): print(f_string.format(rule_id, rule_count)) + + +def _format_value_b(value_b, delim): + str_ = "" + if len(value_b) != 0: + values = ", ".join([f"{key}: {value}" for key, value in value_b.items()]) + str_ = f"{delim}{values}" + return str_ + + +def merge_dicts(dict_a, dict_b, delim): + out = {} + for key, value in dict_a.items(): + value_b = dict_b.get(key, {}) + out[key] = str(value) + _format_value_b(value_b, delim) + return out diff --git a/utils/profile_tool/most_used_components.py b/utils/profile_tool/most_used_components.py index 1ca9b9551a4..6c22fb3874b 100644 --- a/utils/profile_tool/most_used_components.py +++ b/utils/profile_tool/most_used_components.py @@ -5,7 +5,7 @@ import ssg.components from .most_used_rules import _sorted_dict_by_num_value -from .common import generate_output +from .common import generate_output, merge_dicts PYTHON_2 = sys.version_info[0] < 3 @@ -17,10 +17,21 @@ ) -def _count_components(components, rules_list, components_out): +def _count_rules_components(component_name, rules, used_rules_of_components_out): + used_rules = defaultdict(int) + if component_name in used_rules_of_components_out: + used_rules = used_rules_of_components_out[component_name] + for rule in rules: + used_rules[rule] += 1 + used_rules_of_components_out[component_name] = used_rules + + +def _count_components(components, rules_list, components_out, used_rules_of_components_out): for component_name, component in components.items(): - if len(set(component.rules).intersection(set(rules_list))) > 0: + intersection = set(component.rules).intersection(set(rules_list)) + if len(intersection) > 0: components_out[component_name] += 1 + _count_rules_components(component_name, intersection, used_rules_of_components_out) def load_components(product): @@ -33,7 +44,7 @@ def load_components(product): return ssg.components.load(components_dir) -def _process_all_products_from_controls(components_out, products): +def _process_all_products_from_controls(components_out, used_rules_of_components_out, products): if PYTHON_2: raise Exception("This feature is not supported for python2.") @@ -43,14 +54,32 @@ def _process_all_products_from_controls(components_out, products): continue controls_manager = load_controls_manager("./controls/", product) for profile in _get_profiles_for_product(controls_manager, product): - _count_components(components, profile.rules, components_out) + _count_components( + components, profile.rules, components_out, used_rules_of_components_out + ) + + +def _sort_rules_of_components(used_rules_of_components): + out = {} + for key, value in used_rules_of_components.items(): + out[key] = _sorted_dict_by_num_value(value) + return out def command_most_used_components(args): components = defaultdict(int) + used_rules_of_components = {} - _process_all_products_from_controls(components, args.products) + _process_all_products_from_controls(components, used_rules_of_components, args.products) sorted_components = _sorted_dict_by_num_value(components) csv_header = "component_name,count_of_profiles" + if args.used_rules: + csv_header = "component_name,count_of_profiles,used_rules:count_of_profiles" + delim = " " + if args.format == "csv": + delim = "," + sorted_used_rules_of_components = _sort_rules_of_components(used_rules_of_components) + sorted_components = merge_dicts(sorted_components, sorted_used_rules_of_components, delim) + generate_output(sorted_components, args.format, csv_header) From db3a8ebc58d3f081f483c108eda699a8b4fab5f7 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Thu, 16 May 2024 15:30:24 +0200 Subject: [PATCH 3/6] Add new flag to documentation --- docs/manual/developer/05_tools_and_utilities.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/manual/developer/05_tools_and_utilities.md b/docs/manual/developer/05_tools_and_utilities.md index 64d03b0b014..22b31d50380 100644 --- a/docs/manual/developer/05_tools_and_utilities.md +++ b/docs/manual/developer/05_tools_and_utilities.md @@ -79,6 +79,13 @@ Optionally, you can use this command to limit the statistics for a specific prod $ ./build-scripts/profile_tool.py most-used-components --products rhel9 ``` +You can also get a list of the most used components with used rules for the RHEL9 product, you can use the `--used-rules` flag. +As shown in this command: + +```bash + $ ./build-scripts/profile_tool.py most-used-components --products rhel9 --used-rules +``` + The result will be a list of rules with the number of uses in the profiles. The list can be generated as plain text, JSON or CVS. Via the `--format FORMAT` parameter. From 9e50f3c45d39745fde66cad748ca472149e4520b Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 17 May 2024 16:32:38 +0200 Subject: [PATCH 4/6] Fix missing parents levels --- utils/profile_tool/profile.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/profile_tool/profile.py b/utils/profile_tool/profile.py index e73b1145773..14b5be53140 100644 --- a/utils/profile_tool/profile.py +++ b/utils/profile_tool/profile.py @@ -74,10 +74,14 @@ def _get_sel(selected): return policy, control, level @staticmethod - def _get_levels(policy, level): - levels = [level] + def _get_levels(policy, level, levels=None): + if levels is None: + levels = set() + levels.add(level) + if policy.levels_by_id.get(level).inherits_from is not None: - levels.extend(policy.levels_by_id.get(level).inherits_from) + for parent_level in policy.levels_by_id.get(level).inherits_from: + levels.update(Profile._get_levels(policy, parent_level, levels)) return levels def add_from_policy(self, policies, selected): From 174d6cc9fc9d9584682311da2ca6ba330d3f4117 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 17 May 2024 16:59:45 +0200 Subject: [PATCH 5/6] Update flag help message --- build-scripts/profile_tool.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 51ae7f237d7..ba70e56db5d 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -318,7 +318,10 @@ def parse_most_used_components(subparsers): "--used-rules", default=False, action="store_true", - help="Show used rules of component.", + help=( + "For every component, show the usage of each component's rule " + "in the profiles in given product." + ), ) From 95be94a66b082d50c1abe94cae5bebf3529ecb64 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 22 May 2024 16:16:32 +0200 Subject: [PATCH 6/6] Skip default profiles --- utils/profile_tool/most_used_rules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/profile_tool/most_used_rules.py b/utils/profile_tool/most_used_rules.py index 85cd5bd5185..6af5cf9f7b7 100644 --- a/utils/profile_tool/most_used_rules.py +++ b/utils/profile_tool/most_used_rules.py @@ -32,7 +32,8 @@ def _get_profiles_for_product(ctrls_mgr, product): profiles = [] for file in profiles_files: - profiles.append(get_profile(profiles_files, file, ctrls_mgr.policies)) + if "default.profile" not in file: + profiles.append(get_profile(profiles_files, file, ctrls_mgr.policies)) return profiles