Merge branch 'master' into nilclass-publish

.github/workflows/auto_cherry_pick.yml

@@ -78,13 +78,10 @@ jobs:
         id: add-parent-prt-comment
         if: ${{ always() && steps.cherrypick.outcome == 'success' }}
         run: |
-          ISSUE_NUMBER=${{ steps.cherrypick.outputs.number }}
-          COMMENT_BODY=${{ needs.find-the-parent-prt-comment.outputs.prt_comment }}
-
           curl -X POST -H "Authorization: token ${{ secrets.CHERRYPICK_PAT }}" \
             -H "Accept: application/vnd.github.v3+json" \
-            "https://api.github.com/repos/${{ github.repository }}/issues/${ISSUE_NUMBER}/comments" \
-            -d "{\"body\":\"$COMMENT_BODY\"}"
+            "https://api.github.com/repos/${{ github.repository }}/issues/${{ steps.cherrypick.outputs.number }}/comments" \
+            -d "{\"body\":\"${{ needs.find-the-parent-prt-comment.outputs.prt_comment }}\"}"
 
       - name: is autoMerging enabled for Auto CherryPicked PRs ?
         if: ${{ always() && steps.cherrypick.outcome == 'success' && contains(github.event.pull_request.labels.*.name, 'AutoMerge_Cherry_Picked') }}

.github/workflows/pull_request.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set Up Python-${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/stale.yml

@@ -16,7 +16,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-issue-stale: 90

.github/workflows/weekly.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set Up Python-${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

conf/capsule.yaml.template

@@ -10,6 +10,8 @@ CAPSULE:
     # The base os rhel version where the capsule installed
     # RHEL_VERSION:
   # The Ansible Tower workflow used to deploy a capsule
-  DEPLOY_WORKFLOW: deploy-capsule
+  DEPLOY_WORKFLOWS:
+    PRODUCT: deploy-capsule  # workflow to deploy OS with product running on top of it
+    OS: deploy-rhel  # workflow to deploy OS that is ready to run the product
   # Dictionary of arguments which should be passed along to the deploy workflow
   DEPLOY_ARGUMENTS:

conf/dynaconf_hooks.py

@@ -1,5 +1,9 @@
+from inspect import getmembers, isfunction
 import json
 from pathlib import Path
+import sys
+
+from box import Box
 
 from robottelo.logging import logger
 from robottelo.utils.ohsnap import dogfood_repository
@@ -22,6 +26,7 @@ def post(settings):
             )
             data = get_repos_config(settings)
             write_cache(settings_cache_path, data)
+    config_migrations(settings, data)
     data['dynaconf_merge'] = True
     return data
 
@@ -33,7 +38,32 @@ def write_cache(path, data):
 
 def read_cache(path):
     logger.info(f'Using settings cache file: {path}')
-    return json.loads(path.read_text())
+    return Box(json.loads(path.read_text()))
+
+
+def config_migrations(settings, data):
+    """Run config migrations
+
+    Fetch the config migrations from the conf/migrations.py file and run them.
+
+    :param settings: dynaconf settings object
+    :type settings: LazySettings
+    :param data: settings data to be merged with the rest of the settings
+    :type data: dict
+    """
+    logger.info('Running config migration hooks')
+    sys.path.append(str(Path(__file__).parent))
+    from conf import migrations
+
+    migration_functions = [
+        mf for mf in getmembers(migrations, isfunction) if mf[0].startswith('migration_')
+    ]
+    # migration_functions is a sorted list of tuples (name, function)
+    for name, func in migration_functions:
+        logger.debug(f'Running {name}')
+        func(settings, data)
+        logger.debug(f'Finished running {name}')
+    logger.info('Finished running config migration hooks')
 
 
 def get_repos_config(settings):
@@ -46,7 +76,7 @@ def get_repos_config(settings):
             'The Ohsnap URL is invalid! Post-configuration hooks will not run. '
             'Default configuration will be used.'
         )
-    return {'REPOS': data}
+    return Box({'REPOS': data})
 
 
 def get_ohsnap_repos(settings):

conf/migrations.py

@@ -0,0 +1,37 @@
+"""Robottelo configuration migrations
+
+This module contains functions that are run after the configuration is loaded. Each function
+should be named `migration_<YYMMDD>_<description>` and accept two parameters: `settings` and
+`data`. `settings` is a `dynaconf` `Box` object containing the configuration. `data` is a
+`dict` that can be used to store settings that will be merged with the rest of the settings.
+The functions should not return anything.
+"""
+
+from packaging.version import Version
+
+from robottelo.logging import logger
+
+
+def migration_231129_deploy_workflow(settings, data):
+    """Migrates {server,capsule}.deploy_workflow to {server,capsule}.deploy_workflows"""
+    for product_type in ['server', 'capsule']:
+        # If the product_type has a deploy_workflow and it is a string, and
+        # it does not have a deploy_workflows set
+        if (
+            settings[product_type].get('deploy_workflow')
+            and isinstance(settings[product_type].deploy_workflow, str)
+            and not settings[product_type].get('deploy_workflows')
+        ):
+            sat_rhel_version = settings[product_type].version.rhel_version
+            data[product_type] = {}
+            # Set the deploy_workflows to a dict with product and os keys
+            # Get the OS workflow from the content_host config
+            data[product_type].deploy_workflows = {
+                'product': settings[product_type].deploy_workflow,
+                'os': settings.content_host[
+                    f'rhel{Version(str(sat_rhel_version)).major}'
+                ].vm.workflow,
+            }
+            logger.info(
+                f'Migrated {product_type}.DEPLOY_WORKFLOW to {product_type}.DEPLOY_WORKFLOWS'
+            )

conf/server.yaml.template

@@ -27,7 +27,9 @@ SERVER:
   # this setting determines if they will be automatically checked in
   AUTO_CHECKIN: False
   # The Ansible Tower workflow used to deploy a satellite
-  DEPLOY_WORKFLOW: "deploy-sat-jenkins"
+  DEPLOY_WORKFLOWS:
+    PRODUCT: deploy-satellite  # workflow to deploy OS with product running on top of it
+    OS: deploy-rhel  # workflow to deploy OS that is ready to run the product
   # Dictionary of arguments which should be passed along to the deploy workflow
   # DEPLOY_ARGUMENTS:
   # HTTP scheme when building the server URL

conftest.py

@@ -20,6 +20,7 @@
     'pytest_plugins.factory_collection',
     'pytest_plugins.requirements.update_requirements',
     'pytest_plugins.sanity_plugin',
+    'pytest_plugins.video_cleanup',
     # Fixtures
     'pytest_fixtures.core.broker',
     'pytest_fixtures.core.sat_cap_factory',

pytest_fixtures/core/sat_cap_factory.py

@@ -63,7 +63,7 @@ def factory(retry_limit=3, delay=300, workflow=None, **broker_args):
 
         vmb = Broker(
             host_class=Satellite,
-            workflow=workflow or settings.server.deploy_workflow,
+            workflow=workflow or settings.server.deploy_workflows.product,
             **broker_args,
         )
         timeout = (1200 + delay) * retry_limit
@@ -95,7 +95,7 @@ def factory(retry_limit=3, delay=300, workflow=None, **broker_args):
             broker_args.update(settings.capsule.deploy_arguments)
         vmb = Broker(
             host_class=Capsule,
-            workflow=workflow or settings.capsule.deploy_workflow,
+            workflow=workflow or settings.capsule.deploy_workflows.product,
             **broker_args,
         )
         timeout = (1200 + delay) * retry_limit
@@ -202,7 +202,7 @@ def module_lb_capsule(retry_limit=3, delay=300, **broker_args):
         timeout = (1200 + delay) * retry_limit
         hosts = Broker(
             host_class=Capsule,
-            workflow=settings.capsule.deploy_workflow,
+            workflow=settings.capsule.deploy_workflows.product,
             _count=2,
             **broker_args,
         )
@@ -245,12 +245,13 @@ def parametrized_enrolled_sat(
 
 
 def get_deploy_args(request):
+    """Get deploy arguments for Satellite base OS deployment. Should not be used for Capsule."""
     rhel_version = get_sat_rhel_version()
     deploy_args = {
         'deploy_rhel_version': rhel_version.base_version,
         'deploy_flavor': settings.flavors.default,
         'promtail_config_template_file': 'config_sat.j2',
-        'workflow': settings.content_host.get(f'rhel{rhel_version.major}').vm.workflow,
+        'workflow': settings.server.deploy_workflows.os,
     }
     if hasattr(request, 'param'):
         if isinstance(request.param, dict):
@@ -277,11 +278,11 @@ def module_sat_ready_rhels(request):
 @pytest.fixture
 def cap_ready_rhel():
     rhel_version = Version(settings.capsule.version.release)
-    settings.content_host.get(f'rhel{rhel_version.major}').vm.workflow
     deploy_args = {
         'deploy_rhel_version': rhel_version.base_version,
         'deploy_flavor': settings.flavors.default,
-        'workflow': settings.content_host.get(f'rhel{rhel_version.major}').vm.workflow,
+        'promtail_config_template_file': 'config_sat.j2',
+        'workflow': settings.capsule.deploy_workflows.os,
     }
     with Broker(**deploy_args, host_class=Capsule) as host:
         yield host

pytest_plugins/video_cleanup.py

@@ -0,0 +1,74 @@
+from urllib.parse import urlparse
+
+from box import Box
+from broker.hosts import Host
+import pytest
+
+from robottelo.config import settings
+from robottelo.logging import logger
+
+test_results = {}
+test_directories = [
+    'tests/foreman/destructive',
+    'tests/foreman/ui',
+    'tests/foreman/sanity',
+    'tests/foreman/virtwho',
+]
+
+
+def _clean_video(session_id, test):
+    logger.info(f"cleaning up video files for session: {session_id} and test: {test}")
+
+    if settings.ui.grid_url and session_id:
+        grid = urlparse(url=settings.ui.grid_url)
+        infra_grid = Host(hostname=grid.hostname)
+        infra_grid.execute(command=f'rm -rf /var/www/html/videos/{session_id}')
+        logger.info(f"video cleanup for session {session_id} is complete")
+    else:
+        logger.warning("missing grid_url or session_id. unable to clean video files.")
+
+
+def pytest_addoption(parser):
+    """Custom pytest option to skip video cleanup on test success.
+    Args:
+        parser (object): The pytest command-line option parser.
+    Options:
+        --skip-video-cleanup: Skip video cleaning on test success (default: False).
+    """
+    parser.addoption(
+        "--skip-video-cleanup",
+        action="store_true",
+        default=False,
+        help="Skip video cleaning on test success",
+    )
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item):
+    """Custom pytest hook to capture test outcomes and perform video cleanup.
+    Note:
+        This hook captures test results during 'call' and performs video cleanup
+        during 'teardown' if the test passed and the '--skip-video-cleanup' option is not set.
+    """
+    outcome = yield
+    report = outcome.get_result()
+    skip_video_cleanup = item.config.getoption("--skip-video-cleanup", False)
+
+    if not skip_video_cleanup and any(directory in report.fspath for directory in test_directories):
+        if report.when == "call":
+            test_results[item.nodeid] = Box(
+                {
+                    'outcome': report.outcome,
+                    'duration': report.duration,
+                    'longrepr': str(report.longrepr),
+                }
+            )
+        if report.when == "teardown":
+            if item.nodeid in test_results:
+                result_info = test_results[item.nodeid]
+                if result_info.outcome == 'passed':
+                    session_id_tuple = next(
+                        (t for t in report.user_properties if t[0] == 'session_id'), None
+                    )
+                    session_id = session_id_tuple[1] if session_id_tuple else None
+                    _clean_video(session_id, item.nodeid)

requirements-optional.txt

@@ -2,7 +2,7 @@
 flake8==6.1.0
 pytest-cov==4.1.0
 redis==5.0.1
-pre-commit==3.5.0
+pre-commit==3.6.0
 
 # For generating documentation.
 sphinx==7.2.6

requirements.txt

@@ -8,15 +8,15 @@ dynaconf[vault]==3.2.4
 fauxfactory==3.1.0
 jinja2==3.1.2
 manifester==0.0.14
-navmazing==1.1.6
-productmd==1.37
+navmazing==1.2.2
+productmd==1.38
 pyotp==2.9.0
 python-box==7.1.1
 pytest==7.4.3
 pytest-order==1.2.0
 pytest-services==2.2.1
 pytest-mock==3.12.0
-pytest-reportportal==5.3.0
+pytest-reportportal==5.3.1
 pytest-xdist==3.5.0
 pytest-fixturecollection==0.1.1
 pytest-ibutsu==2.2.4

robottelo/cli/user.py

@@ -105,3 +105,29 @@ def access_token(cls, action=None, options=None):
         """
         cls.command_sub = f'access-token {action}'
         return cls.execute(cls._construct_command(options), output_format='csv')
+
+    @classmethod
+    def mail_notification_add(cls, options=None):
+        """
+        Usage:
+        hammer user mail-notification add [OPTIONS]
+
+        Options::
+
+            --interval VALUE                        Mail notification interval option, e.g. Daily,
+                                                    Weekly or Monthly.
+                                                    Required for summary notification
+            --location[-id|-title] VALUE/NUMBER     Set the current location context for the request
+            --mail-notification[-id] VALUE/NUMBER
+            --mail-query VALUE                      Relevant only for audit summary notification
+            --organization[-id|-title] VALUE/NUMBER Set the current organization context
+                                                    for the request
+            --subscription VALUE                    Mail notification subscription option, e.g.
+                                                    Subscribe, Subscribe to my hosts or
+                                                    Subscribe to all hosts. Required for host built
+                                                    and config error state
+            --user[-id] VALUE
+
+        """
+        cls.command_sub = 'mail-notification add'
+        return cls.execute(cls._construct_command(options), output_format='csv')

robottelo/config/validators.py

@@ -23,7 +23,9 @@
         ),
         Validator('server.admin_password', default='changeme'),
         Validator('server.admin_username', default='admin'),
-        Validator('server.deploy_workflow', must_exist=True),
+        Validator('server.deploy_workflows', must_exist=True, is_type_of=dict),
+        Validator('server.deploy_workflows.product', must_exist=True),
+        Validator('server.deploy_workflows.os', must_exist=True),
         Validator('server.deploy_arguments', must_exist=True, is_type_of=dict, default={}),
         Validator('server.scheme', default='https'),
         Validator('server.port', default=443),
@@ -67,7 +69,9 @@
     capsule=[
         Validator('capsule.version.release', must_exist=True),
         Validator('capsule.version.source', must_exist=True),
-        Validator('capsule.deploy_workflow', must_exist=True),
+        Validator('capsule.deploy_workflows', must_exist=True, is_type_of=dict),
+        Validator('capsule.deploy_workflows.product', must_exist=True),
+        Validator('capsule.deploy_workflows.os', must_exist=True),
         Validator('capsule.deploy_arguments', must_exist=True, is_type_of=dict, default={}),
     ],
     certs=[

robottelo/constants/__init__.py

@@ -1739,6 +1739,7 @@ class Colored(Box):
 HAMMER_CONFIG = "~/.hammer/cli.modules.d/foreman.yml"
 HAMMER_SESSIONS = "~/.hammer/sessions"
 
+INSTALLER_CONFIG_FILE = '/etc/foreman-installer/scenarios.d/satellite.yaml'
 SATELLITE_ANSWER_FILE = "/etc/foreman-installer/scenarios.d/satellite-answers.yaml"
 CAPSULE_ANSWER_FILE = "/etc/foreman-installer/scenarios.d/capsule-answers.yaml"
 MAINTAIN_HAMMER_YML = "/etc/foreman-maintain/foreman-maintain-hammer.yml"

robottelo/host_helpers/cli_factory.py

@@ -179,6 +179,8 @@ def create_object(cli_object, options, values=None, credentials=None):
     },
     'compute_resource': {
         'name': gen_alphanumeric,
+        'provider': 'Libvirt',
+        'url': 'qemu+tcp://localhost:16509/system',
     },
     'org': {'_redirect': 'org_with_credentials'},
     'org_with_credentials': {