Merge branch 'resolver_exec'

Signed-off-by: Cleber Rosa <[email protected]>

avocado/plugins/list.py

@@ -214,6 +214,21 @@ def configure(self, parser):
             allow_multiple=True,
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
+        settings.add_argparser_to_option(
+            namespace="resolver.exec_runnables_recipe.arguments",
+            metavar="ARGS",
+            parser=parser,
+            long_arg="--resolver-exec-arguments",
+            allow_multiple=True,
+        )
+
         help_msg = "Writes runnable recipe files to a directory."
         settings.register_option(
             section="list.recipes",

avocado/plugins/resolvers.py

@@ -19,10 +19,12 @@
 import json
 import os
 import re
+import shlex
+import subprocess
 
 from avocado.core.extension_manager import PluginPriority
 from avocado.core.nrunner.runnable import Runnable
-from avocado.core.plugin_interfaces import Resolver
+from avocado.core.plugin_interfaces import Init, Resolver
 from avocado.core.references import reference_split
 from avocado.core.resolver import (
     ReferenceResolution,
@@ -31,16 +33,12 @@
     get_file_assets,
 )
 from avocado.core.safeloader import find_avocado_tests, find_python_unittests
+from avocado.core.settings import settings
 
 
-class ExecTestResolver(Resolver):
-
-    name = "exec-test"
-    description = "Test resolver for executable files to be handled as tests"
-    priority = PluginPriority.VERY_LOW
-
-    def resolve(self, reference):
-
+class BaseExec:
+    @staticmethod
+    def check_exec(reference):
         criteria_check = check_file(
             reference,
             reference,
@@ -52,6 +50,18 @@ def resolve(self, reference):
         if criteria_check is not True:
             return criteria_check
 
+
+class ExecTestResolver(BaseExec, Resolver):
+
+    name = "exec-test"
+    description = "Test resolver for executable files to be handled as tests"
+    priority = PluginPriority.VERY_LOW
+
+    def resolve(self, reference):
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
+
         runnable = Runnable("exec-test", reference, assets=get_file_assets(reference))
         return ReferenceResolution(
             reference, ReferenceResolutionResult.SUCCESS, [runnable]
@@ -121,24 +131,16 @@ def resolve(self, reference):
         )
 
 
-class TapResolver(Resolver):
+class TapResolver(BaseExec, Resolver):
 
     name = "tap"
     description = "Test resolver for executable files to be handled as TAP tests"
     priority = PluginPriority.LAST_RESORT
 
     def resolve(self, reference):
-
-        criteria_check = check_file(
-            reference,
-            reference,
-            suffix=None,
-            type_name="executable file",
-            access_check=os.R_OK | os.X_OK,
-            access_name="executable",
-        )
-        if criteria_check is not True:
-            return criteria_check
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
 
         runnable = Runnable("tap", reference, assets=get_file_assets(reference))
         return ReferenceResolution(
@@ -196,3 +198,102 @@ def resolve(self, reference):
             return criteria_check
 
         return self._validate_and_load_runnables(reference)
+
+
+class ExecRunnablesRecipeInit(Init):
+    name = "exec-runnables-recipe"
+    description = 'Configuration for resolver plugin "exec-runnables-recipe" plugin'
+
+    def initialize(self):
+        help_msg = (
+            'Whether resolvers (such as "exec-runnables-recipe") should '
+            "execute files given as test references that have executable "
+            "permissions. This is disabled by default due to security "
+            "implications of running executables that may not be trusted."
+        )
+        settings.register_option(
+            section="resolver",
+            key="run_executables",
+            key_type=bool,
+            default=False,
+            help_msg=help_msg,
+        )
+
+        help_msg = (
+            "Command line options (space separated) that will be added "
+            "to the executable when executing it as a producer of "
+            "runnables-recipe JSON content."
+        )
+        settings.register_option(
+            section="resolver.exec_runnables_recipe",
+            key="arguments",
+            key_type=str,
+            default="",
+            help_msg=help_msg,
+        )
+
+
+class ExecRunnablesRecipeResolver(BaseExec, Resolver):
+    name = "exec-runnables-recipe"
+    description = "Test resolver for executables that output JSON runnable recipes"
+    priority = PluginPriority.LOW
+
+    def resolve(self, reference):
+        if not self.config.get("resolver.run_executables"):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(
+                    "Running executables is not enabled. Refer to "
+                    '"resolver.run_executables" configuration option'
+                ),
+            )
+
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
+
+        args = self.config.get("resolver.exec_runnables_recipe.arguments")
+        if args:
+            cmd = [reference] + shlex.split(args)
+        else:
+            cmd = reference
+        try:
+            process = subprocess.Popen(
+                cmd,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except (FileNotFoundError, PermissionError) as exc:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(f'Failure while running running executable "{reference}": {exc}'),
+            )
+
+        content, _ = process.communicate()
+        try:
+            runnables = json.loads(content)
+        except json.JSONDecodeError:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f'Content generated by running executable "{reference}" is not JSON',
+            )
+
+        if not (
+            isinstance(runnables, list)
+            and all([isinstance(r, dict) for r in runnables])
+        ):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f"Content generated by running executable {reference} does not look like a runnables recipe JSON content",
+            )
+
+        return ReferenceResolution(
+            reference,
+            ReferenceResolutionResult.SUCCESS,
+            [Runnable.from_dict(r) for r in runnables],
+        )

avocado/plugins/run.py

@@ -267,6 +267,21 @@ def configure(self, parser):
             long_arg="--log-test-data-directories",
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
+        settings.add_argparser_to_option(
+            namespace="resolver.exec_runnables_recipe.arguments",
+            metavar="ARGS",
+            parser=parser,
+            long_arg="--resolver-exec-arguments",
+            allow_multiple=True,
+        )
+
         parser_common_args.add_tag_filter_args(parser)
 
     def run(self, config):

docs/source/guides/writer/chapters/recipes.rst

@@ -76,3 +76,93 @@ That will be parsed by the ``runnables-recipe`` resolver, like in
 
   exec-test /bin/true
   exec-test /bin/false
+
+Using dynamically generated recipes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``exec-runnables-recipe`` resolver allows a user to point to a
+file that will be executed, and that is expected to generate (on its
+``STDOUT``) content compatible with the Runnable recipe format
+mentioned previously.
+
+For security reasons, Avocado won't execute files indiscriminately
+when looking for tests (at the resolution phase). One must set the
+``--resolver-run-executables`` command line option (or the underlying
+``resolver.run_executables`` configuration option) to allow running
+executables at the resolver stage.
+
+.. warning:: It's the user's responsibility to give test references
+             (to be resolved and thus executed) that are well behaved
+             in the sense that they will finish executing quickly,
+             won't execute unintended code (such as running tests),
+             won't destroy data, etc.
+
+A script such as:
+
+.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+Will output JSON that is compatible with the runnable recipe format.
+That can be used directly via either ``avocado list`` or ``avocado
+run``.  Example::
+
+  $ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+  exec-test true-test
+  exec-test false-test
+
+If the executable to be run needs arguments, you can pass it via the
+``--resolver-exec-arguments`` or the underlying
+``resolver.exec_runnable_recipe.arguments`` option.  The following
+script receives an optional parameter that can change the type of the
+tests it generates:
+
+.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe_kind.sh
+
+In order to have those tests resolved as ``tap`` tests, one can run::
+
+  $ avocado list --resolver-run-executables --resolver-exec-arguments tap examples/nrunner/resolvers/exec_runnables_recipe_kind.sh
+
+  tap true-test
+  tap false-test
+
+Behavior of ``exec-runnables-recipe`` and ``exec-test`` resolvers
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+The ``exec-runnables-recipe`` resolver has a higher priority than
+(that is, it runs before) the ``exec-test`` resolver.  That means that
+if, and only if, a user enables the feature itself (by means of the
+``--resolver-run-executables`` command line option or the underlying
+``resolver.run_executables`` configuration option), it
+``exec-runnables-recipe`` will perform any meaningful action.
+
+Even if the ``exec-runnables-recipe`` is activated (through the
+command line or configuration option mentioned before), it may still
+coexist with ``exec-test`` resolver, example::
+
+  $ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh /bin/uname
+
+  exec-test true-test
+  exec-test false-test
+  exec-test /bin/uname
+
+The reason (that can be seen with ``avocado -V list ...``) for that is
+the ``exec-runnables-recipe`` returns a "not found" resolution with
+the message::
+
+  Resolver              Reference  Info
+  ...
+  exec-runnables-recipe /bin/uname Content generated by running executable "/bin/uname" is not JSON
+
+.. warning:: Even though it's possible to have ``exec-test`` and
+             ``exec-runnable-recipes`` in the same Avocado test suite
+             (for instance in an ``avocado run`` command execution)
+             it's not recommended on most cases because ``exec-tests``
+             will end up being run at the test resolution phase
+             in addition to the test execution phase.  It's
+             recommended to use multiple ``avocado run``
+             commands or use the Job API and multiple
+             :class:`avocado.core.suite.TestSuite`, one for
+             ``exec-runnable-recipes`` with the
+             ``resolver.run_executables`` options enabled, and
+             another for ``exec-tests`` with that option in its
+             default state (disabled).

examples/nrunner/resolvers/exec_runnables_recipe.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo '[{"kind": "exec-test","uri": "/bin/true","identifier": "true-test"},{"kind": "exec-test","uri": "/bin/false","identifier": "false-test"}]'

examples/nrunner/resolvers/exec_runnables_recipe_kind.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+kind=${1:-exec-test}
+echo "[{\"kind\": \"$kind\",\"uri\": \"/bin/true\",\"identifier\": \"true-test\"},{\"kind\": \"$kind\",\"uri\": \"/bin/false\",\"identifier\": \"false-test\"}]"

selftests/check.py

@@ -29,7 +29,7 @@
     "nrunner-requirement": 28,
     "unit": 678,
     "jobs": 11,
-    "functional-parallel": 309,
+    "functional-parallel": 312,
     "functional-serial": 7,
     "optional-plugins": 0,
     "optional-plugins-golang": 2,