test(flagd): improve gherkin setup based on current motivation -wip

providers/openfeature-provider-flagd/pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
 keywords = []
 dependencies = [
   "openfeature-sdk>=0.6.0",
-  "grpcio>=1.68.0",
+  "grpcio>=1.68.1",
   "protobuf>=4.25.2",
   "mmh3>=4.1.0",
   "panzi-json-logic>=1.0.1",

providers/openfeature-provider-flagd/pytest.ini

@@ -4,7 +4,15 @@ markers =
     in-process: tests for rpc mode.
     customCert: Supports custom certs.
     unixsocket: Supports unixsockets.
+    targetURI: Supports targetURI.
+    grace: Supports grace attempts.
+    targeting: Supports targeting.
+    fractional: Supports fractional.
+    string: Supports string.
+    semver: Supports semver.
+    reconnect: Supports reconnect.
     events: Supports events.
     sync: Supports sync.
     caching: Supports caching.
     offline: Supports offline.
+bdd_features_base_dir = tests/features

providers/openfeature-provider-flagd/src/openfeature/contrib/provider/flagd/provider.py

@@ -43,14 +43,14 @@
         host: typing.Optional[str] = None,
         port: typing.Optional[int] = None,
         tls: typing.Optional[bool] = None,
-        deadline: typing.Optional[int] = None,
+        deadline_ms: typing.Optional[int] = None,
         timeout: typing.Optional[int] = None,
         retry_backoff_ms: typing.Optional[int] = None,
         resolver_type: typing.Optional[ResolverType] = None,
         offline_flag_source_path: typing.Optional[str] = None,
         stream_deadline_ms: typing.Optional[int] = None,
         keep_alive_time: typing.Optional[int] = None,
-        cache_type: typing.Optional[CacheType] = None,
+        cache: typing.Optional[CacheType] = None,
         max_cache_size: typing.Optional[int] = None,
         retry_backoff_max_ms: typing.Optional[int] = None,
         retry_grace_period: typing.Optional[int] = None,
@@ -62,16 +62,16 @@
         :param host: the host to make requests to
         :param port: the port the flagd service is available on
         :param tls: enable/disable secure TLS connectivity
-        :param deadline: the maximum to wait before a request times out
+        :param deadline_ms: the maximum to wait before a request times out
         :param timeout: the maximum time to wait before a request times out
         :param retry_backoff_ms: the number of milliseconds to backoff
         :param offline_flag_source_path: the path to the flag source file
         :param stream_deadline_ms: the maximum time to wait before a request times out
         :param keep_alive_time: the number of milliseconds to keep alive
         :param resolver_type: the type of resolver to use
         """
-        if deadline is None and timeout is not None:
-            deadline = timeout * 1000
+        if deadline_ms is None and timeout is not None:
+            deadline_ms = timeout * 1000
             warnings.warn(
                 "'timeout' property is deprecated, please use 'deadline' instead, be aware that 'deadline' is in milliseconds",
                 DeprecationWarning,
@@ -82,15 +82,15 @@
             host=host,
             port=port,
             tls=tls,
-            deadline_ms=deadline,
+            deadline_ms=deadline_ms,
             retry_backoff_ms=retry_backoff_ms,
             retry_backoff_max_ms=retry_backoff_max_ms,
             retry_grace_period=retry_grace_period,
             resolver=resolver_type,
             offline_flag_source_path=offline_flag_source_path,
             stream_deadline_ms=stream_deadline_ms,
             keep_alive_time=keep_alive_time,
-            cache=cache_type,
+            cache=cache,
             max_cache_size=max_cache_size,
             cert_path=cert_path,
         )

providers/openfeature-provider-flagd/src/openfeature/contrib/provider/flagd/resolvers/grpc.py

@@ -179,7 +179,6 @@ def listen(self) -> None:
             if self.streamline_deadline_seconds > 0
             else {}
         )
-        call_args["wait_for_ready"] = True
         request = evaluation_pb2.EventStreamRequest()
 
         # defining a never ending loop to recreate the stream

providers/openfeature-provider-flagd/tests/e2e/conftest.py

@@ -1,17 +1,13 @@
 import typing
 
-from tests.e2e.steps import *  # noqa: F403
+from tests.e2e.step.config_steps import *  # noqa: F403
+from tests.e2e.step.context_steps import *  # noqa: F403
+from tests.e2e.step.event_steps import *  # noqa: F403
+from tests.e2e.step.flag_step import *  # noqa: F403
+from tests.e2e.step.provider_steps import *  # noqa: F403
+from tests.e2e.steps import *  # noqa: F403  # noqa: F403
 
 JsonPrimitive = typing.Union[str, bool, float, int]
 
 TEST_HARNESS_PATH = "../../openfeature/test-harness"
 SPEC_PATH = "../../openfeature/spec"
-
-
-# running all gherkin tests, except the ones, not implemented
-def pytest_collection_modifyitems(config):
-    marker = "not customCert and not unixsocket and not sync and not targetURI"
-
-    # this seems to not work with python 3.8
-    if hasattr(config.option, "markexpr") and config.option.markexpr == "":
-        config.option.markexpr = marker

providers/openfeature-provider-flagd/tests/e2e/flagd_container.py

@@ -1,26 +1,38 @@
 import time
+import typing
 from pathlib import Path
 
 import grpc
 from grpc_health.v1 import health_pb2, health_pb2_grpc
 from testcontainers.core.container import DockerContainer
 from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs
 
+from openfeature.contrib.provider.flagd.config import ResolverType
+
 HEALTH_CHECK = 8014
 
 
 class FlagdContainer(DockerContainer):
     def __init__(
         self,
-        image: str = "ghcr.io/open-feature/flagd-testbed",
-        port: int = 8013,
+        feature: typing.Optional[str] = None,
         **kwargs,
     ) -> None:
+        image: str = "ghcr.io/open-feature/flagd-testbed"
+        if feature is not None:
+            image = f"{image}-{feature}"
         path = Path(__file__).parents[2] / "openfeature/test-harness/version.txt"
         data = path.read_text().rstrip()
         super().__init__(f"{image}:v{data}", **kwargs)
-        self.port = port
-        self.with_exposed_ports(self.port, HEALTH_CHECK)
+        self.rpc = 8013
+        self.ipr = 8015
+        self.with_exposed_ports(self.rpc, self.ipr, HEALTH_CHECK)
+
+    def get_port(self, resolver_type: ResolverType):
+        if resolver_type == ResolverType.RPC:
+            return self.get_exposed_port(self.rpc)
+        else:
+            return self.get_exposed_port(self.ipr)
 
     def start(self) -> "FlagdContainer":
         super().start()

providers/openfeature-provider-flagd/tests/e2e/in_process_file/conftest.py

@@ -0,0 +1,136 @@
+import json
+import os
+import tempfile
+
+import pytest
+import yaml
+from pytest_bdd import given, parsers, when
+from tests.e2e.conftest import TEST_HARNESS_PATH
+from tests.e2e.step._utils import wait_for
+from tests.e2e.testFilter import TestFilter
+
+from openfeature import api
+from openfeature.client import OpenFeatureClient
+from openfeature.contrib.provider.flagd import FlagdProvider
+from openfeature.contrib.provider.flagd.config import ResolverType
+from openfeature.provider import ProviderStatus
+
+# from tests.e2e.step.config_steps import *
+# from tests.e2e.step.event_steps import *
+# from tests.e2e.step.provider_steps import *
+
+resolver = ResolverType.IN_PROCESS
+feature_list = {
+    "~targetURI",
+    "~customCert",
+    "~unixsocket",
+    "~events",
+    "~sync",
+    "~caching",
+    "~reconnect",
+    "~grace",
+}
+
+
+def pytest_collection_modifyitems(config, items):
+    test_filter = TestFilter(
+        config, feature_list=feature_list, resolver=resolver.value, base_path=__file__
+    )
+    test_filter.filter_items(items)
+
+
+KEY_EVALUATORS = "$evaluators"
+
+KEY_FLAGS = "flags"
+
+MERGED_FILE = "merged_file"
+
+
+@pytest.fixture()
+def resolver_type() -> ResolverType:
+    return resolver
+
+
+@pytest.fixture(scope="module")
+def all_flags(request):
+    result = {KEY_FLAGS: {}, KEY_EVALUATORS: {}}
+
+    path = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), f"../{TEST_HARNESS_PATH}/flags/")
+    )
+
+    for f in os.listdir(path):
+        with open(path + "/" + f, "rb") as infile:
+            loaded_json = json.load(infile)
+            result[KEY_FLAGS] = {**result[KEY_FLAGS], **loaded_json[KEY_FLAGS]}
+            if loaded_json.get(KEY_EVALUATORS):
+                result[KEY_EVALUATORS] = {
+                    **result[KEY_EVALUATORS],
+                    **loaded_json[KEY_EVALUATORS],
+                }
+
+    return result
+
+
+@pytest.fixture(params=["json", "yaml"], scope="module")
+def file_name(request, all_flags):
+    extension = request.param
+    with tempfile.NamedTemporaryFile(
+        "w", delete=False, suffix="." + extension
+    ) as outfile:
+        write_test_file(outfile, all_flags)
+        yield outfile
+        return outfile
+
+
+def write_test_file(outfile, all_flags):
+    with open(outfile.name, "w") as file:
+        if file.name.endswith("json"):
+            json.dump(all_flags, file)
+        else:
+            yaml.dump(all_flags, file)
+
+
+@when(
+    parsers.cfparse('a flag with key "{flag_key}" is modified'),
+    target_fixture="changed_flag",
+)
+def changed_flag(
+    flag_key: str,
+    all_flags: dict,
+    file_name,
+):
+    flag = all_flags[KEY_FLAGS][flag_key]
+
+    other_variant = [k for k in flag["variants"] if flag["defaultVariant"] in k].pop()
+
+    flag["defaultVariant"] = other_variant
+
+    all_flags[KEY_FLAGS][flag_key] = flag
+    write_test_file(file_name, all_flags)
+    return flag_key
+
+
+@pytest.fixture(autouse=True)
+def containers(request, file_name, all_flags, option_values):
+    api.set_provider(
+        FlagdProvider(
+            resolver_type=ResolverType.IN_PROCESS,
+            deadline_ms=500,
+            stream_deadline_ms=0,
+            retry_backoff_ms=1000,
+            offline_flag_source_path=file_name.name,
+            **option_values,
+        ),
+    )
+    pass
+
+
+@given(parsers.cfparse("a {provider_type} flagd provider"), target_fixture="client")
+def setup_provider(
+    resolver_type: ResolverType, provider_type: str, option_values: dict, file_name
+) -> OpenFeatureClient:
+    client = api.get_client()
+
+    wait_for(lambda: client.get_provider_status() == ProviderStatus.READY)
+    return client

providers/openfeature-provider-flagd/tests/e2e/in_process_file/test_flaqd.py

@@ -0,0 +1,11 @@
+import sys
+
+from pytest_bdd import scenarios
+from tests.e2e.conftest import SPEC_PATH, TEST_HARNESS_PATH
+
+# as soon as we support all the features, we can actually remove this limitation to not run on Python 3.8
+# Python 3.8 does not fully support tagging, hence that it will run all cases
+if sys.version_info >= (3, 9):
+    scenarios(f"{TEST_HARNESS_PATH}/gherkin")
+
+scenarios(f"{SPEC_PATH}/specification/assets/gherkin")

providers/openfeature-provider-flagd/tests/e2e/rpc/conftest.py

@@ -0,0 +1,23 @@
+import pytest
+from tests.e2e.testFilter import TestFilter
+
+from openfeature.contrib.provider.flagd.config import ResolverType
+
+# from tests.e2e.step.config_steps import *
+# from tests.e2e.step.event_steps import *
+# from tests.e2e.step.provider_steps import *
+
+resolver = ResolverType.RPC
+feature_list = ["~targetURI", "~unixsocket", "~sync"]
+
+
+def pytest_collection_modifyitems(config, items):
+    test_filter = TestFilter(
+        config, feature_list=feature_list, resolver=resolver.value, base_path=__file__
+    )
+    test_filter.filter_items(items)
+
+
+@pytest.fixture()
+def resolver_type() -> ResolverType:
+    return resolver

providers/openfeature-provider-flagd/tests/e2e/rpc/test_flaqd.py

@@ -0,0 +1,11 @@
+import sys
+
+from pytest_bdd import scenarios
+from tests.e2e.conftest import SPEC_PATH, TEST_HARNESS_PATH
+
+# as soon as we support all the features, we can actually remove this limitation to not run on Python 3.8
+# Python 3.8 does not fully support tagging, hence that it will run all cases
+if sys.version_info >= (3, 9):
+    scenarios(f"{TEST_HARNESS_PATH}/gherkin")
+
+scenarios(f"{SPEC_PATH}/specification/assets/gherkin")

providers/openfeature-provider-flagd/tests/e2e/step/_utils.py

@@ -0,0 +1,30 @@
+import json
+import time
+import typing
+
+from asserts import assert_true
+
+
+def str2bool(v):
+    return v.lower() in ("yes", "true", "t", "1")
+
+
+type_cast = {
+    "Integer": int,
+    "Float": float,
+    "String": str,
+    "Boolean": str2bool,
+    "Object": json.loads,
+}
+
+
+JsonObject = typing.Union[dict, list]
+JsonPrimitive = typing.Union[str, bool, float, int, JsonObject]
+
+
+def wait_for(pred, poll_sec=2, timeout_sec=10):
+    start = time.time()
+    while not (ok := pred()) and (time.time() - start < timeout_sec):
+        time.sleep(poll_sec)
+    assert_true(pred())
+    return ok

providers/openfeature-provider-flagd/tests/e2e/test_config.py → providers/openfeature-provider-flagd/tests/e2e/step/config_steps.py

@@ -1,11 +1,9 @@
 import re
-import sys
 import typing
 
 import pytest
 from asserts import assert_equal
-from pytest_bdd import given, parsers, scenarios, then, when
-from tests.e2e.conftest import TEST_HARNESS_PATH
+from pytest_bdd import given, parsers, then, when
 
 from openfeature.contrib.provider.flagd.config import CacheType, Config, ResolverType
 
@@ -100,9 +98,3 @@ def check_option_value(option, value, type_info, config):
     value = type_cast[type_info](value)
     value = value if value != "null" else None
     assert_equal(config.__getattribute__(camel_to_snake(option)), value)
-
-
-if sys.version_info >= (3, 9):
-    scenarios(
-        f"{TEST_HARNESS_PATH}/gherkin/config.feature",
-    )