diff --git a/.gherkin-lintrc b/.gherkin-lintrc index 16d3030..36b2d0c 100644 --- a/.gherkin-lintrc +++ b/.gherkin-lintrc @@ -50,8 +50,8 @@ { "steps-length": { "Background": 3, - "Scenario": 15 + "Scenario": 16 } } ] -} \ No newline at end of file +} diff --git a/README.md b/README.md index b796465..31691b0 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ Combined with the _flagd-provider_ for the associated SDK and the flagd-testbed, Included suites: -[flagd.feature](gherkin/flagd.feature) includes tests relevant to flagd and all flagd providers: +[evaluation.feature](gherkin/evaluation.feature) includes tests relevant to flagd and all flagd providers: * default value (zero-value) handling (some proto3 implementations handle these inconsistently). * basic event handling -[flagd-json-evaluator.feature](gherkin/flagd-json-evaluator.feature) includes tests relevant to flagd and in-process providers: +[targeting.feature](gherkin/targeting.feature) includes tests relevant to flagd and in-process providers: * custom JSONLogic operators (`starts_with`, `ends_with`, `fractional`, `sem_ver`) * evaluator reuse via `$ref` diff --git a/gherkin/config.feature b/gherkin/config.feature index 37e1b1c..8a9065d 100644 --- a/gherkin/config.feature +++ b/gherkin/config.feature @@ -110,15 +110,15 @@ Feature: Configuration Test | option | type | value | | targetUri | String | path | @customCert - Examples: + Examples: Custom Certificate | option | type | value | | certPath | String | path | @unixsocket - Examples: + Examples: Unixsocket | option | type | value | | socketPath | String | path | @events - Examples: + Examples: events | option | type | value | | streamDeadlineMs | Integer | 500000 | | keepAliveTime | Long | 5 | @@ -126,7 +126,7 @@ Feature: Configuration Test | retryBackoffMaxMs | Integer | 12000 | | retryGracePeriod | Integer | 10 | @sync - Examples: + Examples: sync | option | type | value | | streamDeadlineMs | Integer | 500000 | | keepAliveTime | Long | 5 | @@ -135,12 +135,12 @@ Feature: Configuration Test | retryGracePeriod | Integer | 10 | | selector | String | selector | @caching - Examples: + Examples: caching | option | type | value | | cache | CacheType | disabled | | maxCacheSize | Integer | 1236 | @offline - Examples: + Examples: offline | option | type | value | | offlineFlagSourcePath | String | path | | offlinePollIntervalMs | Integer | 1000 | @@ -165,15 +165,15 @@ Feature: Configuration Test | option | env | type | value | | targetUri | FLAGD_TARGET_URI | String | path | @customCert - Examples: + Examples: Custom Certificates | option | env | type | value | | certPath | FLAGD_SERVER_CERT_PATH | String | path | @unixsocket - Examples: + Examples: Unixsocket | option | env | type | value | | socketPath | FLAGD_SOCKET_PATH | String | path | @events - Examples: + Examples: Events | option | env | type | value | | streamDeadlineMs | FLAGD_STREAM_DEADLINE_MS | Integer | 500000 | | keepAliveTime | FLAGD_KEEP_ALIVE_TIME_MS | Long | 5 | @@ -181,7 +181,7 @@ Feature: Configuration Test | retryBackoffMaxMs | FLAGD_RETRY_BACKOFF_MAX_MS | Integer | 12000 | | retryGracePeriod | FLAGD_RETRY_GRACE_PERIOD | Integer | 10 | @sync - Examples: + Examples: Sync | option | env | type | value | | streamDeadlineMs | FLAGD_STREAM_DEADLINE_MS | Integer | 500000 | | keepAliveTime | FLAGD_KEEP_ALIVE_TIME_MS | Long | 5 | @@ -190,12 +190,12 @@ Feature: Configuration Test | retryGracePeriod | FLAGD_RETRY_GRACE_PERIOD | Integer | 10 | | selector | FLAGD_SOURCE_SELECTOR | String | selector | @caching - Examples: + Examples: Caching | option | env | type | value | | cache | FLAGD_CACHE | CacheType | disabled | | maxCacheSize | FLAGD_MAX_CACHE_SIZE | Integer | 1236 | @offline - Examples: + Examples: Offline | option | env | type | value | | offlineFlagSourcePath | FLAGD_OFFLINE_FLAG_SOURCE_PATH | String | path | | offlinePollIntervalMs | FLAGD_OFFLINE_POLL_MS | Integer | 1000 | @@ -218,15 +218,15 @@ Feature: Configuration Test | option | env | type | value | env-value | | targetUri | FLAGD_TARGET_URI | String | path | fun | @customCert - Examples: + Examples: Custom Certificates | option | env | type | value | env-value | | certPath | FLAGD_SERVER_CERT_PATH | String | path | rpc | @unixsocket - Examples: + Examples: Unixsocket | option | env | type | value | env-value | | socketPath | FLAGD_SOCKET_PATH | String | path | rpc | @events - Examples: + Examples: Events | option | env | type | value | env-value | | streamDeadlineMs | FLAGD_STREAM_DEADLINE_MS | Integer | 500000 | 400 | | keepAliveTime | FLAGD_KEEP_ALIVE_TIME_MS | Long | 5 | 4 | @@ -234,7 +234,7 @@ Feature: Configuration Test | retryBackoffMaxMs | FLAGD_RETRY_BACKOFF_MAX_MS | Integer | 12000 | 4 | | retryGracePeriod | FLAGD_RETRY_GRACE_PERIOD | Integer | 10 | 4 | @sync - Examples: + Examples: Sync | option | env | type | value | env-value | | streamDeadlineMs | FLAGD_STREAM_DEADLINE_MS | Integer | 500000 | 400 | | keepAliveTime | FLAGD_KEEP_ALIVE_TIME_MS | Long | 5 | 4 | @@ -243,12 +243,12 @@ Feature: Configuration Test | retryGracePeriod | FLAGD_RETRY_GRACE_PERIOD | Integer | 10 | 4 | | selector | FLAGD_SOURCE_SELECTOR | String | selector | sele | @caching - Examples: + Examples: Caching | option | env | type | value | env-value | | cache | FLAGD_CACHE | CacheType | disabled | lru | | maxCacheSize | FLAGD_MAX_CACHE_SIZE | Integer | 1236 | 2345 | @offline - Examples: + Examples: Offline | option | env | type | value | env-value | | offlineFlagSourcePath | FLAGD_OFFLINE_FLAG_SOURCE_PATH | String | path | lll | | offlinePollIntervalMs | FLAGD_OFFLINE_POLL_MS | Integer | 1000 | 4 | diff --git a/gherkin/connection.feature b/gherkin/connection.feature new file mode 100644 index 0000000..019963f --- /dev/null +++ b/gherkin/connection.feature @@ -0,0 +1,57 @@ +@rpc @in-process +Feature: flagd provider disconnect and reconnect functionality + + Scenario Outline: Connection + Given a flagd provider + And a Boolean-flag with key "boolean-flag" and a default value "true" + When the flag was evaluated with details + Then the resolved details value should be "true" + + Examples: Stable + | name | + | stable | + @targetURI + Examples: Target URI + | name | + | target | + @customCert + Examples: Certificates + | name | + | ssl | + @unixsocket @os.linux + Examples: Unixsocket + | name | + | socket | + + @reconnect + # This test suite tests the reconnection functionality of flagd providers + Scenario Outline: Provider reconnection + Given a flagd provider + And a ready event handler + And a error event handler + When a ready event was fired + When the connection is lost for 4s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + Examples: Stable + | name | + | stable | + @targetURI + Examples: Target URI + | name | + | socket | + @customCert + Examples: Certificates + | name | + | ssl | + @unixsocket @os.linux + Examples: Unixsocket + | name | + | socket | + + Scenario: Provider unavailable + Given an option "deadlineMs" of type "Integer" with value "1000" + And a unavailable flagd provider + And a error event handler + Then the error event handler should have been executed within 1000ms diff --git a/gherkin/evaluation.feature b/gherkin/evaluation.feature new file mode 100644 index 0000000..59b2059 --- /dev/null +++ b/gherkin/evaluation.feature @@ -0,0 +1,34 @@ +@rpc @in-process +Feature: flagd providers + + # This test suite contains scenarios to test flagd providers. + # It's associated with the flags configured in flags/changing-flag.json and flags/zero-flags.json. + # It should be used in conjunction with the suites supplied by the OpenFeature specification. + + Background: + Given an option "cache" of type "CacheType" with value "disabled" + And a stable flagd provider + + Scenario Outline: Resolve values + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: + | key | type | default | resolved_value | + | boolean-flag | Boolean | false | true | + | string-flag | String | bye | hi | + | integer-flag | Integer | 1 | 10 | + | float-flag | Float | 0.1 | 0.5 | + + Scenario Outline: Resolves zero value + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: + | key | type | default | resolved_value | + | boolean-zero-flag | Boolean | true | false | + | string-zero-flag | String | hi | | + | integer-zero-flag | Integer | 1 | 0 | + | float-zero-flag | Float | 0.1 | 0.0 | diff --git a/gherkin/events.feature b/gherkin/events.feature index 28dd023..b9ec581 100644 --- a/gherkin/events.feature +++ b/gherkin/events.feature @@ -1,24 +1,44 @@ -@grace +@rpc @in-process Feature: Flagd Provider State Changes Background: - Given a flagd provider is set + Given a stable flagd provider Scenario Outline: Provider events - When a handler is added - Then the handler must run + Given a event handler + When the connection is lost for 6s + Then the event handler should have been executed Examples: - | event | - | PROVIDER_ERROR | - | PROVIDER_STALE | - | PROVIDER_READY | + | event | + | error | + | ready | + @grace + Examples: + | event | + | stale | + @grace Scenario: Provider events chain ready -> stale -> error -> ready - When a PROVIDER_READY handler is added - Then the PROVIDER_READY handler must run - When a PROVIDER_STALE handler is added - Then the PROVIDER_STALE handler must run - When a PROVIDER_ERROR handler is added - Then the PROVIDER_ERROR handler must run - When a PROVIDER_READY handler is added - Then the PROVIDER_READY handler must run + Given a ready event handler + And a error event handler + And a stale event handler + Then the ready event handler should have been executed + When the connection is lost for 6s + Then the stale event handler should have been executed + Then the error event handler should have been executed + Then the ready event handler should have been executed + + Scenario: Provider events chain ready -> error -> ready + Given a ready event handler + And a error event handler + Then the ready event handler should have been executed + When the connection is lost for 6s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + Scenario: Flag change event + Given a String-flag with key "changing-flag" and a default value "false" + And a change event handler + When a change event was fired + And the flag was modified + Then the flag should be part of the event payload diff --git a/gherkin/flagd-json-evaluator.feature b/gherkin/flagd-json-evaluator.feature deleted file mode 100644 index f762912..0000000 --- a/gherkin/flagd-json-evaluator.feature +++ /dev/null @@ -1,126 +0,0 @@ -Feature: flagd json evaluation - - # This test suite contains scenarios to test the json-evaluation of flagd and flag-in-process providers. - # It's associated with the flags configured in flags/changing-flag.json, flags/zero-flags.json, flags/custom-ops.json and evaluator-refs.json. - # It should be used in conjunction with the suites supplied by the OpenFeature specification. - - Background: - Given a flagd provider is set - - # evaluator refs - Scenario Outline: Evaluator reuse - When a string flag with key is evaluated with default value "fallback" - And a context containing a key "email", with value "ballmer@macrosoft.com" - Then the returned value should be - Examples: - | key | value | - | "some-email-targeted-flag" | "hi" | - | "some-other-email-targeted-flag" | "yes" | - - # custom operators - Scenario Outline: Fractional operator - When a string flag with key "fractional-flag" is evaluated with default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value - Then the returned value should be - Examples: - | name | value | - | "jack" | "spades" | - | "queen" | "clubs" | - | "ten" | "diamonds" | - | "nine" | "hearts" | - | 3 | "diamonds" | - - Scenario Outline: Fractional operator shorthand - When a string flag with key "fractional-flag-shorthand" is evaluated with default value "fallback" - And a context containing a targeting key with value - Then the returned value should be - Examples: - | targeting key | value | - | "jon@company.com" | "heads" | - | "jane@company.com" | "tails" | - - Scenario Outline: Fractional operator with shared seed - When a string flag with key "fractional-flag-A-shared-seed" is evaluated with default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value - Then the returned value should be - Examples: - | name | value | - | "jack" | "hearts" | - | "queen" | "spades" | - | "ten" | "hearts" | - | "nine" | "diamonds" | - - Scenario Outline: Second fractional operator with shared seed - When a string flag with key "fractional-flag-B-shared-seed" is evaluated with default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value - Then the returned value should be - Examples: - | name | value | - | "jack" | "ace-of-hearts" | - | "queen" | "ace-of-spades" | - | "ten" | "ace-of-hearts" | - | "nine" | "ace-of-diamonds" | - - Scenario Outline: Substring operators - When a string flag with key "starts-ends-flag" is evaluated with default value "fallback" - And a context containing a key "id", with value - Then the returned value should be - Examples: - | id | value | - | "abcdef" | "prefix" | - | "uvwxyz" | "postfix" | - | "abcxyz" | "prefix" | - | "lmnopq" | "none" | - | 3 | "none" | - - Scenario Outline: Semantic version operator numeric comparison - When a string flag with key "equal-greater-lesser-version-flag" is evaluated with default value "fallback" - And a context containing a key "version", with value - Then the returned value should be - Examples: - | version | value | - | "2.0.0" | "equal" | - | "2.1.0" | "greater" | - | "1.9.0" | "lesser" | - | "2.0.0-alpha" | "lesser" | - | "2.0.0.0" | "none" | - - Scenario Outline: Semantic version operator semantic comparison - When a string flag with key "major-minor-version-flag" is evaluated with default value "fallback" - And a context containing a key "version", with value - Then the returned value should be - Examples: - | version | value | - | "3.0.1" | "minor" | - | "3.1.0" | "major" | - | "4.0.0" | "none" | - - Scenario Outline: Time-based operations - When an integer flag with key "timestamp-flag" is evaluated with default value 0 - And a context containing a key "time", with value