From da08d70d23d08bc799232a587051026e43eaf4ca Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Mon, 8 Jul 2024 11:49:16 -0400 Subject: [PATCH 1/2] Standardize values and type of outcome Signed-off-by: Brett Smith --- conformance/testcaserun_finished.json | 2 +- conformance/testsuiterun_finished.json | 2 +- schemas/pipelinerunfinished.json | 7 ++++++- schemas/testcaserunfinished.json | 5 ++--- schemas/testsuiterunfinished.json | 5 ++--- testing-events.md | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/conformance/testcaserun_finished.json b/conformance/testcaserun_finished.json index 45316363..34c8b708 100644 --- a/conformance/testcaserun_finished.json +++ b/conformance/testcaserun_finished.json @@ -44,7 +44,7 @@ "source": "/event/source/123", "type": "testCaseRun", "content": { - "outcome": "pass", + "outcome": "success", "environment": { "id": "dev", "source": "testkube-dev-123" diff --git a/conformance/testsuiterun_finished.json b/conformance/testsuiterun_finished.json index a14d49a4..8f63e39f 100644 --- a/conformance/testsuiterun_finished.json +++ b/conformance/testsuiterun_finished.json @@ -44,7 +44,7 @@ "source": "/event/source/123", "type": "testSuiteRun", "content": { - "outcome": "fail", + "outcome": "failure", "severity": "critical", "reason": "Host 123.34.23.32 not found", "environment": { diff --git a/schemas/pipelinerunfinished.json b/schemas/pipelinerunfinished.json index 33d30853..b8c0b72c 100644 --- a/schemas/pipelinerunfinished.json +++ b/schemas/pipelinerunfinished.json @@ -79,7 +79,12 @@ "type": "string" }, "outcome": { - "type": "string" + "type": "string", + "enum": [ + "success", + "failure", + "error" + ] }, "errors": { "type": "string" diff --git a/schemas/testcaserunfinished.json b/schemas/testcaserunfinished.json index 062a29ee..9a7d7eae 100644 --- a/schemas/testcaserunfinished.json +++ b/schemas/testcaserunfinished.json @@ -72,9 +72,8 @@ "outcome": { "type": "string", "enum": [ - "pass", - "fail", - "cancel", + "success", + "failure", "error" ] }, diff --git a/schemas/testsuiterunfinished.json b/schemas/testsuiterunfinished.json index 68b3fd49..217b5c3a 100644 --- a/schemas/testsuiterunfinished.json +++ b/schemas/testsuiterunfinished.json @@ -113,9 +113,8 @@ "outcome": { "type": "string", "enum": [ - "pass", - "fail", - "cancel", + "success", + "failure", "error" ] }, diff --git a/testing-events.md b/testing-events.md index 56ac9584..f5733543 100644 --- a/testing-events.md +++ b/testing-events.md @@ -120,7 +120,7 @@ This event represents a finished testCase execution. The event will contain the | environment | `Object` [`environment`](continuous-deployment.md/#environment) | The environment in which this testCaseRun was running | `{"id": "1234"}`, `{"id": "dev", "source": "testkube-dev-123"}` | ✅ | | testCase | `Object` [`testCase`](#testcase) | Definition of the testCase being executed | `{"id": "92834723894", "name": "Login Test", "type": "integration"}` | | | testSuiteRun | `Object` [`testSuiteRun`](#testsuiterun) | A testSuiteRun to associate this testCaseRun with a containing testSuiteRun | `{"id":"Auth-TestSuite-execution-12334", "source": "staging/testkube"}` | | -| outcome | `String (enum)` | The outcome of the testSuite execution, one of `pass`, `fail`, `cancel`, `error` | `pass` | ✅ | +| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `error` or `failure` | `success` | ✅ | | severity | `String (enum)` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical` | | reason | `String` | A reason related to the outcome of the execution | `Canceled by user`, `Failed assertion`, `Timed out` | | @@ -188,7 +188,7 @@ This event represents a finished testSuite execution. The event will contain the | id | `String` | Uniquely identifies the subject within the source. | `unit`, `e2e`, `security` | ✅ | | source | `URI-Reference` | [source](spec.md#source--context-) from the context | | | | environment | `Object` [`environment`](continuous-deployment.md/#environment) | The environment in which this testSuiteRun was running | `{"id": "1234"}`, `{"id": "dev", "source": "testkube-dev-123"}` | ✅ | -| outcome | `String (enum)` | The outcome of the testSuite execution, one of `pass`, `fail`, `cancel`, `error` | `fail` | ✅ | +| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `error` or `failure` | `failure` | ✅ | | severity | `String (enum)` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical`, `low`, `medium`, `high` | | reason | `String` | A reason related to the outcome of the execution | `Canceled by user`, `Failed testCase` | | | testSuite | `Object` [`testSuite`](#testsuite) | Definition of the testSuite being executed | `{"id": "92834723894", "name": "Auth TestSuite"}` | | From 53e8fe1de6a62ccf0eb61d803bb5eb71bb45c5e6 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Mon, 25 Nov 2024 11:20:50 -0500 Subject: [PATCH 2/2] Add cancel to outcome values --- core.md | 16 ++++----- schemas/pipelinerunfinished.json | 33 ++++-------------- schemas/testcaserunfinished.json | 57 ++++++------------------------- schemas/testsuiterunfinished.json | 53 ++++++---------------------- testing-events.md | 4 +-- 5 files changed, 37 insertions(+), 126 deletions(-) diff --git a/core.md b/core.md index e464feb7..30cc60e7 100644 --- a/core.md +++ b/core.md @@ -36,9 +36,9 @@ track the build and release progress on a particular software artifact. | source | `URI-Reference` | See [source](spec.md#source-subject) | | | type | `String` | See [type](spec.md#type-subject) | `pipelineRun` | | pipelineName | `String` | The name of the pipeline | `MyPipeline`, `Unit tests for my repo` | -| outcome | `String` | outcome of a finished `pipelineRun` | `success`, `error` or `failure`| +| outcome | `String` | outcome of a finished `pipelineRun` | `success`, `failure`, `cancel`, or `error` | | url | `URI` | url to the `pipelineRun` | `https://dashboard.org/namespace/pipelinerun-1234`, `https://api.cdsystem.com/namespace/pipelinerun-1234` | -| errors | `String` | In case of error or failed pipeline, provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `pipelineRun canceled by user`, `Unit tests failed`| +| errors | `String` | In case of error, canceled, or failed pipeline , provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `pipelineRun canceled by user`, `Unit tests failed`| ### `taskRun` @@ -56,9 +56,9 @@ associated, in which case it is acceptable to generate only taskRun events. | type | `String` | See [type](spec.md#type-subject) | `taskRun` | | taskName | `String` | The name of the pipeline | `MyPipeline`, `Unit tests for my repo` | | pipelineRun | `Object` ([`pipelineRun`](#pipelinerun)) | The `pipelineRun` that this `taskRun` belongs to. | `{"id": "namespace/pipelinerun-1234"}`| -| outcome | `String` | outcome of a finished `taskRun` | `success`, `error` or `failure`| +| outcome | `String` | outcome of a finished `taskRun` | `success`, `failure`, `cancel`, or `error` | | url | `URI` | url to the `taskRun` | `https://dashboard.org/namespace/taskrun-1234`, `https://api.cdsystem.com/namespace/taskrun-1234` | -| errors | `String` | In case of error or failed pipeline, provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `taskRun canceled by user`, `Unit tests failed`| +| errors | `String` | In case of error, canceled, or failed pipeline , provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `taskRun canceled by user`, `Unit tests failed`| ## Events @@ -111,8 +111,8 @@ A pipelineRun has finished, successfully or not. | type | `String` | See [type](spec.md#type-subject) | `pipelineRun` | | | pipelineName | `String` | The name of the pipeline | `MyPipeline`, `Unit tests for my repo` | | | url | `URI` | url to the `pipelineRun` | `https://dashboard.org/namespace/pipelinerun-1234`, `https://api.cdsystem.com/namespace/pipelinerun-1234` | | -| outcome | `String (enum)` | outcome of a finished `pipelineRun` | `success`, `error` or `failure`| | -| errors | `String` | In case of error or failed pipeline, provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `pipelineRun canceled by user`, `Unit tests failed`| | +| outcome | `String (enum)` | outcome of a finished `pipelineRun` | `success`, `failure`, `cancel`, or `error` | `success`, `failure`, `cancel`, `error` | +| errors | `String` | In case of error, canceled, or failed pipeline , provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `pipelineRun canceled by user`, `Unit tests failed`| | ### [`taskRun Started`](conformance/taskrun_started.json) @@ -147,5 +147,5 @@ A taskRun has finished, successfully or not. | taskName | `String` | The name of the pipeline | `MyPipeline`, `Unit tests for my repo` | | | pipelineRun | `Object` ([`pipelineRun`](#pipelinerun)) | The `pipelineRun` that this `taskRun` belongs to. | `{"id": "namespace/pipelinerun-1234"}`| | | url | `URI` | url to the `taskRun` | `https://dashboard.org/namespace/taskrun-1234`, `https://api.cdsystem.com/namespace/taskrun-1234` | | -| outcome | `String (enum)` | outcome of a finished `taskRun` | `success`, `error` or `failure`| | -| errors | `String` | In case of error or failed pipeline, provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `taskRun canceled by user`, `Unit tests failed`| | +| outcome | `String (enum)` | outcome of a finished `taskRun` | `success`, `failure`, `cancel`, or `error` | `success`, `failure`, `cancel`, `error` | +| errors | `String` | In case of error, canceled, or failed pipeline , provides details about the failure | `Invalid input param 123`, `Timeout during execution`, `taskRun canceled by user`, `Unit tests failed`| | diff --git a/schemas/pipelinerunfinished.json b/schemas/pipelinerunfinished.json index b8c0b72c..e326f768 100644 --- a/schemas/pipelinerunfinished.json +++ b/schemas/pipelinerunfinished.json @@ -19,9 +19,7 @@ }, "type": { "type": "string", - "enum": [ - "dev.cdevents.pipelinerun.finished.0.3.0-draft" - ], + "enum": ["dev.cdevents.pipelinerun.finished.0.3.0-draft"], "default": "dev.cdevents.pipelinerun.finished.0.3.0-draft" }, "timestamp": { @@ -43,13 +41,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "version", - "id", - "source", - "type", - "timestamp" - ] + "required": ["version", "id", "source", "type", "timestamp"] }, "subject": { "properties": { @@ -65,9 +57,7 @@ "type": { "type": "string", "minLength": 1, - "enum": [ - "pipelineRun" - ], + "enum": ["pipelineRun"], "default": "pipelineRun" }, "content": { @@ -80,11 +70,7 @@ }, "outcome": { "type": "string", - "enum": [ - "success", - "failure", - "error" - ] + "enum": ["success", "failure", "cancel", "error"] }, "errors": { "type": "string" @@ -96,11 +82,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "id", - "type", - "content" - ] + "required": ["id", "type", "content"] }, "customData": { "oneOf": [ @@ -119,8 +101,5 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "context", - "subject" - ] + "required": ["context", "subject"] } diff --git a/schemas/testcaserunfinished.json b/schemas/testcaserunfinished.json index 9a7d7eae..aec83bda 100644 --- a/schemas/testcaserunfinished.json +++ b/schemas/testcaserunfinished.json @@ -18,9 +18,7 @@ }, "type": { "type": "string", - "enum": [ - "dev.cdevents.testcaserun.finished.0.3.0-draft" - ], + "enum": ["dev.cdevents.testcaserun.finished.0.3.0-draft"], "default": "dev.cdevents.testcaserun.finished.0.3.0-draft" }, "timestamp": { @@ -42,13 +40,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "version", - "id", - "source", - "type", - "timestamp" - ] + "required": ["version", "id", "source", "type", "timestamp"] }, "subject": { "properties": { @@ -62,29 +54,18 @@ "type": { "type": "string", "minLength": 1, - "enum": [ - "testCaseRun" - ], + "enum": ["testCaseRun"], "default": "testCaseRun" }, "content": { "properties": { "outcome": { "type": "string", - "enum": [ - "success", - "failure", - "error" - ] + "enum": ["success", "failure", "cancel", "error"] }, "severity": { "type": "string", - "enum": [ - "low", - "medium", - "high", - "critical" - ] + "enum": ["low", "medium", "high", "critical"] }, "reason": { "type": "string" @@ -103,9 +84,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "id" - ] + "required": ["id"] }, "testSuiteRun": { "type": "object", @@ -119,16 +98,12 @@ } }, "additionalProperties": false, - "required": [ - "id" - ] + "required": ["id"] }, "testCase": { "type": "object", "additionalProperties": false, - "required": [ - "id" - ], + "required": ["id"], "properties": { "id": { "type": "string", @@ -162,19 +137,12 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "outcome", - "environment" - ] + "required": ["outcome", "environment"] } }, "additionalProperties": false, "type": "object", - "required": [ - "id", - "type", - "content" - ] + "required": ["id", "type", "content"] }, "customData": { "oneOf": [ @@ -193,8 +161,5 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "context", - "subject" - ] + "required": ["context", "subject"] } diff --git a/schemas/testsuiterunfinished.json b/schemas/testsuiterunfinished.json index 217b5c3a..521937df 100644 --- a/schemas/testsuiterunfinished.json +++ b/schemas/testsuiterunfinished.json @@ -18,9 +18,7 @@ }, "type": { "type": "string", - "enum": [ - "dev.cdevents.testsuiterun.finished.0.3.0-draft" - ], + "enum": ["dev.cdevents.testsuiterun.finished.0.3.0-draft"], "default": "dev.cdevents.testsuiterun.finished.0.3.0-draft" }, "timestamp": { @@ -42,13 +40,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "version", - "id", - "source", - "type", - "timestamp" - ] + "required": ["version", "id", "source", "type", "timestamp"] }, "subject": { "properties": { @@ -62,9 +54,7 @@ "type": { "type": "string", "minLength": 1, - "enum": [ - "testSuiteRun" - ], + "enum": ["testSuiteRun"], "default": "testSuiteRun" }, "content": { @@ -83,16 +73,12 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "id" - ] + "required": ["id"] }, "testSuite": { "type": "object", "additionalProperties": false, - "required": [ - "id" - ], + "required": ["id"], "properties": { "id": { "type": "string", @@ -112,20 +98,11 @@ }, "outcome": { "type": "string", - "enum": [ - "success", - "failure", - "error" - ] + "enum": ["success", "failure", "cancel", "error"] }, "severity": { "type": "string", - "enum": [ - "low", - "medium", - "high", - "critical" - ] + "enum": ["low", "medium", "high", "critical"] }, "reason": { "type": "string" @@ -133,19 +110,12 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "outcome", - "environment" - ] + "required": ["outcome", "environment"] } }, "additionalProperties": false, "type": "object", - "required": [ - "id", - "type", - "content" - ] + "required": ["id", "type", "content"] }, "customData": { "oneOf": [ @@ -164,8 +134,5 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "context", - "subject" - ] + "required": ["context", "subject"] } diff --git a/testing-events.md b/testing-events.md index f5733543..dced2c09 100644 --- a/testing-events.md +++ b/testing-events.md @@ -120,7 +120,7 @@ This event represents a finished testCase execution. The event will contain the | environment | `Object` [`environment`](continuous-deployment.md/#environment) | The environment in which this testCaseRun was running | `{"id": "1234"}`, `{"id": "dev", "source": "testkube-dev-123"}` | ✅ | | testCase | `Object` [`testCase`](#testcase) | Definition of the testCase being executed | `{"id": "92834723894", "name": "Login Test", "type": "integration"}` | | | testSuiteRun | `Object` [`testSuiteRun`](#testsuiterun) | A testSuiteRun to associate this testCaseRun with a containing testSuiteRun | `{"id":"Auth-TestSuite-execution-12334", "source": "staging/testkube"}` | | -| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `error` or `failure` | `success` | ✅ | +| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `failure`, `cancel`, or `error` | `success`, `failure`, `cancel`, `error` | ✅ | | severity | `String (enum)` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical` | | reason | `String` | A reason related to the outcome of the execution | `Canceled by user`, `Failed assertion`, `Timed out` | | @@ -188,7 +188,7 @@ This event represents a finished testSuite execution. The event will contain the | id | `String` | Uniquely identifies the subject within the source. | `unit`, `e2e`, `security` | ✅ | | source | `URI-Reference` | [source](spec.md#source--context-) from the context | | | | environment | `Object` [`environment`](continuous-deployment.md/#environment) | The environment in which this testSuiteRun was running | `{"id": "1234"}`, `{"id": "dev", "source": "testkube-dev-123"}` | ✅ | -| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `error` or `failure` | `failure` | ✅ | +| outcome | `String (enum)` | The outcome of the testSuite execution, one of `success`, `failure`, `cancel`, or `error` | `success`, `failure`, `cancel`, `error` | ✅ | | severity | `String (enum)` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical`, `low`, `medium`, `high` | | reason | `String` | A reason related to the outcome of the execution | `Canceled by user`, `Failed testCase` | | | testSuite | `Object` [`testSuite`](#testsuite) | Definition of the testSuite being executed | `{"id": "92834723894", "name": "Auth TestSuite"}` | |