diff --git a/.github/workflows/analyze-pr-changes.yml b/.github/workflows/analyze-pr-changes.yml index d962fdc9..c9f14396 100644 --- a/.github/workflows/analyze-pr-changes.yml +++ b/.github/workflows/analyze-pr-changes.yml @@ -22,7 +22,7 @@ jobs: BEFORE_COVERAGE=/tmp/coverage-api-${BEFORE_SHA}.json AFTER_COVERAGE=/tmp/coverage-api-${AFTER_SHA}.json COVERAGE_DIFF=/tmp/coverage-api-${BEFORE_SHA}-${AFTER_SHA}-DIFF.json - + vars=( BEFORE_SHA AFTER_SHA @@ -33,7 +33,7 @@ jobs: AFTER_COVERAGE COVERAGE_DIFF ) - + { for var in "${vars[@]}" do @@ -48,7 +48,7 @@ jobs: shell: bash -eo pipefail {0} run: | docker build coverage --tag opensearch-with-api-plugin - + docker run \ --name opensearch \ --rm -d \ @@ -56,11 +56,11 @@ jobs: -e "discovery.type=single-node" \ -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \ opensearch-with-api-plugin - + npm ci - + npm run dump-cluster-spec -- --opensearch-insecure --output $CLUSTER_SPEC - + docker stop opensearch env: OPENSEARCH_PASSWORD: BobgG7YrtsdKf9M @@ -92,14 +92,14 @@ jobs: --cluster $CLUSTER_SPEC \ --specification $BEFORE_SPEC \ --output $BEFORE_COVERAGE - + npm run coverage:spec -- \ --cluster $CLUSTER_SPEC \ --specification $AFTER_SPEC \ --output $AFTER_COVERAGE - + jq . $AFTER_COVERAGE - + jq --slurp ' [ .[].counts ] | { @@ -116,6 +116,18 @@ jobs: $AFTER_COVERAGE \ | tee $COVERAGE_DIFF + - name: Display Coverage Checklist + shell: bash -eo pipefail {0} + run: | + jq -sc ' + (map(.endpoints .uncovered | to_entries | sort_by(.value) | map((.value[] | ascii_upcase) + " " + .key)) | .[] | to_entries | map({path: .value, covered: false})) as $uncovered | + (map(.endpoints .covered | to_entries | sort_by(.value) | map((.value[] | ascii_upcase) + " " + .key)) | .[] | to_entries | map({path: .value, covered: true})) as $covered | + $covered+$uncovered | + sort_by(.path) | + map(if .covered then "- [x] \(.path)" else "- [ ] \(.path)" end) | + .[] + ' $AFTER_COVERAGE | jq -r + - name: Upload Coverage Data id: upload-coverage uses: actions/upload-artifact@v4 @@ -150,7 +162,7 @@ jobs: exit 1 fi fi - + gawk ' BEGIN { RS = "(\r|\n|\r\n)" @@ -176,7 +188,7 @@ jobs: HAD_CHANGES = 1 print } - + END { if (!HAD_CHANGES) { print "**NO CHANGES**\n" diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 895ac1e4..b7213aa8 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -126,14 +126,16 @@ jobs: - name: Display Missing Test Paths run: | - jq -sc ' - (map(.operations) | add | unique) as $all | - (map(.evaluated_operations) | add | unique) as $evaluated | - $all-$evaluated | - sort_by(.path) | - .[] | - "\(.method) \(.path)" - ' $(find ./ -name "test-spec-coverage-*.json") + jq -r -sc ' + (map(.operations) | add | unique) as $all | + (map(.evaluated_operations) | add | unique) as $evaluated | + ($all-$evaluated | map({path: .path, method: .method, evaluated: false})) as $unevaluated | + ($evaluated | map({path: .path, method: .method, evaluated: true})) as $evaluated | + $evaluated+$unevaluated | + sort_by(.path) | + map(if .evaluated then "- [x] \(.method) \(.path)" else "- [ ] \(.method) \(.path)" end) | + .[] + ' $(find ./ -name "test-spec-coverage-*.json") - name: Combine Test Coverage Data shell: bash -eo pipefail {0} diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 7ff32656..20905732 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -355,7 +355,7 @@ npm run lint--fix ### [Analyze PR Changes](.github/workflows/analyze-pr-changes.yml) -This workflow runs on all pull requests to analyze any potential changes to the specification. It uses the [coverage](#coverage) tool and [openapi-changes](https://pb33f.io/openapi-changes/) to calculate coverage metrics and provide a report on the changes when comparing with the commit at which the PR was branched off. +This workflow runs on all pull requests to analyze any potential changes to the specification. It uses the [coverage](#coverage) tool and [openapi-changes](https://pb33f.io/openapi-changes/) to calculate coverage metrics and provide a report on the changes when comparing with the commit at which the PR was branched off. In addition, the "Display Coverage Checklist" step will output a list of OpenSearch APIs with a checkbox indicating whether the API exists in the spec that can be directly copy-pasted into the [Add Missing API Specs issue](https://github.com/opensearch-project/opensearch-api-specification/issues/168). ### [Build](.github/workflows/build.yml) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 9e59bd2f..fbb0802d 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -422,7 +422,7 @@ entry: Snapshot builds of OpenSearch are available on Docker Hub under [opensearchstaging/opensearch/tags](https://hub.docker.com/r/opensearchstaging/opensearch/tags). -The following example in the test matrix will use [a snapshot build of OpenSearch 2.18](https://hub.docker.com/layers/opensearchstaging/opensearch/2.18.0/images/sha256-504a9c42bc1b13cb47b39a29db8a9d300d01b8851fb95dbb9db6770f478e45b5?context=explore) to execute the default test suite in [tests/default](tests/default/). +The following example in the test matrix will use [a snapshot build of OpenSearch 2.18](https://hub.docker.com/r/opensearchstaging/opensearch/tags?name=2.18) to execute the default test suite in [tests/default](tests/default/). ```yaml - version: 2.18.0