From 566ae1196c6d5930ef4d3320c05a2bb4ca7e1a0a Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 8 Dec 2024 06:22:13 +0000 Subject: [PATCH 01/44] test: init crl e2e test Signed-off-by: Juncheng Zhu --- Makefile | 20 +- scripts/gen_crl_testing_certs.sh | 229 ++++++++++++++++++ test/bats/cli-test.bats | 5 + .../tests/config/config_notation_crl.json | 50 ++++ 4 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 scripts/gen_crl_testing_certs.sh create mode 100644 test/bats/tests/config/config_notation_crl.json diff --git a/Makefile b/Makefile index fde9cad09..4eb411cb0 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ test-e2e: generate-rotation-certs EXPIRING_CERT_DIR=.staging/rotation/expiring-certs CERT_DIR=.staging/rotation GATEKEEPER_VERSION=${GATEKEEPER_VERSION} bats -t ${BATS_PLUGIN_TESTS_FILE} .PHONY: test-e2e-cli -test-e2e-cli: e2e-dependencies e2e-create-local-registry e2e-notation-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-trivy-setup e2e-schemavalidator-setup e2e-vulnerabilityreport-setup +test-e2e-cli: e2e-dependencies e2e-create-local-registry e2e-notation-setup e2e-notation-leaf-cert-setup e2e-notation-crl-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-trivy-setup e2e-schemavalidator-setup e2e-vulnerabilityreport-setup rm ${GOCOVERDIR} -rf mkdir ${GOCOVERDIR} -p RATIFY_DIR=${INSTALL_DIR} TEST_REGISTRY=${TEST_REGISTRY} ${GITHUB_WORKSPACE}/bin/bats -t ${BATS_CLI_TESTS_FILE} @@ -327,6 +327,24 @@ e2e-notation-leaf-cert-setup: rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "leaf-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:leafSigned --descriptor | jq .digest | xargs` +e2e-notation-crl-setup: + mkdir -p .staging/notation/crl-test + mkdir -p ~/.config/notation/truststore/x509/ca/crl-test + ./scripts/generate-crl-testing_certs.sh .staging/notation/crl-test + cp .staging/notation/crl-test/leaf.crt ~/.config/notation/truststore/x509/ca/crl-test/leaf.crt + cp .staging/notation/crl-test/ca.crt ~/.config/notation/truststore/x509/ca/crl-test/root.crt + cat .staging/notation/crl-test/ca.crt >> .staging/notation/crl-test/leaf.crt + + jq '.keys += [{"name":"crl-test","keyPath":".staging/notation/crl-test/leaf.key","certPath":".staging/notation/crl-test/leaf.crt"}]' ~/.config/notation/signingkeys.json > tmp && mv tmp ~/.config/notation/signingkeys.json + + printf 'FROM ${ALPINE_IMAGE}\nCMD ["echo", "notation crl signed image"]' > .staging/notation/Dockerfile + docker buildx create --use + docker buildx build --output type=oci,dest=.staging/notation/notation.tar -t notation:v0 .staging/notation + ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl + rm .staging/notation/notation.tar + NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` + + e2e-cosign-setup: rm -rf .staging/cosign mkdir -p .staging/cosign diff --git a/scripts/gen_crl_testing_certs.sh b/scripts/gen_crl_testing_certs.sh new file mode 100644 index 000000000..c25cdaa84 --- /dev/null +++ b/scripts/gen_crl_testing_certs.sh @@ -0,0 +1,229 @@ +#!/bin/bash -ex +# Copyright The Notary Project Authors. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file include the script to generate testing certificates for CRL testing. +# The generated files are: +# - certchain_with_crl.pem: the fullchain file that includes the leaf +# certificate with CRL, intermediate certificate with invalid OCSP and valid +# CRL, and the root certificate. +# - leaf.crl: the CRL file that includes the revoked leaf certificate. +# - leaf.key: the private key of the leaf certificate. +# - leaf_revoked.crl: the CRL file that includes the revoked leaf certificate. +# - intermediate.crl: the CRL file that includes the intermediate certificate. +# - intermediate_revoked.crl: the CRL file that includes the revoked intermediate +# - root.crt: the root certificate. +# +# Note: The script will not run in the pipeline, but we need to keep it for +# future maintenance because generating those test certificates with CRL is not +# easy. + +# Create root CA configuration file +cat > root.cnf < demoCA/serial +echo '1002' > demoCA/crlnumber + +# Generate root private key +openssl genrsa -out root.key 2048 + +# Generate self-signed root certificate with extensions +openssl req -x509 -new -key root.key -sha256 -days 36500 -out root.crt \ + -config root.cnf -extensions v3_ca + +# Update intermediate.cnf to include [ca] and [CA_default] sections +cat > intermediate.cnf < intermediateCA/serial +echo '1000' > intermediateCA/crlnumber + +# Generate intermediate private key +openssl genrsa -out intermediate.key 2048 + +# Generate intermediate CSR +openssl req -new -key intermediate.key -out intermediate.csr -config intermediate.cnf + +# Sign intermediate certificate with root CA +openssl ca -config root.cnf -in intermediate.csr -out intermediate.crt -batch -extensions v3_intermediate_ca -extfile intermediate.cnf -notext + +# Update leaf.cnf to remove OCSP server +cat > leaf.cnf < certchain_with_crl.pem diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index abbe8e24a..3e99f821c 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -34,6 +34,11 @@ load helpers assert_cmd_verify_failure } +@test "notation verifier crl test" { + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_success +} + @test "notation verifier with type test" { run bin/ratify verify -c $RATIFY_DIR/config_notation_verifier_with_type.json -s $TEST_REGISTRY/notation:leafSigned assert_cmd_verify_success_with_type diff --git a/test/bats/tests/config/config_notation_crl.json b/test/bats/tests/config/config_notation_crl.json new file mode 100644 index 000000000..3fb4aea33 --- /dev/null +++ b/test/bats/tests/config/config_notation_crl.json @@ -0,0 +1,50 @@ +{ + "store": { + "version": "1.0.0", + "plugins": [ + { + "name": "oras", + "cosignEnabled": true, + "useHttp": true + } + ] + }, + "policy": { + "version": "1.0.0", + "plugin": { + "name": "configPolicy" + } + }, + "verifier": { + "version": "1.0.0", + "plugins": [ + { + "name": "notation", + "artifactTypes": "application/vnd.cncf.notary.signature", + "verificationCerts": [ + "~/.config/notation/truststore/x509/ca/crl-test/root.crt" + ], + "trustPolicyDoc": { + "version": "1.0", + "trustPolicies": [ + { + "name": "default", + "registryScopes": [ + "*" + ], + "signatureVerification": { + "level": "strict" + }, + "trustStores": [ + "ca:certs" + ], + "trustedIdentities": [ + "*" + ] + } + ] + } + } + ] + } +} \ No newline at end of file From b0c7a07d09bdabf22a3f742b3e8553b870780070 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 8 Dec 2024 06:36:16 +0000 Subject: [PATCH 02/44] test: init crl e2e test 2 Signed-off-by: Juncheng Zhu --- scripts/gen_crl_testing_certs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gen_crl_testing_certs.sh b/scripts/gen_crl_testing_certs.sh index c25cdaa84..cedb24339 100644 --- a/scripts/gen_crl_testing_certs.sh +++ b/scripts/gen_crl_testing_certs.sh @@ -1,11 +1,11 @@ #!/bin/bash -ex -# Copyright The Notary Project Authors. +# Copyright The Ratify Authors. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# + # http://www.apache.org/licenses/LICENSE-2.0 -# + # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 17ce77c2549759399cae14958b197104d1ba6fff Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 8 Dec 2024 06:55:41 +0000 Subject: [PATCH 03/44] test: init crl e2e test 3 Signed-off-by: Juncheng Zhu --- Makefile | 2 +- .../{gen_crl_testing_certs.sh => generate-crl-testing-certs.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{gen_crl_testing_certs.sh => generate-crl-testing-certs.sh} (100%) diff --git a/Makefile b/Makefile index 4eb411cb0..58349713f 100644 --- a/Makefile +++ b/Makefile @@ -330,7 +330,7 @@ e2e-notation-leaf-cert-setup: e2e-notation-crl-setup: mkdir -p .staging/notation/crl-test mkdir -p ~/.config/notation/truststore/x509/ca/crl-test - ./scripts/generate-crl-testing_certs.sh .staging/notation/crl-test + ./scripts/generate-crl-testing-certs.sh .staging/notation/crl-test cp .staging/notation/crl-test/leaf.crt ~/.config/notation/truststore/x509/ca/crl-test/leaf.crt cp .staging/notation/crl-test/ca.crt ~/.config/notation/truststore/x509/ca/crl-test/root.crt cat .staging/notation/crl-test/ca.crt >> .staging/notation/crl-test/leaf.crt diff --git a/scripts/gen_crl_testing_certs.sh b/scripts/generate-crl-testing-certs.sh similarity index 100% rename from scripts/gen_crl_testing_certs.sh rename to scripts/generate-crl-testing-certs.sh From 5e2f3d5169763ff95f7c5bce489c5eece5073c33 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 8 Dec 2024 23:42:13 +0000 Subject: [PATCH 04/44] test: crl e2e test add host Signed-off-by: Juncheng Zhu --- Makefile | 7 +++ scripts/crl_server.py | 81 +++++++++++++++++++++++++++ scripts/generate-crl-testing-certs.sh | 0 3 files changed, 88 insertions(+) create mode 100755 scripts/crl_server.py mode change 100644 => 100755 scripts/generate-crl-testing-certs.sh diff --git a/Makefile b/Makefile index 58349713f..26f6bd840 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,10 @@ test: clean: go clean rm ./bin/${BINARY_NAME} + +clean-crl-server: + echo "Stopping CRL server..." + kill ${CRL_SERVER_PID} .PHONY: deploy-demo deploy-demo: deploy-gatekeeper deploy-ratify deploy-demo-constraints @@ -343,6 +347,9 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` + # run the CRL server in the background + python3 ./scripts/crl_server.py & + CRL_SERVER_PID=$(shell $!) e2e-cosign-setup: diff --git a/scripts/crl_server.py b/scripts/crl_server.py new file mode 100755 index 000000000..bc3365c74 --- /dev/null +++ b/scripts/crl_server.py @@ -0,0 +1,81 @@ +# Copyright The Ratify Authors. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import http.server +import socketserver +import os + +PORT = 10086 +DATA_DIR = './testdata/config/crl' +leaf_crl = 'leaf.crl' +intermediate_crl = 'intermediate.crl' + + +class CRLRequestHandler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + global leaf_crl + global intermediate_crl + if self.path == '/leaf.crl': + file_path = os.path.join(DATA_DIR, leaf_crl) + self.crl_response(file_path) + elif self.path == '/intermediate.crl': + file_path = os.path.join(DATA_DIR, intermediate_crl) + self.crl_response(file_path) + else: + self.send_error(404, 'Not Found') + + def crl_response(self, file_path): + if os.path.exists(file_path): + self.send_response(200) + self.send_header('Content-Type', 'application/pkix-crl') + self.end_headers() + with open(file_path, 'rb') as f: + self.wfile.write(f.read()) + else: + self.send_error(404, 'File Not Found') + + def do_POST(self): + global leaf_crl + global intermediate_crl + if self.path == '/leaf/revoke': + leaf_crl = 'leaf_revoked.crl' + self.post_response() + elif self.path == '/leaf/unrevoke': + leaf_crl = 'leaf.crl' + self.post_response() + elif self.path == '/leaf/expired': + leaf_crl = 'leaf_expired.crl' + self.post_response() + elif self.path == '/intermediate/revoke': + intermediate_crl = 'intermediate_revoked.crl' + self.post_response() + elif self.path == '/intermediate/unrevoke': + intermediate_crl = 'intermediate.crl' + self.post_response() + else: + self.send_error(404, 'Not Found') + + def post_response(self): + self.send_response(201) + self.end_headers() + self.wfile.write(b'ok') + +class ReusableTCPServer(socketserver.TCPServer): + allow_reuse_address = True + +with ReusableTCPServer(('', PORT), CRLRequestHandler) as httpd: + print(f"Serving at port {PORT}") + try: + httpd.serve_forever() + finally: + httpd.server_close() \ No newline at end of file diff --git a/scripts/generate-crl-testing-certs.sh b/scripts/generate-crl-testing-certs.sh old mode 100644 new mode 100755 From afd8791970ac461bfdfa3ce5ab9e34e78f698df4 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 8 Dec 2024 23:45:38 +0000 Subject: [PATCH 05/44] test: crl e2e test add host 2 Signed-off-by: Juncheng Zhu --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 26f6bd840..76ccfc346 100644 --- a/Makefile +++ b/Makefile @@ -114,8 +114,7 @@ clean: rm ./bin/${BINARY_NAME} clean-crl-server: - echo "Stopping CRL server..." - kill ${CRL_SERVER_PID} + $(shell echo "Stopping CRL server..." && kill ${CRL_SERVER_PID}) .PHONY: deploy-demo deploy-demo: deploy-gatekeeper deploy-ratify deploy-demo-constraints From 9ce26a76e435251b401ab62c0f1e3df3fe5df7f8 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 9 Dec 2024 00:21:19 +0000 Subject: [PATCH 06/44] test: update e2e script Signed-off-by: Juncheng Zhu --- scripts/generate-crl-testing-certs.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/generate-crl-testing-certs.sh b/scripts/generate-crl-testing-certs.sh index cedb24339..9ce6c1041 100755 --- a/scripts/generate-crl-testing-certs.sh +++ b/scripts/generate-crl-testing-certs.sh @@ -28,6 +28,13 @@ # future maintenance because generating those test certificates with CRL is not # easy. +set -o errexit +set -o nounset +set -o pipefail + +CERT_DIR=$1 + +generate() { # Create root CA configuration file cat > root.cnf < certchain_with_crl.pem + +} + +rm -r ${CERT_DIR} || true +mkdir -p ${CERT_DIR} +pushd "${CERT_DIR}" +generate +popd \ No newline at end of file From 543d43c1768998a375e0e0da98838e6f61ad1fec Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 9 Dec 2024 00:26:11 +0000 Subject: [PATCH 07/44] test: update e2e script 2 Signed-off-by: Juncheng Zhu --- scripts/crl_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/crl_server.py b/scripts/crl_server.py index bc3365c74..9f9294a3e 100755 --- a/scripts/crl_server.py +++ b/scripts/crl_server.py @@ -16,7 +16,7 @@ import os PORT = 10086 -DATA_DIR = './testdata/config/crl' +DATA_DIR = '.staging/notation/crl-test' leaf_crl = 'leaf.crl' intermediate_crl = 'intermediate.crl' From e5a719c357517ce074840890095bebf8f1b0df4f Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 9 Dec 2024 00:38:33 +0000 Subject: [PATCH 08/44] test: update e2e script 3 Signed-off-by: Juncheng Zhu --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 76ccfc346..1cd7d9cb0 100644 --- a/Makefile +++ b/Makefile @@ -335,8 +335,8 @@ e2e-notation-crl-setup: mkdir -p ~/.config/notation/truststore/x509/ca/crl-test ./scripts/generate-crl-testing-certs.sh .staging/notation/crl-test cp .staging/notation/crl-test/leaf.crt ~/.config/notation/truststore/x509/ca/crl-test/leaf.crt - cp .staging/notation/crl-test/ca.crt ~/.config/notation/truststore/x509/ca/crl-test/root.crt - cat .staging/notation/crl-test/ca.crt >> .staging/notation/crl-test/leaf.crt + cp .staging/notation/crl-test/root.crt ~/.config/notation/truststore/x509/ca/crl-test/root.crt + cat .staging/notation/crl-test/root.crt >> .staging/notation/crl-test/leaf.crt jq '.keys += [{"name":"crl-test","keyPath":".staging/notation/crl-test/leaf.key","certPath":".staging/notation/crl-test/leaf.crt"}]' ~/.config/notation/signingkeys.json > tmp && mv tmp ~/.config/notation/signingkeys.json From c2615b069de1cfe654eb7f0b903a38035f35d701 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 9 Dec 2024 01:51:34 +0000 Subject: [PATCH 09/44] test: update e2e script 4 Signed-off-by: Juncheng Zhu --- Makefile | 5 ++--- test/bats/tests/config/config_notation_crl.json | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1cd7d9cb0..0d89b147c 100644 --- a/Makefile +++ b/Makefile @@ -334,11 +334,10 @@ e2e-notation-crl-setup: mkdir -p .staging/notation/crl-test mkdir -p ~/.config/notation/truststore/x509/ca/crl-test ./scripts/generate-crl-testing-certs.sh .staging/notation/crl-test - cp .staging/notation/crl-test/leaf.crt ~/.config/notation/truststore/x509/ca/crl-test/leaf.crt cp .staging/notation/crl-test/root.crt ~/.config/notation/truststore/x509/ca/crl-test/root.crt - cat .staging/notation/crl-test/root.crt >> .staging/notation/crl-test/leaf.crt + cp .staging/notation/crl-test/certchain_with_crl.pem ~/.config/notation/truststore/x509/ca/crl-test/certchain_with_crl.pem - jq '.keys += [{"name":"crl-test","keyPath":".staging/notation/crl-test/leaf.key","certPath":".staging/notation/crl-test/leaf.crt"}]' ~/.config/notation/signingkeys.json > tmp && mv tmp ~/.config/notation/signingkeys.json + jq '.keys += [{"name":"crl-test","keyPath":".staging/notation/crl-test/leaf.key","certPath":".staging/notation/crl-test/certchain_with_crl.pem"}]' ~/.config/notation/signingkeys.json > tmp && mv tmp ~/.config/notation/signingkeys.json printf 'FROM ${ALPINE_IMAGE}\nCMD ["echo", "notation crl signed image"]' > .staging/notation/Dockerfile docker buildx create --use diff --git a/test/bats/tests/config/config_notation_crl.json b/test/bats/tests/config/config_notation_crl.json index 3fb4aea33..88dcfb952 100644 --- a/test/bats/tests/config/config_notation_crl.json +++ b/test/bats/tests/config/config_notation_crl.json @@ -33,7 +33,7 @@ "*" ], "signatureVerification": { - "level": "strict" + "level": "audit" }, "trustStores": [ "ca:certs" From 31e37ae5d4068231c7479c095f866c601acfd628 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 9 Dec 2024 23:54:11 +0000 Subject: [PATCH 10/44] test: update e2e script 4 Signed-off-by: Juncheng Zhu --- Makefile | 3 --- test/bats/cli-test.bats | 4 ++++ test/bats/tests/config/config_notation_crl.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0d89b147c..ecb2285c9 100644 --- a/Makefile +++ b/Makefile @@ -112,9 +112,6 @@ test: clean: go clean rm ./bin/${BINARY_NAME} - -clean-crl-server: - $(shell echo "Stopping CRL server..." && kill ${CRL_SERVER_PID}) .PHONY: deploy-demo deploy-demo: deploy-gatekeeper deploy-ratify deploy-demo-constraints diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 3e99f821c..4f7a6e716 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -37,6 +37,10 @@ load helpers @test "notation verifier crl test" { run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success + + $(shell echo "Stopping CRL server..." && kill ${CRL_SERVER_PID}) + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_failure } @test "notation verifier with type test" { diff --git a/test/bats/tests/config/config_notation_crl.json b/test/bats/tests/config/config_notation_crl.json index 88dcfb952..3fb4aea33 100644 --- a/test/bats/tests/config/config_notation_crl.json +++ b/test/bats/tests/config/config_notation_crl.json @@ -33,7 +33,7 @@ "*" ], "signatureVerification": { - "level": "audit" + "level": "strict" }, "trustStores": [ "ca:certs" From 8c7dccb38cfe640d0e4313a476e400c057397ea8 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Tue, 10 Dec 2024 00:08:34 +0000 Subject: [PATCH 11/44] test: update e2e script 5 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 4f7a6e716..3e99f821c 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -37,10 +37,6 @@ load helpers @test "notation verifier crl test" { run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - - $(shell echo "Stopping CRL server..." && kill ${CRL_SERVER_PID}) - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_failure } @test "notation verifier with type test" { From d3ad414e6d61ded20cd8f93bc8cff2ea5dc2bebd Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 11 Dec 2024 02:57:53 +0000 Subject: [PATCH 12/44] test: e2e k8s Signed-off-by: Juncheng Zhu --- Makefile | 2 +- test/bats/base-test.bats | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ecb2285c9..4f244b911 100644 --- a/Makefile +++ b/Makefile @@ -556,7 +556,7 @@ e2e-build-crd-image: load-build-crd-image: kind load docker-image --name kind localbuildcrd:test -e2e-deploy-base-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-base-image +e2e-deploy-base-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-notation-crl-setup e2e-cosign-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-base-image printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json ./.staging/helm/linux-amd64/helm install ${RATIFY_NAME} \ diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index bc30766df..5ff4482e4 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -125,6 +125,16 @@ RATIFY_NAMESPACE=gatekeeper-system assert_success } +@test "notation test crl" { + teardown() { + echo "cleaning up" + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' + } + + run kubectl run demo --namespace default --image=registry:5000/notation:crl + assert_success +} + @test "notation test with certs across namespace" { teardown() { echo "cleaning up" From bee2ce39f836f48b239cce1de5ec6658dc747b49 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 11 Dec 2024 09:38:55 +0000 Subject: [PATCH 13/44] test: e2e k8s 2 Signed-off-by: Juncheng Zhu --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4f244b911..adda7c62a 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ delete-gatekeeper: helm delete gatekeeper --namespace ${GATEKEEPER_NAMESPACE} .PHONY: test-e2e -test-e2e: generate-rotation-certs +test-e2e: generate-rotation-certs e2e-notation-crl-setup timeout 20m bats -t ${BATS_BASE_TESTS_FILE} EXPIRING_CERT_DIR=.staging/rotation/expiring-certs CERT_DIR=.staging/rotation GATEKEEPER_VERSION=${GATEKEEPER_VERSION} bats -t ${BATS_PLUGIN_TESTS_FILE} @@ -556,7 +556,7 @@ e2e-build-crd-image: load-build-crd-image: kind load docker-image --name kind localbuildcrd:test -e2e-deploy-base-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-notation-crl-setup e2e-cosign-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-base-image +e2e-deploy-base-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-base-image printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json ./.staging/helm/linux-amd64/helm install ${RATIFY_NAME} \ From afa4dcb903c195a0a35d6432d645de80afddc431 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 11 Dec 2024 22:19:00 +0000 Subject: [PATCH 14/44] test: update test Signed-off-by: Juncheng Zhu --- Makefile | 3 --- test/bats/base-test.bats | 2 +- test/bats/cli-test.bats | 2 ++ test/bats/helpers.bash | 6 ++++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index adda7c62a..6d0d4c05d 100644 --- a/Makefile +++ b/Makefile @@ -342,9 +342,6 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` - # run the CRL server in the background - python3 ./scripts/crl_server.py & - CRL_SERVER_PID=$(shell $!) e2e-cosign-setup: diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index 5ff4482e4..88c476bcd 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -130,7 +130,7 @@ RATIFY_NAMESPACE=gatekeeper-system echo "cleaning up" wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' } - + run_crl_server run kubectl run demo --namespace default --image=registry:5000/notation:crl assert_success } diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 3e99f821c..94819cd95 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,6 +35,8 @@ load helpers } @test "notation verifier crl test" { + run_crl_server + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success } diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index d8c872686..aa95ab664 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -111,3 +111,9 @@ wait_for_process() { done return 1 } + +# run the CRL server in the background +run_crl_server() { + python3 ./scripts/crl_server.py & + echo $(shell $!) +} \ No newline at end of file From c249d88fd38225711e61fdad31cca102a7eb6f86 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 00:24:54 +0000 Subject: [PATCH 15/44] test: update e2e Signed-off-by: Juncheng Zhu --- scripts/generate-crl-testing-certs.sh | 6 ++--- test/bats/base-test.bats | 14 ++++++++++- test/bats/helpers.bash | 4 ++++ .../config_v1beta1_verifier_notation_crl.yaml | 24 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml diff --git a/scripts/generate-crl-testing-certs.sh b/scripts/generate-crl-testing-certs.sh index 9ce6c1041..1ba529afd 100755 --- a/scripts/generate-crl-testing-certs.sh +++ b/scripts/generate-crl-testing-certs.sh @@ -205,7 +205,7 @@ openssl ca -config intermediate.cnf -in leaf.csr -out leaf.crt -batch -extension # Generate intermediate CRL using root.cnf (before revocation) openssl ca -config root.cnf -gencrl -out intermediate.crl -# Convert root CRL to DER format +# Convert intermediate CRL to DER format openssl crl -in intermediate.crl -outform der -out intermediate.crl # Revoke intermediate certificate using root CA @@ -214,7 +214,7 @@ openssl ca -config root.cnf -revoke intermediate.crt # Generate intermediate CRL including revoked intermediate certificate openssl ca -config root.cnf -gencrl -out intermediate_revoked.crl -# Convert intermediate CRL to DER format +# Convert intermediate revoked CRL to DER format openssl crl -in intermediate_revoked.crl -outform der -out intermediate_revoked.crl # Generate leaf CRL @@ -229,7 +229,7 @@ openssl ca -config intermediate.cnf -revoke leaf.crt # Generate leaf CRL including revoked leaf certificate openssl ca -config intermediate.cnf -gencrl -out leaf_revoked.crl -# Convert leaf CRL to DER format +# Convert leaf revoked CRL to DER format openssl crl -in leaf_revoked.crl -outform der -out leaf_revoked.crl # merge leaf cert and root cert to create fullchain file diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index 88c476bcd..3b73a12cc 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -129,8 +129,20 @@ RATIFY_NAMESPACE=gatekeeper-system teardown() { echo "cleaning up" wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' + + # restore the original notation verifier for other tests + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl replace -f ./config/samples/clustered/verifier/config_v1beta1_verifier_notation.yaml' } - run_crl_server + run_crl_server + expose_localhost + + # add the tsaroot certificate as an inline key management provider + cat ./test/bats/tests/config/config_v1beta1_keymanagementprovider_inline.yaml >> crlkmprovider.yaml + cat .staging/notation/crl-test/root.crt | sed 's/^/ /g' >> crlkmprovider.yaml + run kubectl apply -f crlkmprovider.yaml --namespace ${RATIFY_NAMESPACE} + assert_success + run kubectl replace -f ./test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml + run kubectl run demo --namespace default --image=registry:5000/notation:crl assert_success } diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index aa95ab664..e9db0c62b 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -116,4 +116,8 @@ wait_for_process() { run_crl_server() { python3 ./scripts/crl_server.py & echo $(shell $!) +} + +expose_localhost() { + kubectl proxy --address='0.0.0.0' --accept-hosts='.*' } \ No newline at end of file diff --git a/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml b/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml new file mode 100644 index 000000000..56b5bb00b --- /dev/null +++ b/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml @@ -0,0 +1,24 @@ +apiVersion: config.ratify.deislabs.io/v1beta1 +kind: Verifier +metadata: + name: verifier-notation +spec: + name: notation + artifactTypes: application/vnd.cncf.notary.signature + parameters: + verificationCertStores: + ca: + ca-crl: + - keymanagementprovider-inline + trustPolicyDoc: + version: "1.0" + trustPolicies: + - name: default + registryScopes: + - "*" + signatureVerification: + level: strict + trustStores: + - ca:ca-crl + trustedIdentities: + - "*" From 3570acf4491ec2c322bbc791e75356831252c7f5 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 04:13:44 +0000 Subject: [PATCH 16/44] test: update e2e test Signed-off-by: Juncheng Zhu --- test/bats/base-test.bats | 2 -- test/bats/cli-test.bats | 7 ++++++- .../tests/config/config_v1beta1_verifier_notation_crl.yaml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index 3b73a12cc..a20f3cdb0 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -133,8 +133,6 @@ RATIFY_NAMESPACE=gatekeeper-system # restore the original notation verifier for other tests wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl replace -f ./config/samples/clustered/verifier/config_v1beta1_verifier_notation.yaml' } - run_crl_server - expose_localhost # add the tsaroot certificate as an inline key management provider cat ./test/bats/tests/config/config_v1beta1_keymanagementprovider_inline.yaml >> crlkmprovider.yaml diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 94819cd95..421310e62 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,10 +35,15 @@ load helpers } @test "notation verifier crl test" { - run_crl_server + run python3 ./scripts/crl_server.py & CRL_SERVER_PID = $(shell $!) run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success + + run kill $CRL_SERVER_PID + + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_failure } @test "notation verifier with type test" { diff --git a/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml b/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml index 56b5bb00b..1449898a7 100644 --- a/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml +++ b/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml @@ -17,7 +17,7 @@ spec: registryScopes: - "*" signatureVerification: - level: strict + level: audit trustStores: - ca:ca-crl trustedIdentities: From d3a1302f6f54d28c40083cd8745db481b1eacea0 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 06:11:54 +0000 Subject: [PATCH 17/44] test: update e2e CLI Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 4 ++-- test/bats/helpers.bash | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 421310e62..23449a5c0 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,12 +35,12 @@ load helpers } @test "notation verifier crl test" { - run python3 ./scripts/crl_server.py & CRL_SERVER_PID = $(shell $!) + python3 ./scripts/crl_server.py & CRL_SERVER_PID = $(shell $!) run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - run kill $CRL_SERVER_PID + kill $CRL_SERVER_PID run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index e9db0c62b..d8c872686 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -111,13 +111,3 @@ wait_for_process() { done return 1 } - -# run the CRL server in the background -run_crl_server() { - python3 ./scripts/crl_server.py & - echo $(shell $!) -} - -expose_localhost() { - kubectl proxy --address='0.0.0.0' --accept-hosts='.*' -} \ No newline at end of file From c4d46c57155d0281e6501ca7541bd0b838550dfb Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 06:21:23 +0000 Subject: [PATCH 18/44] test: update e2e CLI 2 Signed-off-by: Juncheng Zhu --- scripts/generate-crl-testing-certs.sh | 2 +- test/bats/cli-test.bats | 2 +- test/bats/helpers.bash | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/generate-crl-testing-certs.sh b/scripts/generate-crl-testing-certs.sh index 1ba529afd..2fd13481b 100755 --- a/scripts/generate-crl-testing-certs.sh +++ b/scripts/generate-crl-testing-certs.sh @@ -17,7 +17,7 @@ # - certchain_with_crl.pem: the fullchain file that includes the leaf # certificate with CRL, intermediate certificate with invalid OCSP and valid # CRL, and the root certificate. -# - leaf.crl: the CRL file that includes the revoked leaf certificate. +# - leaf.crl: the CRL file that includes the leaf certificate. # - leaf.key: the private key of the leaf certificate. # - leaf_revoked.crl: the CRL file that includes the revoked leaf certificate. # - intermediate.crl: the CRL file that includes the intermediate certificate. diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 23449a5c0..dc4180e98 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,7 +35,7 @@ load helpers } @test "notation verifier crl test" { - python3 ./scripts/crl_server.py & CRL_SERVER_PID = $(shell $!) + CRL_SERVER_PID = run_crl_server run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index d8c872686..729d64a20 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -111,3 +111,8 @@ wait_for_process() { done return 1 } + +run_crl_server() { + python3 ./scripts/crl_server.py & CRL_SERVER_PID=$! + return $CRL_SERVER_PID +} \ No newline at end of file From 3f3e700db69cecd1f2382c36df1121a9c2032280 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 07:30:13 +0000 Subject: [PATCH 19/44] test: update e2e CLI 3 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 4 ++-- test/bats/helpers.bash | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index dc4180e98..978b189c9 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,12 +35,12 @@ load helpers } @test "notation verifier crl test" { - CRL_SERVER_PID = run_crl_server + run_crl_server run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - kill $CRL_SERVER_PID + pkill python3 run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 729d64a20..2b18e2aaa 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -113,6 +113,5 @@ wait_for_process() { } run_crl_server() { - python3 ./scripts/crl_server.py & CRL_SERVER_PID=$! - return $CRL_SERVER_PID + python3 ./scripts/crl_server.py } \ No newline at end of file From 5489ca229c2ba1e3ba6a263ee6258656f9e19a2e Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 07:40:56 +0000 Subject: [PATCH 20/44] test: update e2e CLI 4 Signed-off-by: Juncheng Zhu --- Makefile | 2 +- test/bats/helpers.bash | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6d0d4c05d..bd09b2f42 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ delete-gatekeeper: helm delete gatekeeper --namespace ${GATEKEEPER_NAMESPACE} .PHONY: test-e2e -test-e2e: generate-rotation-certs e2e-notation-crl-setup +test-e2e: generate-rotation-certs timeout 20m bats -t ${BATS_BASE_TESTS_FILE} EXPIRING_CERT_DIR=.staging/rotation/expiring-certs CERT_DIR=.staging/rotation GATEKEEPER_VERSION=${GATEKEEPER_VERSION} bats -t ${BATS_PLUGIN_TESTS_FILE} diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 2b18e2aaa..1c6a22db1 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -113,5 +113,6 @@ wait_for_process() { } run_crl_server() { - python3 ./scripts/crl_server.py + python3 ./scripts/crl_server.py & + echo "CRL server started" } \ No newline at end of file From 82c29c0d9b6c6e96347ecd4191cc6dec1a7f8087 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 07:46:42 +0000 Subject: [PATCH 21/44] test: update e2e CLI 5 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 978b189c9..973a9cdad 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -40,7 +40,7 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - pkill python3 + pkill -9 python3 run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure From 2088512b02e5368d62f052706e5372fcbc65895b Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 12 Dec 2024 23:06:25 +0000 Subject: [PATCH 22/44] test: update e2e CLI 6 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 973a9cdad..581272733 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,15 +35,13 @@ load helpers } @test "notation verifier crl test" { - run_crl_server - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success + assert_cmd_verify_failure - pkill -9 python3 + run_crl_server run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_failure + assert_cmd_verify_success } @test "notation verifier with type test" { From 4224702f768655e8d6951c9d66935207148cf4fd Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Fri, 13 Dec 2024 00:29:15 +0000 Subject: [PATCH 23/44] test: update e2e CLI 7 Signed-off-by: Juncheng Zhu --- Makefile | 2 +- scripts/generate-crl-testing-certs.sh | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index bd09b2f42..9b55481d5 100644 --- a/Makefile +++ b/Makefile @@ -577,7 +577,7 @@ e2e-deploy-base-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-cosi rm mount_config.json -e2e-deploy-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-trivy-setup e2e-schemavalidator-setup e2e-vulnerabilityreport-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-image load-local-ratify-image e2e-helm-deploy-ratify +e2e-deploy-ratify: e2e-notation-setup e2e-notation-leaf-cert-setup e2e-notation-crl-setup e2e-cosign-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-trivy-setup e2e-schemavalidator-setup e2e-vulnerabilityreport-setup e2e-inlinecert-setup e2e-build-crd-image load-build-crd-image e2e-build-local-ratify-image load-local-ratify-image e2e-helm-deploy-ratify e2e-build-local-ratify-base-image: docker build --progress=plain --no-cache \ diff --git a/scripts/generate-crl-testing-certs.sh b/scripts/generate-crl-testing-certs.sh index 2fd13481b..a02391aef 100755 --- a/scripts/generate-crl-testing-certs.sh +++ b/scripts/generate-crl-testing-certs.sh @@ -24,9 +24,6 @@ # - intermediate_revoked.crl: the CRL file that includes the revoked intermediate # - root.crt: the root certificate. # -# Note: The script will not run in the pipeline, but we need to keep it for -# future maintenance because generating those test certificates with CRL is not -# easy. set -o errexit set -o nounset From 3508fc70a37c0ea1f5208cad1dd55fe74c6eb00c Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 15 Dec 2024 07:19:16 +0000 Subject: [PATCH 24/44] fix: update test Signed-off-by: Juncheng Zhu --- test/bats/helpers.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 1c6a22db1..c6e3a1f84 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -113,6 +113,5 @@ wait_for_process() { } run_crl_server() { - python3 ./scripts/crl_server.py & - echo "CRL server started" + python3 ./scripts/crl_server.py & echo "$!" } \ No newline at end of file From f7b177c7156b3a58c1ca9174215c67cc616bff6b Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 15 Dec 2024 08:39:18 +0000 Subject: [PATCH 25/44] fix: update test Signed-off-by: Juncheng Zhu --- test/bats/helpers.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index c6e3a1f84..0f7cd580a 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -113,5 +113,6 @@ wait_for_process() { } run_crl_server() { - python3 ./scripts/crl_server.py & echo "$!" + python3 ./scripts/crl_server.py & + echo $! } \ No newline at end of file From aab1e2c3b18fe5c7967f373ba573097c021ff011 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 15 Dec 2024 13:12:05 +0000 Subject: [PATCH 26/44] fix: update test 2 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 2 ++ test/bats/helpers.bash | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 581272733..6f5611f35 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -42,6 +42,8 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success + + stop_crl_server } @test "notation verifier with type test" { diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 0f7cd580a..36d9a7e17 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -113,6 +113,9 @@ wait_for_process() { } run_crl_server() { - python3 ./scripts/crl_server.py & - echo $! + python3 ./scripts/crl_server.py +} + +stop_crl_server() { + pkill -f crl_server.py } \ No newline at end of file From 7acc57ccf32cb2eabc90a91de31c88892aa6f462 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 15 Dec 2024 23:38:07 +0000 Subject: [PATCH 27/44] fix: update test 3 Signed-off-by: Juncheng Zhu --- Makefile | 2 +- test/bats/cli-test.bats | 8 +++----- test/bats/helpers.bash | 4 ---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 9b55481d5..fed8fb5e5 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` - + python3 ./scripts/crl_server.py & echo "crl server started" e2e-cosign-setup: rm -rf .staging/cosign diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 6f5611f35..506edd40b 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,15 +35,13 @@ load helpers } @test "notation verifier crl test" { - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_failure - - run_crl_server - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success stop_crl_server + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_failure + } @test "notation verifier with type test" { diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 36d9a7e17..dafb4243f 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -112,10 +112,6 @@ wait_for_process() { return 1 } -run_crl_server() { - python3 ./scripts/crl_server.py -} - stop_crl_server() { pkill -f crl_server.py } \ No newline at end of file From 94aba4ffbc0b546942cf8a53833cb7f6133f0168 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Sun, 15 Dec 2024 23:51:52 +0000 Subject: [PATCH 28/44] fix: update test 4 Signed-off-by: Juncheng Zhu --- scripts/crl_server.py | 4 +++- test/bats/cli-test.bats | 1 - test/bats/helpers.bash | 4 ---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/crl_server.py b/scripts/crl_server.py index 9f9294a3e..666cb94cb 100755 --- a/scripts/crl_server.py +++ b/scripts/crl_server.py @@ -22,10 +22,12 @@ class CRLRequestHandler(http.server.SimpleHTTPRequestHandler): + flag = False def do_GET(self): global leaf_crl global intermediate_crl - if self.path == '/leaf.crl': + if self.path == '/leaf.crl' && not self.flag: + flag = True file_path = os.path.join(DATA_DIR, leaf_crl) self.crl_response(file_path) elif self.path == '/intermediate.crl': diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 506edd40b..b9f461cf8 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -38,7 +38,6 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - stop_crl_server run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index dafb4243f..d8c872686 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -111,7 +111,3 @@ wait_for_process() { done return 1 } - -stop_crl_server() { - pkill -f crl_server.py -} \ No newline at end of file From 7b520f1435decf0fd8145974155cf4a92f03976f Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 00:08:43 +0000 Subject: [PATCH 29/44] fix: update test script 5 Signed-off-by: Juncheng Zhu --- scripts/crl_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/crl_server.py b/scripts/crl_server.py index 666cb94cb..193757bd4 100755 --- a/scripts/crl_server.py +++ b/scripts/crl_server.py @@ -19,13 +19,13 @@ DATA_DIR = '.staging/notation/crl-test' leaf_crl = 'leaf.crl' intermediate_crl = 'intermediate.crl' - +flag = False class CRLRequestHandler(http.server.SimpleHTTPRequestHandler): - flag = False def do_GET(self): global leaf_crl global intermediate_crl + global flag if self.path == '/leaf.crl' && not self.flag: flag = True file_path = os.path.join(DATA_DIR, leaf_crl) From 14e4cd31564df035f608679acb91d5212b962fe5 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 00:50:09 +0000 Subject: [PATCH 30/44] fix: update test script 6 Signed-off-by: Juncheng Zhu --- scripts/crl_server.py | 4 +--- test/bats/cli-test.bats | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/crl_server.py b/scripts/crl_server.py index 193757bd4..c6671bc34 100755 --- a/scripts/crl_server.py +++ b/scripts/crl_server.py @@ -19,15 +19,13 @@ DATA_DIR = '.staging/notation/crl-test' leaf_crl = 'leaf.crl' intermediate_crl = 'intermediate.crl' -flag = False class CRLRequestHandler(http.server.SimpleHTTPRequestHandler): def do_GET(self): global leaf_crl global intermediate_crl global flag - if self.path == '/leaf.crl' && not self.flag: - flag = True + if self.path == '/leaf.crl': file_path = os.path.join(DATA_DIR, leaf_crl) self.crl_response(file_path) elif self.path == '/intermediate.crl': diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index b9f461cf8..b5727e623 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -38,7 +38,7 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:unsigned assert_cmd_verify_failure } From 6f05a67c15e9580383b92c36372ffe2b463a53d3 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 08:16:45 +0000 Subject: [PATCH 31/44] fix: another test Signed-off-by: Juncheng Zhu --- Makefile | 1 - test/bats/cli-test.bats | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index fed8fb5e5..31ead534e 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,6 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` - python3 ./scripts/crl_server.py & echo "crl server started" e2e-cosign-setup: rm -rf .staging/cosign diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index b5727e623..d0d3f6f97 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -36,11 +36,13 @@ load helpers @test "notation verifier crl test" { run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success - - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:unsigned assert_cmd_verify_failure + timeout 60 python3 ./scripts/crl_server.py + + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_success + } @test "notation verifier with type test" { From ac1693ba7d29d5efe4c93c72fe9c52b47a8b288a Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 08:29:27 +0000 Subject: [PATCH 32/44] fix: another test 2 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index d0d3f6f97..82fa4b2f1 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -38,11 +38,12 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure - timeout 60 python3 ./scripts/crl_server.py + python3 ./scripts/crl_server.py & CRL_SERVER_PID=$! run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success + kill $CRL_SERVER_PID } @test "notation verifier with type test" { From 61dc0e1fa0bb8eafbcbad78c1d29b9fe18f8f151 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 08:40:29 +0000 Subject: [PATCH 33/44] fix: another test 3 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 82fa4b2f1..c5d2512c1 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -38,12 +38,12 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure - python3 ./scripts/crl_server.py & CRL_SERVER_PID=$! + python3 ./scripts/crl_server.py run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - kill $CRL_SERVER_PID + pkill -f ./scripts/crl_server.py } @test "notation verifier with type test" { From c80d3989417641c2fe2b13f5daefae72d4432044 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 08:42:18 +0000 Subject: [PATCH 34/44] fix: another test 4 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index c5d2512c1..cfa111ad4 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,15 +35,16 @@ load helpers } @test "notation verifier crl test" { + teardown() { + run sudo pkill -f python3 + } run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure - python3 ./scripts/crl_server.py + run python3 ./scripts/crl_server.py run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_success - - pkill -f ./scripts/crl_server.py } @test "notation verifier with type test" { From 4b334dd4ce76eb8ed467cc658d5b85ecd4d27606 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 08:50:46 +0000 Subject: [PATCH 35/44] fix: another test 5 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index cfa111ad4..411c81197 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -36,7 +36,7 @@ load helpers @test "notation verifier crl test" { teardown() { - run sudo pkill -f python3 + run sudo pkill -f ./scripts/crl_server.py } run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure From a4a737ce39708d536a6986b629e1e5fc53a18aec Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 21:09:38 +0000 Subject: [PATCH 36/44] fix: another test 6 Signed-off-by: Juncheng Zhu --- Makefile | 2 ++ test/bats/cli-test.bats | 9 +++------ test/bats/helpers.bash | 7 +++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 31ead534e..eab605e55 100644 --- a/Makefile +++ b/Makefile @@ -342,6 +342,8 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` + python3 ./scripts/crl_server.py + e2e-cosign-setup: rm -rf .staging/cosign diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 411c81197..56fba09c8 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,16 +35,13 @@ load helpers } @test "notation verifier crl test" { - teardown() { - run sudo pkill -f ./scripts/crl_server.py - } run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_failure + assert_cmd_verify_success - run python3 ./scripts/crl_server.py + update_crl_server run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success + assert_cmd_verify_failure } @test "notation verifier with type test" { diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index d8c872686..14a629a28 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -111,3 +111,10 @@ wait_for_process() { done return 1 } + +update_crl_server() { + URL_LEAF="http://localhost:10086/leaf/revoke" + curl -X POST "$URL_LEAF" -H "Content-Type: application/json" + URL_INTER=http://localhost:10086/intermediate/revoke + curl -X POST "$URL_INTER" -H "Content-Type: application/json" +} \ No newline at end of file From 708f438cf84881c679203d1c2a3b77350f289f1e Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 22:54:14 +0000 Subject: [PATCH 37/44] fix: another test 7 Signed-off-by: Juncheng Zhu --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eab605e55..ad46daf15 100644 --- a/Makefile +++ b/Makefile @@ -342,8 +342,7 @@ e2e-notation-crl-setup: ${GITHUB_WORKSPACE}/bin/oras cp --from-oci-layout .staging/notation/notation.tar:v0 ${TEST_REGISTRY}/notation:crl rm .staging/notation/notation.tar NOTATION_EXPERIMENTAL=1 .staging/notation/notation sign -u ${TEST_REGISTRY_USERNAME} -p ${TEST_REGISTRY_PASSWORD} --key "crl-test" ${TEST_REGISTRY}/notation@`${GITHUB_WORKSPACE}/bin/oras manifest fetch ${TEST_REGISTRY}/notation:crl --descriptor | jq .digest | xargs` - python3 ./scripts/crl_server.py - + python3 ./scripts/crl_server.py & echo "started crl server" e2e-cosign-setup: rm -rf .staging/cosign From 4d6d181372a53066b9f90493d74d0f53565cf8dd Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 23:22:31 +0000 Subject: [PATCH 38/44] test: crl e2e Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 2 ++ test/bats/helpers.bash | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 56fba09c8..7fd86eae0 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -40,6 +40,8 @@ load helpers update_crl_server + sleep 5 + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure } diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 14a629a28..69991edb6 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -115,6 +115,6 @@ wait_for_process() { update_crl_server() { URL_LEAF="http://localhost:10086/leaf/revoke" curl -X POST "$URL_LEAF" -H "Content-Type: application/json" - URL_INTER=http://localhost:10086/intermediate/revoke + URL_INTER=http://localhost:10086/intermediate/unrevoke curl -X POST "$URL_INTER" -H "Content-Type: application/json" } \ No newline at end of file From 6d118950fa00ac8d0e8b2afb6b77aced86a2d1ef Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 23:35:26 +0000 Subject: [PATCH 39/44] test: crl e2e 2 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 9 +++++---- test/bats/helpers.bash | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 7fd86eae0..1d692db31 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -35,15 +35,16 @@ load helpers } @test "notation verifier crl test" { - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success update_crl_server - sleep 5 - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure + + restore_crl_server + + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_success } @test "notation verifier with type test" { diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 69991edb6..a444a7d84 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -117,4 +117,12 @@ update_crl_server() { curl -X POST "$URL_LEAF" -H "Content-Type: application/json" URL_INTER=http://localhost:10086/intermediate/unrevoke curl -X POST "$URL_INTER" -H "Content-Type: application/json" +} + + +restore_crl_server() { + URL_LEAF="http://localhost:10086/leaf/unrevoke" + curl -X POST "$URL_LEAF" -H "Content-Type: application/json" + URL_INTER=http://localhost:10086/intermediate/unrevoke + curl -X POST "$URL_INTER" -H "Content-Type: application/json" } \ No newline at end of file From b1f8258c286498b1584173bcf9be8692dda0cfb9 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Mon, 16 Dec 2024 23:43:26 +0000 Subject: [PATCH 40/44] test: crl e2e 3 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 1d692db31..fb421a67e 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -40,11 +40,6 @@ load helpers run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure - - restore_crl_server - - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success } @test "notation verifier with type test" { From 57b6d1f66b23a99bdf6b5ccfd8e5bd23a94534a0 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 18 Dec 2024 22:16:19 +0000 Subject: [PATCH 41/44] test: address comments Signed-off-by: Juncheng Zhu --- test/bats/base-test.bats | 2 +- test/bats/cli-test.bats | 8 ++++++-- test/bats/helpers.bash | 10 +--------- ...=> config_v1beta1_verifier_notation_audit_crl.yaml} | 0 4 files changed, 8 insertions(+), 12 deletions(-) rename test/bats/tests/config/{config_v1beta1_verifier_notation_crl.yaml => config_v1beta1_verifier_notation_audit_crl.yaml} (100%) diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index a20f3cdb0..812c359bf 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -125,7 +125,7 @@ RATIFY_NAMESPACE=gatekeeper-system assert_success } -@test "notation test crl" { +@test "notation verification pass on CRL check with audit trust policy" { teardown() { echo "cleaning up" wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index fb421a67e..61d163e66 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -34,9 +34,13 @@ load helpers assert_cmd_verify_failure } -@test "notation verifier crl test" { +@test "notation verifier positive crl test" { + run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl + assert_cmd_verify_success +} - update_crl_server +@test "notation verifier negative crl test" { + revoke_crl run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl assert_cmd_verify_failure diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index a444a7d84..bc72fbd10 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -112,17 +112,9 @@ wait_for_process() { return 1 } -update_crl_server() { +revoke_crl() { URL_LEAF="http://localhost:10086/leaf/revoke" curl -X POST "$URL_LEAF" -H "Content-Type: application/json" URL_INTER=http://localhost:10086/intermediate/unrevoke curl -X POST "$URL_INTER" -H "Content-Type: application/json" } - - -restore_crl_server() { - URL_LEAF="http://localhost:10086/leaf/unrevoke" - curl -X POST "$URL_LEAF" -H "Content-Type: application/json" - URL_INTER=http://localhost:10086/intermediate/unrevoke - curl -X POST "$URL_INTER" -H "Content-Type: application/json" -} \ No newline at end of file diff --git a/test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml b/test/bats/tests/config/config_v1beta1_verifier_notation_audit_crl.yaml similarity index 100% rename from test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml rename to test/bats/tests/config/config_v1beta1_verifier_notation_audit_crl.yaml From 769e82d8bc32b35b115ce1aebb6b01dea793ae4b Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 18 Dec 2024 22:26:04 +0000 Subject: [PATCH 42/44] test: address comments 2 Signed-off-by: Juncheng Zhu --- test/bats/helpers.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index bc72fbd10..0a5ec51ee 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -117,4 +117,5 @@ revoke_crl() { curl -X POST "$URL_LEAF" -H "Content-Type: application/json" URL_INTER=http://localhost:10086/intermediate/unrevoke curl -X POST "$URL_INTER" -H "Content-Type: application/json" + sleep 10 } From 3e2f5033b2bb0a3bf71f7d3ecf5c8be3aeab3b8f Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Wed, 18 Dec 2024 22:57:18 +0000 Subject: [PATCH 43/44] test: address comments 3 Signed-off-by: Juncheng Zhu --- test/bats/cli-test.bats | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/bats/cli-test.bats b/test/bats/cli-test.bats index 61d163e66..85df8096b 100644 --- a/test/bats/cli-test.bats +++ b/test/bats/cli-test.bats @@ -34,11 +34,6 @@ load helpers assert_cmd_verify_failure } -@test "notation verifier positive crl test" { - run bin/ratify verify -c $RATIFY_DIR/config_notation_crl.json -s $TEST_REGISTRY/notation:crl - assert_cmd_verify_success -} - @test "notation verifier negative crl test" { revoke_crl From d987f11c5b192b4708ac524d9141af6c8c077b21 Mon Sep 17 00:00:00 2001 From: Juncheng Zhu Date: Thu, 19 Dec 2024 07:01:59 +0000 Subject: [PATCH 44/44] test: fix ci test Signed-off-by: Juncheng Zhu --- test/bats/base-test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/base-test.bats b/test/bats/base-test.bats index 812c359bf..28e708845 100644 --- a/test/bats/base-test.bats +++ b/test/bats/base-test.bats @@ -139,7 +139,7 @@ RATIFY_NAMESPACE=gatekeeper-system cat .staging/notation/crl-test/root.crt | sed 's/^/ /g' >> crlkmprovider.yaml run kubectl apply -f crlkmprovider.yaml --namespace ${RATIFY_NAMESPACE} assert_success - run kubectl replace -f ./test/bats/tests/config/config_v1beta1_verifier_notation_crl.yaml + run kubectl replace -f ./test/bats/tests/config/config_v1beta1_verifier_notation_audit_crl.yaml run kubectl run demo --namespace default --image=registry:5000/notation:crl assert_success