From f9c5702295b36a9bf284c1b71af0b2d88e1d81bc Mon Sep 17 00:00:00 2001 From: Zach Steindler Date: Tue, 6 Aug 2024 16:16:45 -0400 Subject: [PATCH] Conformance testing for cosign (#3806) * Adding conformance helper and Action Also add e2e test and some helpful error messages about what flags go together Signed-off-by: Zach Steindler * Allow conformance driver to call cosign with user-supplied args Signed-off-by: Zach Steindler * fix e2e test Signed-off-by: Zach Steindler * Detail TODO comments; remove unneeded trusted root in e2e tests Signed-off-by: Zach Steindler --------- Signed-off-by: Zach Steindler --- .github/workflows/conformance.yml | 42 +++ Makefile | 5 +- cmd/conformance/main.go | 261 ++++++++++++++++++ cmd/cosign/cli/verify/verify_blob.go | 5 + .../cli/verify/verify_blob_attestation.go | 5 + cmd/cosign/cli/verify/verify_blob_test.go | 4 + test/e2e_test.go | 44 +++ 7 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 cmd/conformance/main.go diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000000..70f7e463444 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,42 @@ +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Conformance Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: '1.22' + check-latest: true + + - run: make cosign conformance + + - uses: sigstore/sigstore-conformance@ee4de0e602873beed74cf9e49d5332529fe69bf6 # v0.0.11 + with: + entrypoint: ${{ github.workspace }}/conformance diff --git a/Makefile b/Makefile index 07d138360da..c67b4032fb9 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ export KO_DOCKER_REPO=$(KO_PREFIX) GHCR_PREFIX ?= ghcr.io/sigstore/cosign LATEST_TAG ?= -.PHONY: all lint test clean cosign cross +.PHONY: all lint test clean cosign conformance cross all: cosign log-%: @@ -106,6 +106,9 @@ lint: golangci-lint ## Run golangci-lint linter test: $(GOEXE) test $(shell $(GOEXE) list ./... | grep -v third_party/) +conformance: + $(GOEXE) build -trimpath -ldflags "$(LDFLAGS)" -o $@ ./cmd/conformance + clean: rm -rf cosign rm -rf dist/ diff --git a/cmd/conformance/main.go b/cmd/conformance/main.go new file mode 100644 index 00000000000..420e16bd631 --- /dev/null +++ b/cmd/conformance/main.go @@ -0,0 +1,261 @@ +// Copyright 2024 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/pem" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + protobundle "github.com/sigstore/protobuf-specs/gen/pb-go/bundle/v1" + protocommon "github.com/sigstore/protobuf-specs/gen/pb-go/common/v1" + "github.com/sigstore/sigstore-go/pkg/bundle" + "google.golang.org/protobuf/encoding/protojson" +) + +var bundlePath *string +var certPath *string +var certOIDC *string +var certSAN *string +var identityToken *string +var signaturePath *string +var trustedRootPath *string + +func usage() { + fmt.Println("Usage:") + fmt.Printf("\t%s sign --identity-token TOKEN --signature FILE --certificate FILE FILE\n", os.Args[0]) + fmt.Printf("\t%s sign-bundle --identity-token TOKEN --bundle FILE FILE\n", os.Args[0]) + fmt.Printf("\t%s verify --signature FILE --certificate FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE\n", os.Args[0]) + fmt.Printf("\t%s verify-bundle --bundle FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE\n", os.Args[0]) +} + +func parseArgs() { + for i := 2; i < len(os.Args); { + switch os.Args[i] { + // TODO: support staging (see https://github.com/sigstore/cosign/issues/2434) + // + // Today cosign signing does not yet use sigstore-go, and so we would + // need to make some clever invocation of `cosign initialize` to + // support staging. Instead it might make sense to wait for cosign + // signing to use sigstore-go. + case "--bundle": + bundlePath = &os.Args[i+1] + i += 2 + case "--certificate": + certPath = &os.Args[i+1] + i += 2 + case "--certificate-oidc-issuer": + certOIDC = &os.Args[i+1] + i += 2 + case "--certificate-identity": + certSAN = &os.Args[i+1] + i += 2 + case "--identity-token": + identityToken = &os.Args[i+1] + i += 2 + case "--signature": + signaturePath = &os.Args[i+1] + i += 2 + case "--trusted-root": + trustedRootPath = &os.Args[i+1] + i += 2 + default: + i++ + } + } +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(1) + } + + parseArgs() + + args := []string{} + + switch os.Args[1] { + case "sign": + args = append(args, "sign-blob") + if signaturePath != nil { + args = append(args, "--output-signature", *signaturePath) + } + if certPath != nil { + args = append(args, "--output-certificate", *certPath) + } + args = append(args, "-y") + + case "sign-bundle": + args = append(args, "sign-blob") + args = append(args, "-y") + + case "verify": + args = append(args, "verify-blob") + + // TODO: for now, we handle `verify` by constructing a bundle + // (see https://github.com/sigstore/cosign/issues/3700) + // + // Today cosign only supports `--trusted-root` with the new bundle + // format. When cosign supports `--trusted-root` with detached signed + // material, we can supply this content with `--certificate` + // and `--signature` instead. + fileBytes, err := os.ReadFile(os.Args[len(os.Args)-1]) + if err != nil { + log.Fatal(err) + } + + fileDigest := sha256.Sum256(fileBytes) + + pb := protobundle.Bundle{ + MediaType: "application/vnd.dev.sigstore.bundle+json;version=0.1", + } + + if signaturePath != nil { + sig, err := os.ReadFile(*signaturePath) + if err != nil { + log.Fatal(err) + } + + sigBytes, err := base64.StdEncoding.DecodeString(string(sig)) + if err != nil { + log.Fatal(err) + } + + pb.Content = &protobundle.Bundle_MessageSignature{ + MessageSignature: &protocommon.MessageSignature{ + MessageDigest: &protocommon.HashOutput{ + Algorithm: protocommon.HashAlgorithm_SHA2_256, + Digest: fileDigest[:], + }, + Signature: sigBytes, + }, + } + } + if certPath != nil { + cert, err := os.ReadFile(*certPath) + if err != nil { + log.Fatal(err) + } + + pemCert, _ := pem.Decode(cert) + if pemCert == nil { + log.Fatalf("unable to load cerficate from %s", *certPath) + } + + signingCert := protocommon.X509Certificate{ + RawBytes: pemCert.Bytes, + } + + pb.VerificationMaterial = &protobundle.VerificationMaterial{ + Content: &protobundle.VerificationMaterial_X509CertificateChain{ + X509CertificateChain: &protocommon.X509CertificateChain{ + Certificates: []*protocommon.X509Certificate{&signingCert}, + }, + }, + } + } + + bundleFile, err := os.CreateTemp(os.TempDir(), "bundle.sigstore.json") + if err != nil { + log.Fatal(err) + } + bundleFileName := bundleFile.Name() + pbBytes, err := protojson.Marshal(&pb) + if err != nil { + log.Fatal(err) + } + if err := os.WriteFile(bundleFileName, pbBytes, 0600); err != nil { + log.Fatal(err) + } + bundlePath = &bundleFileName + args = append(args, "--insecure-ignore-tlog") + + case "verify-bundle": + args = append(args, "verify-blob") + + // How do we know if we should expect signed timestamps or not? + // Let's crack open the bundle + if bundlePath != nil { + b, err := bundle.LoadJSONFromPath(*bundlePath) + if err != nil { + log.Fatal(err) + } + ts, err := b.Timestamps() + if err != nil { + log.Fatal(err) + } + if len(ts) > 0 { + args = append(args, "--use-signed-timestamps") + } + } + + default: + log.Fatalf("Unsupported command %s", os.Args[1]) + } + + if bundlePath != nil { + args = append(args, "--bundle", *bundlePath) + args = append(args, "--new-bundle-format") + } + if identityToken != nil { + args = append(args, "--identity-token", *identityToken) + } + if certSAN != nil { + args = append(args, "--certificate-identity", *certSAN) + } + if certOIDC != nil { + args = append(args, "--certificate-oidc-issuer", *certOIDC) + } + if trustedRootPath != nil { + args = append(args, "--trusted-root", *trustedRootPath) + } + args = append(args, os.Args[len(os.Args)-1]) + + dir := filepath.Dir(os.Args[0]) + cmd := exec.Command(filepath.Join(dir, "cosign"), args...) // #nosec G204 + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + err := cmd.Run() + + fmt.Println(out.String()) + + if err != nil { + log.Fatal(err) + } + + if os.Args[1] == "sign" && certPath != nil { + // We want the signature to be base64 encoded, but not the certificate + // So base64 decode the certificate + cert, err := os.ReadFile(*certPath) + if err != nil { + log.Fatal(err) + } + certB64Decode, err := base64.StdEncoding.DecodeString(string(cert)) + if err != nil { + log.Fatal(err) + } + if err := os.WriteFile(*certPath, certB64Decode, 0600); err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/cosign/cli/verify/verify_blob.go b/cmd/cosign/cli/verify/verify_blob.go index b1574bf2ae2..79475c90d80 100644 --- a/cmd/cosign/cli/verify/verify_blob.go +++ b/cmd/cosign/cli/verify/verify_blob.go @@ -93,11 +93,16 @@ func (c *VerifyBlobCmd) Exec(ctx context.Context, blobRef string) error { } if c.KeyOpts.NewBundleFormat { + if options.NOf(c.RFC3161TimestampPath, c.TSACertChainPath, c.RekorURL, c.CertChain, c.CARoots, c.CAIntermediates, c.CertRef, c.SigRef, c.SCTRef) > 1 { + return fmt.Errorf("when using --new-bundle-format, please supply signed content with --bundle and verification content with --trusted-root") + } err := verifyNewBundle(ctx, c.BundlePath, c.TrustedRootPath, c.KeyRef, c.Slot, c.CertVerifyOptions.CertOidcIssuer, c.CertVerifyOptions.CertOidcIssuerRegexp, c.CertVerifyOptions.CertIdentity, c.CertVerifyOptions.CertIdentityRegexp, c.CertGithubWorkflowTrigger, c.CertGithubWorkflowSHA, c.CertGithubWorkflowName, c.CertGithubWorkflowRepository, c.CertGithubWorkflowRef, blobRef, c.Sk, c.IgnoreTlog, c.UseSignedTimestamps, c.IgnoreSCT) if err == nil { ui.Infof(ctx, "Verified OK") } return err + } else if c.TrustedRootPath != "" { + return fmt.Errorf("--trusted-root only supported with --new-bundle-format") } var cert *x509.Certificate diff --git a/cmd/cosign/cli/verify/verify_blob_attestation.go b/cmd/cosign/cli/verify/verify_blob_attestation.go index b8bec7b1756..3f2c33cc63b 100644 --- a/cmd/cosign/cli/verify/verify_blob_attestation.go +++ b/cmd/cosign/cli/verify/verify_blob_attestation.go @@ -93,11 +93,16 @@ func (c *VerifyBlobAttestationCommand) Exec(ctx context.Context, artifactPath st } if c.KeyOpts.NewBundleFormat { + if options.NOf(c.RFC3161TimestampPath, c.TSACertChainPath, c.RekorURL, c.CertChain, c.CARoots, c.CAIntermediates, c.CertRef, c.SCTRef) > 1 { + return fmt.Errorf("when using --new-bundle-format, please supply signed content with --bundle and verification content with --trusted-root") + } err = verifyNewBundle(ctx, c.BundlePath, c.TrustedRootPath, c.KeyRef, c.Slot, c.CertVerifyOptions.CertOidcIssuer, c.CertVerifyOptions.CertOidcIssuerRegexp, c.CertVerifyOptions.CertIdentity, c.CertVerifyOptions.CertIdentityRegexp, c.CertGithubWorkflowTrigger, c.CertGithubWorkflowSHA, c.CertGithubWorkflowName, c.CertGithubWorkflowRepository, c.CertGithubWorkflowRef, artifactPath, c.Sk, c.IgnoreTlog, c.UseSignedTimestamps, c.IgnoreSCT) if err == nil { fmt.Fprintln(os.Stderr, "Verified OK") } return err + } else if c.TrustedRootPath != "" { + return fmt.Errorf("--trusted-root only supported with --new-bundle-format") } var identities []cosign.Identity diff --git a/cmd/cosign/cli/verify/verify_blob_test.go b/cmd/cosign/cli/verify/verify_blob_test.go index 8dd77c81af9..6b1b127052c 100644 --- a/cmd/cosign/cli/verify/verify_blob_test.go +++ b/cmd/cosign/cli/verify/verify_blob_test.go @@ -619,6 +619,10 @@ func TestVerifyBlob(t *testing.T) { } if tt.newBundle { cmd.TrustedRootPath = writeTrustedRootFile(t, td, "{\"mediaType\":\"application/vnd.dev.sigstore.trustedroot+json;version=0.1\"}") + cmd.KeyOpts.RekorURL = "" + cmd.KeyOpts.RFC3161TimestampPath = "" + cmd.KeyOpts.TSACertChainPath = "" + cmd.CertChain = "" } err := cmd.Exec(context.Background(), blobPath) diff --git a/test/e2e_test.go b/test/e2e_test.go index 97444ad002a..f0e36328b6f 100644 --- a/test/e2e_test.go +++ b/test/e2e_test.go @@ -1619,6 +1619,50 @@ func TestSignBlobBundle(t *testing.T) { must(verifyBlobCmd.Exec(ctx, bp), t) } +func TestSignBlobNewBundle(t *testing.T) { + td1 := t.TempDir() + + blob := "someblob" + blobPath := filepath.Join(td1, blob) + if err := os.WriteFile(blobPath, []byte(blob), 0644); err != nil { + t.Fatal(err) + } + + bundlePath := filepath.Join(td1, "bundle.sigstore.json") + + ctx := context.Background() + _, privKeyPath, pubKeyPath := keypair(t, td1) + + ko1 := options.KeyOpts{ + KeyRef: pubKeyPath, + BundlePath: bundlePath, + NewBundleFormat: true, + } + + verifyBlobCmd := cliverify.VerifyBlobCmd{ + KeyOpts: ko1, + IgnoreTlog: true, + } + + // Verify should fail before bundle is written + mustErr(verifyBlobCmd.Exec(ctx, blobPath), t) + + // Produce signed bundle + ko := options.KeyOpts{ + KeyRef: privKeyPath, + PassFunc: passFunc, + BundlePath: bundlePath, + NewBundleFormat: true, + } + + if _, err := sign.SignBlobCmd(ro, ko, blobPath, true, "", "", false); err != nil { + t.Fatal(err) + } + + // Verify should succeed now that bundle is written + must(verifyBlobCmd.Exec(ctx, blobPath), t) +} + func TestSignBlobRFC3161TimestampBundle(t *testing.T) { td := t.TempDir() err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td)