From e9054f3a4b387958eb4ba7498b07c953afcef469 Mon Sep 17 00:00:00 2001 From: Alex Vilensky Date: Wed, 31 Jul 2024 15:53:09 -0700 Subject: [PATCH 1/4] Initial proposal to create new counters/timeseries to account for tasknames and namespaces --- pkg/chains/constants.go | 48 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/pkg/chains/constants.go b/pkg/chains/constants.go index 870b441072..256e63e4c2 100644 --- a/pkg/chains/constants.go +++ b/pkg/chains/constants.go @@ -14,24 +14,32 @@ limitations under the License. package chains const ( - SignedMessagesCount = "sgcount" - SignsStoredCount = "stcount" - PayloadUploadeCount = "plcount" - MarkedAsSignedCount = "mrcount" - PipelineRunSignedName = "pipelinerun_sign_created_total" - PipelineRunSignedDesc = "Total number of signed messages for pipelineruns" - PipelineRunUploadedName = "pipelinerun_payload_uploaded_total" - PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns" - PipelineRunStoredName = "pipelinerun_payload_stored_total" - PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns" - PipelineRunMarkedName = "pipelinerun_marked_signed_total" - PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns" - TaskRunSignedName = "taskrun_sign_created_total" - TaskRunSignedDesc = "Total number of signed messages for taskruns" - TaskRunUploadedName = "taskrun_payload_uploaded_total" - TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns" - TaskRunStoredName = "taskrun_payload_stored_total" - TaskRunStoredDesc = "Total number of stored payloads for taskruns" - TaskRunMarkedName = "taskrun_marked_signed_total" - TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns" + SignedMessagesCount = "sgcount" + SignsStoredCount = "stcount" + PayloadUploadeCount = "plcount" + MarkedAsSignedCount = "mrcount" + PipelineRunSignedName = "pipelinerun_sign_created_total" + PipelineRunSignedDesc = "Total number of signed messages for pipelineruns" + PipelineRunUploadedName = "pipelinerun_payload_uploaded_total" + PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns" + PipelineRunStoredName = "pipelinerun_payload_stored_total" + PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns" + PipelineRunMarkedName = "pipelinerun_marked_signed_total" + PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns" + PipelineRunSignedMsg = "pipelinerun_signed_messages" + PipelineRunSignedMsgDesc = "Number of signed messages for pipelineruns" + PipelineRunUplPayload = "pipelinerun_payload_uploaded" + PipelineRunUplPayloadDesc = "Number of uploaded payloads for pipelineruns" + PipelineRunPayloadStored = "pipelinerun_payload_stored" + PipelineRunPayloadStoredDesc = "Number of stored payloads for pipelineruns" + PipelineRunMarkedSigned = "pipelinerun_marked_signed" + PipelineRunMarkedDSignedesc = "Number of objects marked as signed for pipelineruns" + TaskRunSignedName = "taskrun_sign_created_total" + TaskRunSignedDesc = "Total number of signed messages for taskruns" + TaskRunUploadedName = "taskrun_payload_uploaded_total" + TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns" + TaskRunStoredName = "taskrun_payload_stored_total" + TaskRunStoredDesc = "Total number of stored payloads for taskruns" + TaskRunMarkedName = "taskrun_marked_signed_total" + TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns" ) From cd383dc9d879848469d72beae0c2e146911e53b5 Mon Sep 17 00:00:00 2001 From: Alex Vilensky Date: Thu, 1 Aug 2024 13:29:44 -0700 Subject: [PATCH 2/4] Renamed namespace aware timeseries/counters --- pkg/chains/constants.go | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/pkg/chains/constants.go b/pkg/chains/constants.go index 256e63e4c2..695a47ee19 100644 --- a/pkg/chains/constants.go +++ b/pkg/chains/constants.go @@ -14,32 +14,32 @@ limitations under the License. package chains const ( - SignedMessagesCount = "sgcount" - SignsStoredCount = "stcount" - PayloadUploadeCount = "plcount" - MarkedAsSignedCount = "mrcount" - PipelineRunSignedName = "pipelinerun_sign_created_total" - PipelineRunSignedDesc = "Total number of signed messages for pipelineruns" - PipelineRunUploadedName = "pipelinerun_payload_uploaded_total" - PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns" - PipelineRunStoredName = "pipelinerun_payload_stored_total" - PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns" - PipelineRunMarkedName = "pipelinerun_marked_signed_total" - PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns" - PipelineRunSignedMsg = "pipelinerun_signed_messages" - PipelineRunSignedMsgDesc = "Number of signed messages for pipelineruns" - PipelineRunUplPayload = "pipelinerun_payload_uploaded" - PipelineRunUplPayloadDesc = "Number of uploaded payloads for pipelineruns" - PipelineRunPayloadStored = "pipelinerun_payload_stored" - PipelineRunPayloadStoredDesc = "Number of stored payloads for pipelineruns" - PipelineRunMarkedSigned = "pipelinerun_marked_signed" - PipelineRunMarkedDSignedesc = "Number of objects marked as signed for pipelineruns" - TaskRunSignedName = "taskrun_sign_created_total" - TaskRunSignedDesc = "Total number of signed messages for taskruns" - TaskRunUploadedName = "taskrun_payload_uploaded_total" - TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns" - TaskRunStoredName = "taskrun_payload_stored_total" - TaskRunStoredDesc = "Total number of stored payloads for taskruns" - TaskRunMarkedName = "taskrun_marked_signed_total" - TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns" + SignedMessagesCount = "sgcount" + SignsStoredCount = "stcount" + PayloadUploadeCount = "plcount" + MarkedAsSignedCount = "mrcount" + PipelineRunSignedName = "pipelinerun_sign_created_total" + PipelineRunSignedDesc = "Total number of signed messages for pipelineruns" + PipelineRunUploadedName = "pipelinerun_payload_uploaded_total" + PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns" + PipelineRunStoredName = "pipelinerun_payload_stored_total" + PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns" + PipelineRunMarkedName = "pipelinerun_marked_signed_total" + PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns" + PipelineRunSignedMsgPerNamespace = "pipelinerun_signed_messages" + PipelineRunSignedMsgDescPerNamespace = "Namespace aware number of signed messages for pipelineruns" + PipelineRunUplPayloadPerNamespace = "pipelinerun_payload_uploaded" + PipelineRunUplPayloadDescPerNamespace = "Namespace aware number of uploaded payloads for pipelineruns" + PipelineRunPayloadStoredPerNamespace = "pipelinerun_payload_stored" + PipelineRunPayloadStoredDescPerNamespace = "Namespace aware number of stored payloads for pipelineruns" + PipelineRunMarkedSignedPerNamespace = "pipelinerun_marked_signed" + PipelineRunMarkedDSigneDescPerNamespace = "Namespace aware number of objects marked as signed for pipelineruns" + TaskRunSignedName = "taskrun_sign_created_total" + TaskRunSignedDesc = "Total number of signed messages for taskruns" + TaskRunUploadedName = "taskrun_payload_uploaded_total" + TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns" + TaskRunStoredName = "taskrun_payload_stored_total" + TaskRunStoredDesc = "Total number of stored payloads for taskruns" + TaskRunMarkedName = "taskrun_marked_signed_total" + TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns" ) From 9060c18f874bcc56999f7b21b8c49a796b09b488 Mon Sep 17 00:00:00 2001 From: Alex Vilensky Date: Tue, 20 Aug 2024 16:02:43 -0700 Subject: [PATCH 3/4] Added additional namespace aware metrics --- pkg/chains/constants.go | 4 ++ pkg/pipelinerunmetrics/metrics.go | 73 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/pkg/chains/constants.go b/pkg/chains/constants.go index 695a47ee19..88624ae567 100644 --- a/pkg/chains/constants.go +++ b/pkg/chains/constants.go @@ -18,6 +18,10 @@ const ( SignsStoredCount = "stcount" PayloadUploadeCount = "plcount" MarkedAsSignedCount = "mrcount" + SignedMessagesCountPerNamespace = "sgcountns" + SignsStoredCountPerNamespace = "stcountns" + PayloadUploadeCountPerNamespace = "plcountns" + MarkedAsSignedCountPerNamespace = "mrcountns" PipelineRunSignedName = "pipelinerun_sign_created_total" PipelineRunSignedDesc = "Total number of signed messages for pipelineruns" PipelineRunUploadedName = "pipelinerun_payload_uploaded_total" diff --git a/pkg/pipelinerunmetrics/metrics.go b/pkg/pipelinerunmetrics/metrics.go index cdc8bfa36c..4005c437e0 100644 --- a/pkg/pipelinerunmetrics/metrics.go +++ b/pkg/pipelinerunmetrics/metrics.go @@ -23,8 +23,10 @@ import ( "github.com/tektoncd/chains/pkg/chains" "go.opencensus.io/stats" "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "knative.dev/pkg/logging" "knative.dev/pkg/metrics" + "knative.dev/pkg/metrics/metricskey" ) var ( @@ -51,6 +53,35 @@ var ( stats.UnitDimensionless) mrCountView *view.View + + sgCountNS = stats.Float64(chains.PipelineRunSignedMsgPerNamespace, + chains.PipelineRunSignedMsgDescPerNamespace, + stats.UnitDimensionless) + + sgCountViewNS *view.View + + plCountNS = stats.Float64(chains.PipelineRunUplPayloadPerNamespace, + chains.PipelineRunUplPayloadDescPerNamespace, + stats.UnitDimensionless) + + plCountViewNS *view.View + + stCountNS = stats.Float64(chains.PipelineRunPayloadStoredPerNamespace, + chains.PipelineRunPayloadStoredDescPerNamespace, + stats.UnitDimensionless) + + stCountViewNS *view.View + + mrCountNS = stats.Float64(chains.PipelineRunMarkedSignedPerNamespace, + chains.PipelineRunMarkedDSigneDescPerNamespace, + stats.UnitDimensionless) + + mrCountViewNS *view.View + + // NamespaceTagKey marks metrics with a namespace. + NamespaceTagKey = tag.MustNewKey(metricskey.LabelNamespaceName) + + successTagKey = tag.MustNewKey("success") ) // Recorder holds keys for Tekton metrics @@ -71,6 +102,7 @@ var ( func NewRecorder(ctx context.Context) (*Recorder, error) { var errRegistering error logger := logging.FromContext(ctx) + once.Do(func() { r = &Recorder{ initialized: true, @@ -110,11 +142,44 @@ func viewRegister() error { Measure: mrCount, Aggregation: view.Count(), } + + sgCountViewNS = &view.View{ + Description: sgCountNS.Description(), + Measure: sgCountNS, + Aggregation: view.Count(), + TagKeys: []tag.Key{NamespaceTagKey, successTagKey}, + } + + plCountViewNS = &view.View{ + Description: plCountNS.Description(), + Measure: plCountNS, + Aggregation: view.Count(), + TagKeys: []tag.Key{NamespaceTagKey, successTagKey}, + } + + stCountViewNS = &view.View{ + Description: stCountNS.Description(), + Measure: stCountNS, + Aggregation: view.Count(), + TagKeys: []tag.Key{NamespaceTagKey, successTagKey}, + } + + mrCountViewNS = &view.View{ + Description: mrCountNS.Description(), + Measure: mrCountNS, + Aggregation: view.Count(), + TagKeys: []tag.Key{NamespaceTagKey, successTagKey}, + } + return view.Register( sgCountView, plCountView, stCountView, mrCountView, + sgCountViewNS, + plCountViewNS, + stCountViewNS, + mrCountViewNS, ) } @@ -133,6 +198,14 @@ func (r *Recorder) RecordCountMetrics(ctx context.Context, metricType string) { r.countMetrics(ctx, stCount) case chains.MarkedAsSignedCount: r.countMetrics(ctx, mrCount) + case chains.SignedMessagesCountPerNamespace: + r.countMetrics(ctx, sgCountNS) + case chains.PayloadUploadeCountPerNamespace: + r.countMetrics(ctx, plCountNS) + case chains.SignsStoredCountPerNamespace: + r.countMetrics(ctx, stCountNS) + case chains.MarkedAsSignedCountPerNamespace: + r.countMetrics(ctx, mrCountNS) default: logger.Errorf("Ignoring the metrics recording as valid Metric type matching %v was not found", mt) } From ea8981622a86cfe2cc43b6fa4c58387027289212 Mon Sep 17 00:00:00 2001 From: Alex Vilensky Date: Mon, 16 Sep 2024 14:48:36 -0700 Subject: [PATCH 4/4] Honor storage.oci.repository.insecure flag for various code paths --- pkg/artifacts/signable.go | 9 ++++++++- pkg/chains/storage/oci/legacy.go | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pkg/artifacts/signable.go b/pkg/artifacts/signable.go index 2f50d20094..f258c4087f 100644 --- a/pkg/artifacts/signable.go +++ b/pkg/artifacts/signable.go @@ -211,8 +211,15 @@ func ExtractOCIImagesFromResults(ctx context.Context, results []objects.Result) digestSuffix: OCIImageDigestResultName, isValid: hasImageRequirements, } + + cfg := config.FromContext(ctx) + var opts []name.Option + if cfg.Storage.OCI.Insecure { + opts = append(opts, name.Insecure) + } + for _, s := range extractor.extract(ctx, results) { - dgst, err := name.NewDigest(fmt.Sprintf("%s@%s", s.URI, s.Digest)) + dgst, err := name.NewDigest(fmt.Sprintf("%s@%s", s.URI, s.Digest), opts...) if err != nil { logger.Errorf("error getting digest: %v", err) continue diff --git a/pkg/chains/storage/oci/legacy.go b/pkg/chains/storage/oci/legacy.go index fdf355067f..c54b7e60d3 100644 --- a/pkg/chains/storage/oci/legacy.go +++ b/pkg/chains/storage/oci/legacy.go @@ -119,8 +119,13 @@ func (b *Backend) uploadSignature(ctx context.Context, format simple.SimpleConta imageName := format.ImageName() logger.Infof("Uploading %s signature", imageName) + cfg := config.FromContext(ctx) + var opts []name.Option + if cfg.Storage.OCI.Insecure { + opts = append(opts, name.Insecure) + } - ref, err := name.NewDigest(imageName) + ref, err := name.NewDigest(imageName, opts...) if err != nil { return errors.Wrap(err, "getting digest") } @@ -154,13 +159,18 @@ func (b *Backend) uploadSignature(ctx context.Context, format simple.SimpleConta func (b *Backend) uploadAttestation(ctx context.Context, attestation *intoto.Statement, signature string, storageOpts config.StorageOpts, remoteOpts ...remote.Option) error { logger := logging.FromContext(ctx) + cfg := config.FromContext(ctx) + var opts []name.Option + if cfg.Storage.OCI.Insecure { + opts = append(opts, name.Insecure) + } + // upload an attestation for each subject logger.Info("Starting to upload attestations to OCI ...") for _, subj := range attestation.Subject { imageName := fmt.Sprintf("%s@sha256:%s", subj.Name, subj.Digest["sha256"]) logger.Infof("Starting attestation upload to OCI for %s...", imageName) - - ref, err := name.NewDigest(imageName) + ref, err := name.NewDigest(imageName, opts...) if err != nil { return errors.Wrapf(err, "getting digest for subj %s", imageName) }