Define and implement secret querier via func interface to let pipeline compiler secrets

agent/logger.go

@@ -36,10 +36,7 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w
 
 		uploads.Add(1)
 
-		var secrets []string
-		for _, secret := range workflow.Config.Secrets {
-			secrets = append(secrets, secret.Value)
-		}
+		secrets := workflow.Config.SecretValues
 
 		logger.Debug().Msg("log stream opened")
 

cli/exec/exec.go

@@ -146,13 +146,17 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
 	}
 
 	environ := metadata.Environ()
-	var secrets []compiler.Secret
-	for key, val := range metadata.Workflow.Matrix {
-		environ[key] = val
-		secrets = append(secrets, compiler.Secret{
-			Name:  key,
-			Value: val,
-		})
+
+	secretMap := metadata.Workflow.Matrix
+	secretQuerier := func(name string) (*compiler.Secret, error) {
+		secret, found := secretMap[strings.ToLower(name)]
+		if !found {
+			fmt.Errorf("secret %q not found", name)
+		}
+		return &compiler.Secret{
+			Name:  name,
+			Value: secret,
+		}, nil
 	}
 
 	pipelineEnv := make(map[string]string)
@@ -255,7 +259,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
 			c.String("netrc-machine"),
 		),
 		compiler.WithMetadata(*metadata),
-		compiler.WithSecret(secrets...),
+		compiler.WithSecret(secretQuerier),
 		compiler.WithEnviron(pipelineEnv),
 	).Compile(conf)
 	if err != nil {

pipeline/backend/types/config.go

@@ -16,10 +16,10 @@ package types
 
 // Config defines the runtime configuration of a workflow.
 type Config struct {
-	Stages   []*Stage   `json:"pipeline"` // workflow stages
-	Networks []*Network `json:"networks"` // network definitions
-	Volumes  []*Volume  `json:"volumes"`  // volume definitions
-	Secrets  []*Secret  `json:"secrets"`  // secret definitions
+	Stages       []*Stage   `json:"pipeline"`      // workflow stages (TODO(3009): use dag)
+	Networks     []*Network `json:"networks"`      // network definitions
+	Volumes      []*Volume  `json:"volumes"`       // volume definitions
+	SecretValues []string   `json:"secret_values"` // secret_values definitions
 }
 
 // CliCommand is the context key to pass cli context to backends if needed.

pipeline/frontend/yaml/compiler/compiler.go

@@ -81,6 +81,8 @@ func (s *Secret) Match(event string) bool {
 	return false
 }
 
+type SecretQuerier = func(name string) (*Secret, error)
+
 // Compiler compiles the yaml.
 type Compiler struct {
 	local                   bool
@@ -94,7 +96,7 @@ type Compiler struct {
 	workspacePath           string
 	metadata                metadata.Metadata
 	registries              []Registry
-	secrets                 map[string]Secret
+	secretQuerier           SecretQuerier
 	defaultClonePlugin      string
 	trustedClonePlugins     []string
 	securityTrustedPipeline bool
@@ -106,9 +108,9 @@ func New(opts ...Option) *Compiler {
 	compiler := &Compiler{
 		env:                 map[string]string{},
 		cloneEnv:            map[string]string{},
-		secrets:             map[string]Secret{},
 		defaultClonePlugin:  constant.DefaultClonePlugin,
 		trustedClonePlugins: constant.TrustedClonePlugins,
+		secretQuerier:       func(name string) (*Secret, error) { return nil, fmt.Errorf("not implemented") },
 	}
 	for _, opt := range opts {
 		opt(compiler)
@@ -139,14 +141,6 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		Name: fmt.Sprintf("%s_default", c.prefix),
 	})
 
-	// create secrets for mask
-	for _, sec := range c.secrets {
-		config.Secrets = append(config.Secrets, &backend_types.Secret{
-			Name:  sec.Name,
-			Value: sec.Value,
-		})
-	}
-
 	// overrides the default workspace paths when specified
 	// in the YAML file.
 	if len(conf.Workspace.Base) != 0 {
@@ -171,7 +165,7 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		for k, v := range c.cloneEnv {
 			container.Environment[k] = v
 		}
-		step, err := c.createProcess(container, backend_types.StepTypeClone)
+		step, err := c.createProcess(config, container, backend_types.StepTypeClone)
 		if err != nil {
 			return nil, err
 		}
@@ -190,7 +184,7 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 
 			stage := new(backend_types.Stage)
 
-			step, err := c.createProcess(container, backend_types.StepTypeClone)
+			step, err := c.createProcess(config, container, backend_types.StepTypeClone)
 			if err != nil {
 				return nil, err
 			}
@@ -219,7 +213,7 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 				return nil, err
 			}
 
-			step, err := c.createProcess(container, backend_types.StepTypeService)
+			step, err := c.createProcess(config, container, backend_types.StepTypeService)
 			if err != nil {
 				return nil, err
 			}
@@ -247,7 +241,7 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		if container.IsPlugin() {
 			stepType = backend_types.StepTypePlugin
 		}
-		step, err := c.createProcess(container, stepType)
+		step, err := c.createProcess(config, container, stepType)
 		if err != nil {
 			return nil, err
 		}

pipeline/frontend/yaml/compiler/compiler_test.go

@@ -15,6 +15,7 @@
 package compiler
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -71,6 +72,9 @@ func TestCompilerCompile(t *testing.T) {
 				ForgeURL: repoURL,
 				CloneURL: "https://github.com/octocat/hello-world.git",
 			},
+			Curr: metadata.Pipeline{
+				Event: "push",
+			},
 		}),
 		WithEnviron(map[string]string{
 			"VERBOSE": "true",
@@ -79,6 +83,22 @@ func TestCompilerCompile(t *testing.T) {
 		WithPrefix("test"),
 		// we use "/test" as custom workspace base to ensure the enforcement of the pluginWorkspaceBase is applied
 		WithWorkspaceFromURL("/test", repoURL),
+		WithSecret(func(name string) (*Secret, error) {
+			if name == "exist" {
+				return &Secret{
+					Name:   name,
+					Value:  "geheim",
+					Events: []string{"push", "tag"},
+				}, nil
+			} else if name == "filter" {
+				return &Secret{
+					Name:   name,
+					Value:  "geheim",
+					Events: []string{"tag"},
+				}, nil
+			}
+			return nil, fmt.Errorf("secret \"%s\" not found", name)
+		}),
 	)
 
 	defaultNetworks := []*backend_types.Network{{
@@ -278,6 +298,37 @@ func TestCompilerCompile(t *testing.T) {
 				}},
 			},
 		},
+		{
+			name: "workflow with existing secret",
+			fronConf: &yaml_types.Workflow{Steps: yaml_types.ContainerList{ContainerList: []*yaml_types.Container{{
+				Name:     "step",
+				Image:    "bash",
+				Commands: []string{"env"},
+				Environment: yaml_base_types.EnvironmentMap{
+					"SOME_SECRET": map[string]any{"from_secret": "exist"},
+				},
+			}}}},
+			backConf: &backend_types.Config{
+				Networks: defaultNetworks,
+				Volumes:  defaultVolumes,
+				Stages: []*backend_types.Stage{defaultCloneStage, {
+					Steps: []*backend_types.Step{{
+						Name:          "step",
+						Type:          backend_types.StepTypeCommands,
+						Image:         "bash",
+						Commands:      []string{"env"},
+						OnSuccess:     true,
+						Failure:       "fail",
+						Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+						WorkingDir:    "/test/src/github.com/octocat/hello-world",
+						WorkspaceBase: "/test",
+						Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"step"}}},
+						ExtraHosts:    []backend_types.HostAlias{},
+					}},
+				}},
+				SecretValues: []string{"geheim"},
+			},
+		},
 		{
 			name: "workflow with missing secret",
 			fronConf: &yaml_types.Workflow{Steps: yaml_types.ContainerList{ContainerList: []*yaml_types.Container{{
@@ -289,6 +340,17 @@ func TestCompilerCompile(t *testing.T) {
 			backConf:    nil,
 			expectedErr: "secret \"missing\" not found",
 		},
+		{
+			name: "workflow with filter secret",
+			fronConf: &yaml_types.Workflow{Steps: yaml_types.ContainerList{ContainerList: []*yaml_types.Container{{
+				Name:     "step",
+				Image:    "bash",
+				Commands: []string{"env"},
+				Secrets:  []string{"filter"},
+			}}}},
+			backConf:    nil,
+			expectedErr: "secret \"filter\" is not allowed to be used with pipeline event \"push\"",
+		},
 		{
 			name: "workflow with broken step dependency",
 			fronConf: &yaml_types.Workflow{Steps: yaml_types.ContainerList{ContainerList: []*yaml_types.Container{{
@@ -306,7 +368,7 @@ func TestCompilerCompile(t *testing.T) {
 			backConf, err := compiler.Compile(test.fronConf)
 			if test.expectedErr != "" {
 				assert.Error(t, err)
-				assert.Equal(t, err.Error(), test.expectedErr)
+				assert.Equal(t, test.expectedErr, err.Error())
 			} else {
 				// we ignore uuids in steps and only check if global env got set ...
 				for _, st := range backConf.Stages {

pipeline/frontend/yaml/compiler/convert.go

@@ -37,7 +37,26 @@ const (
 	DefaultWorkspaceBase = pluginWorkspaceBase
 )
 
-func (c *Compiler) createProcess(container *yaml_types.Container, stepType backend_types.StepType) (*backend_types.Step, error) {
+func (c *Compiler) getSecretValueFunc(workflowConfig *backend_types.Config, container *yaml_types.Container) func(name string) (string, error) {
+	querier := c.secretQuerier
+	return func(name string) (string, error) {
+		secret, err := querier(name)
+		if err != nil {
+			return "", err
+		}
+
+		if err := secret.Available(c.metadata.Curr.Event, container); err != nil {
+			return "", err
+		}
+
+		// add secret value to the values to mask from log streaming
+		workflowConfig.SecretValues = append(workflowConfig.SecretValues, secret.Value)
+
+		return secret.Value, nil
+	}
+}
+
+func (c *Compiler) createProcess(workflowConfig *backend_types.Config, container *yaml_types.Container, stepType backend_types.StepType) (*backend_types.Step, error) {
 	var (
 		uuid = ulid.Make()
 
@@ -98,21 +117,7 @@ func (c *Compiler) createProcess(container *yaml_types.Container, stepType backe
 
 	workingDir = c.stepWorkingDir(container)
 
-	getSecretValue := func(name string) (string, error) {
-		name = strings.ToLower(name)
-		secret, ok := c.secrets[name]
-		if !ok {
-			return "", fmt.Errorf("secret %q not found", name)
-		}
-
-		event := c.metadata.Curr.Event
-		err := secret.Available(event, container)
-		if err != nil {
-			return "", err
-		}
-
-		return secret.Value, nil
-	}
+	getSecretValue := c.getSecretValueFunc(workflowConfig, container)
 
 	if err := settings.ParamsToEnv(container.Settings, environment, "PLUGIN_", true, getSecretValue); err != nil {
 		return nil, err

pipeline/frontend/yaml/compiler/option.go

@@ -17,7 +17,6 @@ package compiler
 import (
 	"net/url"
 	"path"
-	"strings"
 
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
 )
@@ -58,11 +57,9 @@ func WithRegistry(registries ...Registry) Option {
 
 // WithSecret configures the compiler with external secrets
 // to be injected into the container at runtime.
-func WithSecret(secrets ...Secret) Option {
+func WithSecret(secretQuerier SecretQuerier) Option {
 	return func(compiler *Compiler) {
-		for _, secret := range secrets {
-			compiler.secrets[strings.ToLower(secret.Name)] = secret
-		}
+		compiler.secretQuerier = secretQuerier
 	}
 }
 

server/pipeline/stepbuilder/stepBuilder.go

@@ -248,20 +248,30 @@ func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matr
 	return environ
 }
 
+func convertModelSecret(sec *model.Secret) *compiler.Secret {
+	var events []string
+	for _, event := range sec.Events {
+		events = append(events, string(event))
+	}
+	return &compiler.Secret{
+		Name:           sec.Name,
+		Value:          sec.Value,
+		AllowedPlugins: sec.Images,
+		Events:         events,
+	}
+}
+
 func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) {
-	var secrets []compiler.Secret
+	secretsMap := make(map[string]*compiler.Secret)
 	for _, sec := range b.Secs {
-		var events []string
-		for _, event := range sec.Events {
-			events = append(events, string(event))
+		secretsMap[strings.ToLower(sec.Name)] = convertModelSecret(sec)
+	}
+	secretsQueryer := func(name string) (*compiler.Secret, error) {
+		secret, found := secretsMap[strings.ToLower(name)]
+		if !found {
+			fmt.Errorf("secret %q not found", name)
 		}
-
-		secrets = append(secrets, compiler.Secret{
-			Name:           sec.Name,
-			Value:          sec.Value,
-			AllowedPlugins: sec.Images,
-			Events:         events,
-		})
+		return secret, nil
 	}
 
 	var registries []compiler.Registry
@@ -292,7 +302,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi
 		compiler.WithDefaultClonePlugin(server.Config.Pipeline.DefaultClonePlugin),
 		compiler.WithTrustedClonePlugins(server.Config.Pipeline.TrustedClonePlugins),
 		compiler.WithRegistry(registries...),
-		compiler.WithSecret(secrets...),
+		compiler.WithSecret(secretsQueryer),
 		compiler.WithPrefix(
 			fmt.Sprintf(
 				"wp_%s_%d",