From 54fd6d7dd6dc4990a001244473c1fd18e72e6c3b Mon Sep 17 00:00:00 2001 From: Fanny Jiang Date: Fri, 15 Nov 2024 10:44:12 -0500 Subject: [PATCH 1/3] wip --- .../examples/kind_local_operator_test.go | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 test/new-e2e/examples/kind_local_operator_test.go diff --git a/test/new-e2e/examples/kind_local_operator_test.go b/test/new-e2e/examples/kind_local_operator_test.go new file mode 100644 index 0000000000000..31a7ed42873df --- /dev/null +++ b/test/new-e2e/examples/kind_local_operator_test.go @@ -0,0 +1,122 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package examples + +import ( + "context" + "github.com/DataDog/test-infra-definitions/components/datadog/agent" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" + "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" + compkube "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/DataDog/test-infra-definitions/resources/local" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" +) + +type localOperatorKindSuite struct { + e2e.BaseSuite[environments.Kubernetes] +} + +func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes] { + return func(ctx *pulumi.Context, env *environments.Kubernetes) error { + kindEnv, err := local.NewEnvironment(ctx) + if err != nil { + return err + } + + kindCluster, err := compkube.NewLocalKindCluster(&kindEnv, kindEnv.CommonNamer().ResourceName("kind"), "kind-operator", kindEnv.KubernetesVersion()) + if err != nil { + return err + } + + if &env.KubernetesCluster != nil && &env.KubernetesCluster.ClusterOutput != nil { + err = kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput) + if err != nil { + return err + } + } + + // Building Kubernetes provider + kindKubeProvider, err := kubernetes.NewProvider(ctx, kindEnv.CommonNamer().ResourceName("k8s-provider"), &kubernetes.ProviderArgs{ + Kubeconfig: kindCluster.KubeConfig, + EnableServerSideApply: pulumi.BoolPtr(true), + DeleteUnreachable: pulumi.BoolPtr(true), + }) + if err != nil { + return err + } + + namespace := "datadog" // TODO: fix namespace for ddaw/operator component + + fakeIntake, err := fakeintakeComp.NewLocalDockerFakeintake(&kindEnv, "fakeintake") + if err != nil { + return err + } + err = fakeIntake.Export(ctx, &env.FakeIntake.FakeintakeOutput) + if err != nil { + return err + } + + // Setup operator options + operatorOpts := make([]operatorparams.Option, 0) + operatorOpts = append( + operatorOpts, + operatorparams.WithNamespace(namespace), + ) + // Setup DDA options + ddaOptions := make([]agentwithoperatorparams.Option, 0) + ddaOptions = append( + ddaOptions, + agentwithoperatorparams.WithNamespace(namespace), + agentwithoperatorparams.WithTLSKubeletVerify(false), + ) + + if fakeIntake != nil { + ddaOptions = append( + ddaOptions, + agentwithoperatorparams.WithFakeIntake(fakeIntake), + ) + } + + operatorAgentComponent, err := agent.NewDDAWithOperator(&kindEnv, kindEnv.CommonNamer().ResourceName("dd-operator-agent"), kindKubeProvider, operatorOpts, ddaOptions...) + + if err != nil { + return err + } + + if err := operatorAgentComponent.Export(kindEnv.Ctx(), nil); err != nil { + return err + } + + return nil + } +} + +func TestLocalOpKindSuite(t *testing.T) { + e2e.Run(t, &localOperatorKindSuite{}, e2e.WithPulumiProvisioner(localKindOperatorProvisioner(), nil), + e2e.WithDevMode()) +} + +func (v *localOperatorKindSuite) TestClusterAgentInstalled() { + res, _ := v.Env().KubernetesCluster.Client().CoreV1().Pods("datadog").List(context.TODO(), v1.ListOptions{}) + containsClusterAgent := false + for _, pod := range res.Items { + if strings.Contains(pod.Name, "cluster-agent") { + containsClusterAgent = true + break + } + } + assert.True(v.T(), containsClusterAgent, "Cluster Agent not found") +} From febebfc13f46476199731959a2df208987e194fc Mon Sep 17 00:00:00 2001 From: Fanny Jiang Date: Mon, 18 Nov 2024 18:53:06 -0500 Subject: [PATCH 2/3] Add deployOperator option to aws kind provisioner, add examples --- .../examples/kind_local_operator_test.go | 46 +++++++-------- test/new-e2e/examples/kind_operator_test.go | 50 ++++++++++++++++ .../pkg/environments/aws/kubernetes/kind.go | 48 ++++++++++++++- .../pkg/environments/aws/kubernetes/params.go | 58 ++++++++++++++----- 4 files changed, 161 insertions(+), 41 deletions(-) create mode 100644 test/new-e2e/examples/kind_operator_test.go diff --git a/test/new-e2e/examples/kind_local_operator_test.go b/test/new-e2e/examples/kind_local_operator_test.go index 31a7ed42873df..dda9c1766a9e2 100644 --- a/test/new-e2e/examples/kind_local_operator_test.go +++ b/test/new-e2e/examples/kind_local_operator_test.go @@ -25,7 +25,7 @@ import ( "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" ) -type localOperatorKindSuite struct { +type localKindOperatorSuite struct { e2e.BaseSuite[environments.Kubernetes] } @@ -36,29 +36,26 @@ func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes return err } - kindCluster, err := compkube.NewLocalKindCluster(&kindEnv, kindEnv.CommonNamer().ResourceName("kind"), "kind-operator", kindEnv.KubernetesVersion()) + kindCluster, err := compkube.NewLocalKindCluster(&kindEnv, kindEnv.CommonNamer().ResourceName("kind-operator"), kindEnv.KubernetesVersion()) if err != nil { return err } - if &env.KubernetesCluster != nil && &env.KubernetesCluster.ClusterOutput != nil { - err = kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput) - if err != nil { - return err - } + err = kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput) + if err != nil { + return err } - // Building Kubernetes provider + // Build Kubernetes provider kindKubeProvider, err := kubernetes.NewProvider(ctx, kindEnv.CommonNamer().ResourceName("k8s-provider"), &kubernetes.ProviderArgs{ Kubeconfig: kindCluster.KubeConfig, EnableServerSideApply: pulumi.BoolPtr(true), - DeleteUnreachable: pulumi.BoolPtr(true), }) if err != nil { return err } - namespace := "datadog" // TODO: fix namespace for ddaw/operator component + namespace := "e2e-operator" fakeIntake, err := fakeintakeComp.NewLocalDockerFakeintake(&kindEnv, "fakeintake") if err != nil { @@ -75,28 +72,30 @@ func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes operatorOpts, operatorparams.WithNamespace(namespace), ) + + customDDA := ` +spec: + features: + apm: + enabled: true +` // Setup DDA options ddaOptions := make([]agentwithoperatorparams.Option, 0) ddaOptions = append( ddaOptions, agentwithoperatorparams.WithNamespace(namespace), agentwithoperatorparams.WithTLSKubeletVerify(false), + agentwithoperatorparams.WithDDAConfig(customDDA), ) - if fakeIntake != nil { - ddaOptions = append( - ddaOptions, - agentwithoperatorparams.WithFakeIntake(fakeIntake), - ) - } - + // Create Datadog Agent with Operator component operatorAgentComponent, err := agent.NewDDAWithOperator(&kindEnv, kindEnv.CommonNamer().ResourceName("dd-operator-agent"), kindKubeProvider, operatorOpts, ddaOptions...) if err != nil { return err } - if err := operatorAgentComponent.Export(kindEnv.Ctx(), nil); err != nil { + if err := operatorAgentComponent.Export(ctx, &env.Agent.KubernetesAgentOutput); err != nil { return err } @@ -104,13 +103,12 @@ func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes } } -func TestLocalOpKindSuite(t *testing.T) { - e2e.Run(t, &localOperatorKindSuite{}, e2e.WithPulumiProvisioner(localKindOperatorProvisioner(), nil), - e2e.WithDevMode()) +func TestOperatorKindSuite(t *testing.T) { + e2e.Run(t, &localKindOperatorSuite{}, e2e.WithPulumiProvisioner(localKindOperatorProvisioner(), nil)) } -func (v *localOperatorKindSuite) TestClusterAgentInstalled() { - res, _ := v.Env().KubernetesCluster.Client().CoreV1().Pods("datadog").List(context.TODO(), v1.ListOptions{}) +func (k *localKindOperatorSuite) TestClusterAgentInstalled() { + res, _ := k.Env().KubernetesCluster.Client().CoreV1().Pods("blahblah").List(context.TODO(), v1.ListOptions{}) containsClusterAgent := false for _, pod := range res.Items { if strings.Contains(pod.Name, "cluster-agent") { @@ -118,5 +116,5 @@ func (v *localOperatorKindSuite) TestClusterAgentInstalled() { break } } - assert.True(v.T(), containsClusterAgent, "Cluster Agent not found") + assert.True(k.T(), containsClusterAgent, "Cluster Agent not found") } diff --git a/test/new-e2e/examples/kind_operator_test.go b/test/new-e2e/examples/kind_operator_test.go new file mode 100644 index 0000000000000..95398ba062253 --- /dev/null +++ b/test/new-e2e/examples/kind_operator_test.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package examples + +import ( + "context" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + "testing" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/stretchr/testify/assert" +) + +type kindOperatorSuite struct { + e2e.BaseSuite[environments.Kubernetes] +} + +func TestKindOperatorSuite(t *testing.T) { + customDDA := ` +spec: + features: + apm: + enabled: true +` + + e2e.Run(t, &kindOperatorSuite{}, e2e.WithProvisioner(awskubernetes.KindProvisioner( + awskubernetes.WithOperator(), + awskubernetes.WithOperatorDDAOptions(agentwithoperatorparams.WithDDAConfig(customDDA)), + )), + ) +} + +func (v *kindOperatorSuite) TestClusterAgentInstalled() { + res, _ := v.Env().KubernetesCluster.Client().CoreV1().Pods("datadog").List(context.TODO(), v1.ListOptions{}) + containsClusterAgent := false + for _, pod := range res.Items { + if strings.Contains(pod.Name, "cluster-agent") { + containsClusterAgent = true + break + } + } + assert.True(v.T(), containsClusterAgent, "Cluster Agent not found") +} diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go b/test/new-e2e/pkg/environments/aws/kubernetes/kind.go index 53bc559b4db0e..19a0a82e46f9f 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go +++ b/test/new-e2e/pkg/environments/aws/kubernetes/kind.go @@ -9,6 +9,10 @@ package awskubernetes import ( "context" "fmt" + "github.com/DataDog/test-infra-definitions/components/datadog/agent" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + "github.com/DataDog/test-infra-definitions/components/datadog/operator" + "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" "github.com/DataDog/test-infra-definitions/common/utils" @@ -88,7 +92,7 @@ func KindRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *Prov return err } - kindCluster, err := kubeComp.NewKindCluster(&awsEnv, host, params.name, awsEnv.KubernetesVersion(), utils.PulumiDependsOn(installEcrCredsHelperCmd)) + kindCluster, err := kubeComp.NewKindCluster(&awsEnv, host, awsEnv.CommonNamer().ResourceName("kind"), awsEnv.KubernetesVersion(), utils.PulumiDependsOn(installEcrCredsHelperCmd)) if err != nil { return err } @@ -127,9 +131,10 @@ func KindRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *Prov env.FakeIntake = nil } + kindClusterName := ctx.Stack() + var dependsOnCrd []pulumi.Resource - if params.agentOptions != nil { - kindClusterName := ctx.Stack() + if params.agentOptions != nil && !params.deployOperator { helmValues := fmt.Sprintf(` datadog: kubelet: @@ -150,6 +155,43 @@ agents: return err } dependsOnCrd = append(dependsOnCrd, agent) + } + + if params.deployOperator { + operatorOpts := make([]operatorparams.Option, 0) + operatorOpts = append( + operatorOpts, + params.operatorOptions..., + ) + + if params.operatorDDAOptions != nil && params.deployOperator { + ddaOptions := make([]agentwithoperatorparams.Option, 0) + ddaOptions = append( + ddaOptions, + params.operatorDDAOptions..., + ) + + ddaWithOperatorComp, err := agent.NewDDAWithOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator-agent"), kubeProvider, operatorOpts, ddaOptions...) + + if err != nil { + return err + } + + if err := ddaWithOperatorComp.Export(ctx, &env.Agent.KubernetesAgentOutput); err != nil { + return err + } + dependsOnCrd = append(dependsOnCrd, ddaWithOperatorComp) + + } else { + operatorComp, err := operator.NewOperator(&awsEnv, kindClusterName, kubeProvider, operatorOpts...) + if err != nil { + return err + } + err = operatorComp.Export(ctx, nil) + if err != nil { + return err + } + } } else { env.Agent = nil } diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/params.go b/test/new-e2e/pkg/environments/aws/kubernetes/params.go index 9f7e9f1c394ff..8a3d67abbe483 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/params.go +++ b/test/new-e2e/pkg/environments/aws/kubernetes/params.go @@ -8,6 +8,8 @@ package awskubernetes import ( "fmt" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -25,13 +27,15 @@ import ( // ProvisionerParams contains all the parameters needed to create the environment type ProvisionerParams struct { - name string - vmOptions []ec2.VMOption - agentOptions []kubernetesagentparams.Option - fakeintakeOptions []fakeintake.Option - eksOptions []eks.Option - extraConfigParams runner.ConfigMap - workloadAppFuncs []WorkloadAppFunc + name string + vmOptions []ec2.VMOption + agentOptions []kubernetesagentparams.Option + fakeintakeOptions []fakeintake.Option + eksOptions []eks.Option + extraConfigParams runner.ConfigMap + workloadAppFuncs []WorkloadAppFunc + operatorOptions []operatorparams.Option + operatorDDAOptions []agentwithoperatorparams.Option eksLinuxNodeGroup bool eksLinuxARMNodeGroup bool @@ -40,23 +44,27 @@ type ProvisionerParams struct { awsEnv *aws.Environment deployDogstatsd bool deployTestWorkload bool + deployOperator bool } func newProvisionerParams() *ProvisionerParams { return &ProvisionerParams{ - name: defaultVMName, - vmOptions: []ec2.VMOption{}, - agentOptions: []kubernetesagentparams.Option{}, - fakeintakeOptions: []fakeintake.Option{}, - eksOptions: []eks.Option{}, - extraConfigParams: runner.ConfigMap{}, - workloadAppFuncs: []WorkloadAppFunc{}, + name: defaultVMName, + vmOptions: []ec2.VMOption{}, + agentOptions: []kubernetesagentparams.Option{}, + fakeintakeOptions: []fakeintake.Option{}, + eksOptions: []eks.Option{}, + extraConfigParams: runner.ConfigMap{}, + workloadAppFuncs: []WorkloadAppFunc{}, + operatorOptions: []operatorparams.Option{}, + operatorDDAOptions: []agentwithoperatorparams.Option{}, eksLinuxNodeGroup: false, eksLinuxARMNodeGroup: false, eksBottlerocketNodeGroup: false, eksWindowsNodeGroup: false, deployDogstatsd: false, + deployOperator: false, } } @@ -171,3 +179,25 @@ func WithAwsEnv(env *aws.Environment) ProvisionerOption { return nil } } + +// WithOperator Deploys the Datadog Operator +func WithOperator() ProvisionerOption { + return func(params *ProvisionerParams) error { + params.deployOperator = true + return nil + } +} + +func WithOperatorOptions(opts ...operatorparams.Option) ProvisionerOption { + return func(params *ProvisionerParams) error { + params.operatorOptions = opts + return nil + } +} + +func WithOperatorDDAOptions(opts ...agentwithoperatorparams.Option) ProvisionerOption { + return func(params *ProvisionerParams) error { + params.operatorDDAOptions = opts + return nil + } +} From 24c789a5f4f9ae5456ff7b3934f0867a0ecf5e27 Mon Sep 17 00:00:00 2001 From: Fanny Jiang Date: Mon, 18 Nov 2024 19:09:14 -0500 Subject: [PATCH 3/3] fix namespace --- test/new-e2e/examples/kind_local_operator_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/new-e2e/examples/kind_local_operator_test.go b/test/new-e2e/examples/kind_local_operator_test.go index dda9c1766a9e2..49f3f7fb738b5 100644 --- a/test/new-e2e/examples/kind_local_operator_test.go +++ b/test/new-e2e/examples/kind_local_operator_test.go @@ -25,6 +25,8 @@ import ( "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" ) +const namespace = "e2e-operator" + type localKindOperatorSuite struct { e2e.BaseSuite[environments.Kubernetes] } @@ -41,8 +43,7 @@ func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes return err } - err = kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput) - if err != nil { + if err := kindCluster.Export(ctx, &env.KubernetesCluster.ClusterOutput); err != nil { return err } @@ -55,14 +56,11 @@ func localKindOperatorProvisioner() e2e.PulumiEnvRunFunc[environments.Kubernetes return err } - namespace := "e2e-operator" - fakeIntake, err := fakeintakeComp.NewLocalDockerFakeintake(&kindEnv, "fakeintake") if err != nil { return err } - err = fakeIntake.Export(ctx, &env.FakeIntake.FakeintakeOutput) - if err != nil { + if err := fakeIntake.Export(ctx, &env.FakeIntake.FakeintakeOutput); err != nil { return err } @@ -108,7 +106,7 @@ func TestOperatorKindSuite(t *testing.T) { } func (k *localKindOperatorSuite) TestClusterAgentInstalled() { - res, _ := k.Env().KubernetesCluster.Client().CoreV1().Pods("blahblah").List(context.TODO(), v1.ListOptions{}) + res, _ := k.Env().KubernetesCluster.Client().CoreV1().Pods(namespace).List(context.TODO(), v1.ListOptions{}) containsClusterAgent := false for _, pod := range res.Items { if strings.Contains(pod.Name, "cluster-agent") {