From ee82fe184fdf3aa6f79fb0b28a1ef54c742d05cc Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Thu, 26 Sep 2024 15:02:00 +0800 Subject: [PATCH] Add tracing --- go.mod | 13 ++--- go.sum | 24 +++++---- .../v1alpha1/kwok_configuration_types.go | 16 ++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 27 ++++++++++ .../kwok_configuration_types.go | 14 ++++++ .../zz_generated.conversion.go | 46 +++++++++++++++++ .../internalversion/zz_generated.deepcopy.go | 17 +++++++ pkg/kwok/cmd/root.go | 50 +++++++++++++++---- pkg/kwok/server/server.go | 6 +++ pkg/kwokctl/components/kwok_controller.go | 8 +++ pkg/kwokctl/runtime/binary/cluster.go | 10 ++-- pkg/kwokctl/runtime/compose/cluster.go | 6 +++ pkg/kwokctl/runtime/kind/cluster.go | 6 +++ .../binary/create_cluster_with_verbosity.txt | 2 +- 14 files changed, 216 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index af95f4413..a0dbc5b92 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,12 @@ require ( github.com/wzshiming/getch v0.0.0-20201023133301-8e758c21cf27 github.com/wzshiming/httpseek v0.1.0 go.etcd.io/etcd/client/v3 v3.5.15 + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.55.0 + go.opentelemetry.io/otel v1.30.0 + go.opentelemetry.io/otel/sdk v1.30.0 + go.opentelemetry.io/otel/trace v1.30.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.24.0 + golang.org/x/sys v0.25.0 golang.org/x/term v0.23.0 google.golang.org/grpc v1.65.0 k8s.io/api v0.31.0 @@ -34,6 +38,7 @@ require ( k8s.io/apiserver v0.31.0 k8s.io/client-go v0.31.0 k8s.io/code-generator v0.31.0 + k8s.io/component-base v0.31.0 k8s.io/cri-api v0.31.0 k8s.io/cri-client v0.31.0 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 @@ -126,12 +131,9 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.starlark.net v0.0.0-20240520160348-046347dcd104 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -153,7 +155,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect - k8s.io/component-base v0.31.0 // indirect k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect k8s.io/klog/v2 v2.130.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 51fa171fc..b08682fd3 100644 --- a/go.sum +++ b/go.sum @@ -249,22 +249,26 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5 go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.55.0 h1:cmalGDJPtKfkOhzvlX9ey9sVqKhLckBj8jSIp7EzmfQ= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.55.0/go.mod h1:g4xHjwbbj/Bu7c21hrOvJmd4VhZiMK5nkdqbOi1lIaI= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/contrib/propagators/b3 v1.30.0 h1:vumy4r1KMyaoQRltX7cJ37p3nluzALX9nugCjNNefuY= +go.opentelemetry.io/contrib/propagators/b3 v1.30.0/go.mod h1:fRbvRsaeVZ82LIl3u0rIvusIel2UUf+JcaaIpy5taho= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20240520160348-046347dcd104 h1:3qhteRISupnJvaWshOmeqEUs2y9oc/+/ePPvDh3Eygg= @@ -307,8 +311,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/apis/config/v1alpha1/kwok_configuration_types.go b/pkg/apis/config/v1alpha1/kwok_configuration_types.go index 16980bfbc..a230a64a2 100644 --- a/pkg/apis/config/v1alpha1/kwok_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwok_configuration_types.go @@ -36,6 +36,8 @@ type KwokConfiguration struct { metav1.ObjectMeta `json:"metadata,omitempty"` // Options holds information about the default value. Options KwokConfigurationOptions `json:"options,omitempty"` + + Tracing TracingConfiguration `json:"tracing,omitempty"` } // KwokConfigurationOptions holds information about the options. @@ -147,3 +149,17 @@ type KwokConfigurationOptions struct { // +default=4 NodeLeaseParallelism uint `json:"nodeLeaseParallelism,omitempty"` } + +// TracingConfiguration provides versioned configuration for OpenTelemetry tracing clients. +type TracingConfiguration struct { + // Endpoint of the collector this component will report traces to. + // The connection is insecure, and does not currently support TLS. + // +optional + Endpoint *string `json:"endpoint,omitempty"` + + // SamplingRatePerMillion is the number of samples to collect per million spans. + // Recommended is unset. If unset, sampler respects its parent span's sampling + // rate, but otherwise never samples. + // +optional + SamplingRatePerMillion *int32 `json:"samplingRatePerMillion,omitempty"` +} diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 1b5f881f1..5fb098fc7 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -172,6 +172,7 @@ func (in *KwokConfiguration) DeepCopyInto(out *KwokConfiguration) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Options.DeepCopyInto(&out.Options) + in.Tracing.DeepCopyInto(&out.Tracing) return } @@ -420,6 +421,32 @@ func (in *Port) DeepCopy() *Port { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TracingConfiguration) DeepCopyInto(out *TracingConfiguration) { + *out = *in + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint + *out = new(string) + **out = **in + } + if in.SamplingRatePerMillion != nil { + in, out := &in.SamplingRatePerMillion, &out.SamplingRatePerMillion + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingConfiguration. +func (in *TracingConfiguration) DeepCopy() *TracingConfiguration { + if in == nil { + return nil + } + out := new(TracingConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Volume) DeepCopyInto(out *Volume) { *out = *in diff --git a/pkg/apis/internalversion/kwok_configuration_types.go b/pkg/apis/internalversion/kwok_configuration_types.go index 40fa11653..5549433cc 100644 --- a/pkg/apis/internalversion/kwok_configuration_types.go +++ b/pkg/apis/internalversion/kwok_configuration_types.go @@ -27,6 +27,8 @@ type KwokConfiguration struct { metav1.ObjectMeta // Options holds information about the default value. Options KwokConfigurationOptions + + Tracing TracingConfiguration } // KwokConfigurationOptions holds information about the options. @@ -101,3 +103,15 @@ type KwokConfigurationOptions struct { // NodeLeaseParallelism is the number of NodeLeases that are allowed to be processed in parallel. NodeLeaseParallelism uint } + +// TracingConfiguration provides versioned configuration for OpenTelemetry tracing clients. +type TracingConfiguration struct { + // Endpoint of the collector this component will report traces to. + // The connection is insecure, and does not currently support TLS. + Endpoint string + + // SamplingRatePerMillion is the number of samples to collect per million spans. + // Recommended is unset. If unset, sampler respects its parent span's sampling + // rate, but otherwise never samples. + SamplingRatePerMillion int32 +} diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index bee475131..c286cc735 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -635,6 +635,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*TracingConfiguration)(nil), (*configv1alpha1.TracingConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration(a.(*TracingConfiguration), b.(*configv1alpha1.TracingConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.TracingConfiguration)(nil), (*TracingConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration(a.(*configv1alpha1.TracingConfiguration), b.(*TracingConfiguration), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*Volume)(nil), (*configv1alpha1.Volume)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_Volume_To_v1alpha1_Volume(a.(*Volume), b.(*configv1alpha1.Volume), scope) }); err != nil { @@ -1462,6 +1472,9 @@ func autoConvert_internalversion_KwokConfiguration_To_v1alpha1_KwokConfiguration if err := Convert_internalversion_KwokConfigurationOptions_To_v1alpha1_KwokConfigurationOptions(&in.Options, &out.Options, s); err != nil { return err } + if err := Convert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration(&in.Tracing, &out.Tracing, s); err != nil { + return err + } return nil } @@ -1476,6 +1489,9 @@ func autoConvert_v1alpha1_KwokConfiguration_To_internalversion_KwokConfiguration if err := Convert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfigurationOptions(&in.Options, &out.Options, s); err != nil { return err } + if err := Convert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration(&in.Tracing, &out.Tracing, s); err != nil { + return err + } return nil } @@ -2622,6 +2638,36 @@ func Convert_v1alpha1_StageSpec_To_internalversion_StageSpec(in *v1alpha1.StageS return autoConvert_v1alpha1_StageSpec_To_internalversion_StageSpec(in, out, s) } +func autoConvert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration(in *TracingConfiguration, out *configv1alpha1.TracingConfiguration, s conversion.Scope) error { + if err := v1.Convert_string_To_Pointer_string(&in.Endpoint, &out.Endpoint, s); err != nil { + return err + } + if err := v1.Convert_int32_To_Pointer_int32(&in.SamplingRatePerMillion, &out.SamplingRatePerMillion, s); err != nil { + return err + } + return nil +} + +// Convert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration is an autogenerated conversion function. +func Convert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration(in *TracingConfiguration, out *configv1alpha1.TracingConfiguration, s conversion.Scope) error { + return autoConvert_internalversion_TracingConfiguration_To_v1alpha1_TracingConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration(in *configv1alpha1.TracingConfiguration, out *TracingConfiguration, s conversion.Scope) error { + if err := v1.Convert_Pointer_string_To_string(&in.Endpoint, &out.Endpoint, s); err != nil { + return err + } + if err := v1.Convert_Pointer_int32_To_int32(&in.SamplingRatePerMillion, &out.SamplingRatePerMillion, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration(in *configv1alpha1.TracingConfiguration, out *TracingConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_TracingConfiguration_To_internalversion_TracingConfiguration(in, out, s) +} + func autoConvert_internalversion_Volume_To_v1alpha1_Volume(in *Volume, out *configv1alpha1.Volume, s conversion.Scope) error { out.Name = in.Name if err := v1.Convert_bool_To_Pointer_bool(&in.ReadOnly, &out.ReadOnly, s); err != nil { diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 3bbf08b6a..fcfc7cdea 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -661,6 +661,7 @@ func (in *KwokConfiguration) DeepCopyInto(out *KwokConfiguration) { *out = *in in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Options.DeepCopyInto(&out.Options) + out.Tracing = in.Tracing return } @@ -1424,6 +1425,22 @@ func (in *StageSpec) DeepCopy() *StageSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TracingConfiguration) DeepCopyInto(out *TracingConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingConfiguration. +func (in *TracingConfiguration) DeepCopy() *TracingConfiguration { + if in == nil { + return nil + } + out := new(TracingConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Volume) DeepCopyInto(out *Volume) { *out = *in diff --git a/pkg/kwok/cmd/root.go b/pkg/kwok/cmd/root.go index c49ee08d6..7b73f97cd 100644 --- a/pkg/kwok/cmd/root.go +++ b/pkg/kwok/cmd/root.go @@ -28,6 +28,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/clock" + "k8s.io/component-base/tracing" + "go.opentelemetry.io/otel/attribute" + tracingapi "k8s.io/component-base/tracing/api/v1" + otelsdkresource "go.opentelemetry.io/otel/sdk/resource" nodefast "sigs.k8s.io/kwok/kustomize/stage/node/fast" nodeheartbeat "sigs.k8s.io/kwok/kustomize/stage/node/heartbeat" @@ -95,6 +99,8 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Options.ServerAddress, "server-address", flags.Options.ServerAddress, "Address to expose the server on") cmd.Flags().UintVar(&flags.Options.NodeLeaseDurationSeconds, "node-lease-duration-seconds", flags.Options.NodeLeaseDurationSeconds, "Duration of node lease seconds") cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") + cmd.Flags().StringVar(&flags.Tracing.Endpoint, "tracing-endpoint", flags.Tracing.Endpoint, "Tracing endpoint") + cmd.Flags().Int32Var(&flags.Tracing.SamplingRatePerMillion, "tracing-sampling-rate-per-million", flags.Tracing.SamplingRatePerMillion, "Tracing sampling rate per million") cmd.Flags().BoolVar(&flags.Options.EnableCNI, "experimental-enable-cni", flags.Options.EnableCNI, "Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux") _ = cmd.Flags().MarkDeprecated("experimental-enable-cni", "It will be removed and will be supported in the form of plugins") @@ -119,6 +125,12 @@ var crdDefines = map[string]struct{}{ func runE(ctx context.Context, flags *flagpole) error { logger := log.FromContext(ctx) + id, err := controllers.Identity() + if err != nil { + return err + } + ctx = log.NewContext(ctx, logger.With("id", id)) + if flags.Kubeconfig != "" { var err error flags.Kubeconfig, err = path.Expand(flags.Kubeconfig) @@ -139,7 +151,7 @@ func runE(ctx context.Context, flags *flagpole) error { } stagesData := config.FilterWithTypeFromContext[*internalversion.Stage](ctx) - err := checkConfigOrCRD(flags.Options.EnableCRDs, v1alpha1.StageKind, stagesData) + err = checkConfigOrCRD(flags.Options.EnableCRDs, v1alpha1.StageKind, stagesData) if err != nil { return err } @@ -174,6 +186,24 @@ func runE(ctx context.Context, flags *flagpole) error { logger.Warn("Neither --kubeconfig nor --master was specified") logger.Info("Using the inClusterConfig") } + + var tracingProvider tracing.TracerProvider + if flags.Tracing.Endpoint != "" { + resourceOpts := []otelsdkresource.Option{ + otelsdkresource.WithAttributes( + attribute.Key("service.name").String("kwok-controller"), + attribute.Key("service.instance.id").String(id), + ), + } + tracingProvider, err = tracing.NewProvider(ctx, &tracingapi.TracingConfiguration{ + Endpoint: &flags.Tracing.Endpoint, + SamplingRatePerMillion: &flags.Tracing.SamplingRatePerMillion, + }, nil, resourceOpts) + if err != nil { + return err + } + } + clientset, err := client.NewClientset(flags.Master, flags.Kubeconfig) if err != nil { return err @@ -184,6 +214,10 @@ func runE(ctx context.Context, flags *flagpole) error { return err } + if tracingProvider != nil { + restConfig.Wrap(tracing.WrapperFor(tracingProvider)) + } + dynamicClient, err := clientset.ToDynamicClient() if err != nil { return err @@ -229,12 +263,6 @@ func runE(ctx context.Context, flags *flagpole) error { ) } - id, err := controllers.Identity() - if err != nil { - return err - } - ctx = log.NewContext(ctx, logger.With("id", id)) - metrics := config.FilterWithTypeFromContext[*internalversion.Metric](ctx) enableMetrics := len(metrics) != 0 || slices.Contains(flags.Options.EnableCRDs, v1alpha1.MetricKind) ctr, err := controllers.NewController(controllers.Config{ @@ -274,7 +302,7 @@ func runE(ctx context.Context, flags *flagpole) error { return err } - err = startServer(ctx, flags, ctr, typedKwokClient) + err = startServer(ctx, flags, ctr, typedKwokClient, tracingProvider) if err != nil { return err } @@ -283,7 +311,7 @@ func runE(ctx context.Context, flags *flagpole) error { return nil } -func startServer(ctx context.Context, flags *flagpole, ctr *controllers.Controller, typedKwokClient versioned.Interface) (err error) { +func startServer(ctx context.Context, flags *flagpole, ctr *controllers.Controller, typedKwokClient versioned.Interface, tracingProvider tracing.TracerProvider) (err error) { logger := log.FromContext(ctx) serverAddress := flags.Options.ServerAddress @@ -384,6 +412,10 @@ func startServer(ctx context.Context, flags *flagpole, ctr *controllers.Controll svc.InstallServiceDiscovery() + if tracingProvider != nil { + svc.InstallTracingFilter(tracingProvider) + } + if flags.Options.EnableDebuggingHandlers { svc.InstallDebuggingHandlers() svc.InstallProfilingHandler(flags.Options.EnableProfilingHandler, flags.Options.EnableContentionProfiling) diff --git a/pkg/kwok/server/server.go b/pkg/kwok/server/server.go index 30608a85d..f58b7c47f 100644 --- a/pkg/kwok/server/server.go +++ b/pkg/kwok/server/server.go @@ -30,6 +30,8 @@ import ( "github.com/wzshiming/cmux/pattern" corev1 "k8s.io/api/core/v1" remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand" + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful" "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/apis/v1alpha1" @@ -441,6 +443,10 @@ func (s *Server) InstallCRD(ctx context.Context) error { return nil } +func (s *Server) InstallTracingFilter(tp oteltrace.TracerProvider) { + s.restfulCont.Filter(otelrestful.OTelFilter("kwok-controller", otelrestful.WithTracerProvider(tp))) +} + // Run runs the specified Server. // This should never exit. func (s *Server) Run(ctx context.Context, address string, certFile, privateKeyFile string) error { diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index db25b1d38..277ac13c4 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -48,6 +48,7 @@ type BuildKwokControllerComponentConfig struct { Verbosity log.Level NodeLeaseDurationSeconds uint EnableCRDs []string + OtlpGrpcAddress string } // BuildKwokControllerComponent builds a kwok controller component. @@ -173,6 +174,13 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp kwokControllerArgs = append(kwokControllerArgs, "--enable-crds="+strings.Join(conf.EnableCRDs, ",")) } + if conf.OtlpGrpcAddress != "" { + kwokControllerArgs = append(kwokControllerArgs, + "--tracing-endpoint="+conf.OtlpGrpcAddress, + "--tracing-sampling-rate-per-million=1000000", + ) + } + envs := []internalversion.Env{} return internalversion.Component{ diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index aa7062915..fb4d9c0a9 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -579,6 +579,11 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return err } + otlpGrpcAddress := "" + if conf.JaegerOtlpGrpcPort != 0 { + otlpGrpcAddress = net.LocalAddress + ":" + format.String(conf.JaegerOtlpGrpcPort) + } + kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -596,10 +601,9 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, EnableCRDs: conf.EnableCRDs, + OtlpGrpcAddress: otlpGrpcAddress, }) - if err != nil { - return err - } + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) return nil } diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 086efac12..520f54bed 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -586,6 +586,11 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return err } + otlpGrpcAddress := "" + if conf.JaegerOtlpGrpcPort != 0 { + otlpGrpcAddress = net.LocalAddress + ":" + format.String(conf.JaegerOtlpGrpcPort) + } + logVolumes := runtime.GetLogVolumes(ctx) kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ @@ -605,6 +610,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, EnableCRDs: conf.EnableCRDs, + OtlpGrpcAddress: otlpGrpcAddress, }) kwokControllerComponent.Volumes = append(kwokControllerComponent.Volumes, logVolumes...) diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 40446b945..49cd790bc 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -632,6 +632,11 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return v }) + otlpGrpcAddress := "" + if conf.JaegerOtlpGrpcPort != 0 { + otlpGrpcAddress = net.LocalAddress + ":" + format.String(conf.JaegerOtlpGrpcPort) + } + kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, ProjectName: c.Name(), @@ -651,6 +656,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { Verbosity: env.verbosity, NodeLeaseDurationSeconds: 40, EnableCRDs: conf.EnableCRDs, + OtlpGrpcAddress: otlpGrpcAddress, }) kwokControllerComponent.Volumes = append(kwokControllerComponent.Volumes, logVolumes...) diff --git a/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt index 760716f51..387edd0eb 100644 --- a/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt @@ -166,7 +166,7 @@ cd /workdir/clusters/ && kube-controller-manager --node- echo $! >/workdir/clusters//pids/kube-controller-manager.pid cd /workdir/clusters/ && kube-scheduler --config=/workdir/clusters//scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32760 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=200 --tracing-endpoint=127.0.0.1:32762 --tracing-sampling-rate-per-million=1000000 >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid cd /workdir/clusters/ && metrics-server --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=32759 --kubeconfig=/workdir/clusters//kubeconfig --authentication-kubeconfig=/workdir/clusters//kubeconfig --authorization-kubeconfig=/workdir/clusters//kubeconfig --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key >/workdir/clusters//logs/metrics-server.log 2>&1 & echo $! >/workdir/clusters//pids/metrics-server.pid