From 0dacd527405a7501d00e273dc37f19ce0db6fdaa Mon Sep 17 00:00:00 2001 From: Saman Mahdanian Date: Sat, 17 Feb 2024 13:01:46 +0330 Subject: [PATCH] not done yet. moving to new sia-mac :D --- go.mod | 10 +++++++ go.sum | 22 +++++++++++++++ internal/settings/flags.go | 3 ++ internal/settings/settings.go | 30 ++++++++++++++++++++ main.go | 46 +++++++++++++++++++----------- pkg/tracing/tracing.go | 53 +++++++++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 internal/settings/flags.go create mode 100644 internal/settings/settings.go create mode 100644 pkg/tracing/tracing.go diff --git a/go.mod b/go.mod index d339125..dc41cbd 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,15 @@ go 1.21 require ( github.com/envoyproxy/go-control-plane v0.11.1 github.com/go-logr/logr v1.2.4 + github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/prometheus/client_golang v1.16.0 github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/otel v1.17.0 + go.opentelemetry.io/otel/exporters/jaeger v1.17.0 + go.opentelemetry.io/otel/sdk v1.17.0 + go.opentelemetry.io/otel/trace v1.17.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 google.golang.org/grpc v1.58.0 k8s.io/api v0.28.1 @@ -18,11 +23,16 @@ require ( ) require ( + github.com/BurntSushi/toml v1.2.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/otel/metric v1.17.0 // indirect golang.org/x/tools v0.13.0 // indirect + olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) require ( diff --git a/go.sum b/go.sum index e28d602..513ef5d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -35,8 +37,11 @@ github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -75,8 +80,12 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= +github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -128,6 +137,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -141,6 +151,16 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= +go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= +go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc= +go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= +go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE= +go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= +go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= +go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= @@ -285,6 +305,8 @@ k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f h1:eeEUOoGYWhOz7EyXqhlR2z k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/internal/settings/flags.go b/internal/settings/flags.go new file mode 100644 index 0000000..eb5d8e2 --- /dev/null +++ b/internal/settings/flags.go @@ -0,0 +1,3 @@ +package settings + +func (s Settings) RegisterFlagPointers() diff --git a/internal/settings/settings.go b/internal/settings/settings.go new file mode 100644 index 0000000..7df6945 --- /dev/null +++ b/internal/settings/settings.go @@ -0,0 +1,30 @@ +package settings + +import "github.com/ilyakaznacheev/cleanenv" + +type Settings struct { + AuthServerAddress string `yaml:"bindAddress" env:"BIND_ADDRESS" env-default:":8082" env-description:"The address the authorization service binds to."` + MetricsAddress string `yaml:"metricsBindAddress" env:"METRICS_BIND_ADDRESS" env-default:":8080" env-description:"The address the metric endpoint binds to."` + ProbeAddress string `yaml:"healthProbeBindAddress" env:"PROBE_BIND_ADDRESS" env-default:":8081" env-description:"The address the probe endpoint binds to."` + + TLS struct { + CertPath string `yaml:"certPath" env:"AUTH_SERVER_TLS_CERT_PATH" env-default:"" env-description:"grpc Authentication server TLS certificate file path"` + KeyPath string `yaml:"keyPath" env:"AUTH_SERVER_TLS_KEY_PATH" env-default:"" env-description:"grpc Authentication server TLS Key file path"` + CaPath string `yaml:"caPath" env:"AUTH_SERVER_TLS_CA_PATH" env-default:"" env-description:"grpc Authentication server TLS CA file path"` + } `yaml:"tls"` + + LeaderElection struct { + Enabled bool `yaml:"enabled" env:"LEADER_ELECTION_ENABLED" env-default:"false" env-description:"Enable leader election for controller manager."` + ID string `yaml:"id" env:"LEADER_ELECTION_ID" env-default:"f5d1781e.snappcloud.io" env-description:"ID determines the name of the resource that leader election will use for holding the leader lock."` + } `yaml:"leaderElection"` + + Tracing struct { + Enabled bool `yaml:"enabled" env:"ENABLE_TRACING" env-default:"false" env-description:"Enable OpenTelemetry tracing."` + Provider string `yaml:"provider" env:"TRACING_PROVIDER" env-default:"jaeger" env-description:"only jaeger is available now"` + } `yaml:"tracing"` +} + +func GetSettings() (st Settings, err error) { + cleanenv.ReadEnv(&st) + return +} diff --git a/main.go b/main.go index 04ab7cf..0d0b06d 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( cerberusv1alpha1 "github.com/snapp-incubator/Cerberus/api/v1alpha1" "github.com/snapp-incubator/Cerberus/controllers" "github.com/snapp-incubator/Cerberus/pkg/auth" + "github.com/snapp-incubator/Cerberus/pkg/tracing" //+kubebuilder:scaffold:imports ) @@ -61,14 +62,6 @@ func init() { } func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var authAddr string - - var tlsCertPath string - var tlsKeyPath string - var tlsCaPath string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -78,34 +71,51 @@ func main() { flag.StringVar(&tlsKeyPath, "tls-key-path", "", "grpc Authentication server TLS key") flag.StringVar(&tlsCaPath, "tls-ca-path", "", "grpc Authentication server CA certificate") + flag.BoolVar(&enableTracing, "enable-tracing", false, + "Enable OpenTelemetry Tracing. "+ + "After enabling this you should add --tracing-provider") + flag.StringVar(&tracingProvider, "tracing-provider", "jaeger", + "Tracing provider, for now only 'jaeger'. "+ + "You should also set OTEL_EXPORTER_JAEGER_AGENT_HOST and OTEL_EXPORTER_JAEGER_AGENT_PORT.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + opts := zap.Options{ Development: true, } opts.BindFlags(flag.CommandLine) flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + var err error + if enableTracing { + if tracingProvider == "jaeger" { + err = tracing.SetTracingProvider(tracing.JaegerTracing) + } else { + reportFatalErrorAndExit(fmt.Errorf("invalid-tracing-provider"), "unable to setup environment") + } + } + if err != nil { + reportFatalErrorAndExit(err, "setup tracing provider encountered error") + } + listener, srv, authenticator, err := setupAuthenticationServer(authAddr, tlsCertPath, tlsKeyPath, tlsCaPath) if err != nil { - setupLog.Error(err, "unable to set up authentication server") - os.Exit(1) + reportFatalErrorAndExit(err, "unable to setup authentication server") } mgr, err := setupManager(metricsAddr, probeAddr, enableLeaderElection, authenticator) if err != nil { - setupLog.Error(err, "unable to set up manager") - os.Exit(1) + reportFatalErrorAndExit(err, "unable to setup manager") } //+kubebuilder:scaffold:builder err = setupHealthChecks(mgr) if err != nil { - setupLog.Error(err, "unable to set up health/ready check") - os.Exit(1) + reportFatalErrorAndExit(err, "unable to set up health/ready check") } errChan := make(chan error) @@ -116,8 +126,7 @@ func main() { select { case err := <-errChan: - setupLog.Error(err, "cerberus error") - os.Exit(1) + reportFatalErrorAndExit(err, "cerberus error") case <-ctx.Done(): os.Exit(0) } @@ -265,3 +274,8 @@ func runManager(ctx context.Context, mgr ctrl.Manager, errChan chan error) { errChan <- nil } + +func reportFatalErrorAndExit(err error, msg string) { + setupLog.Error(err, msg) + os.Exit(1) +} diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go new file mode 100644 index 0000000..6417483 --- /dev/null +++ b/pkg/tracing/tracing.go @@ -0,0 +1,53 @@ +package tracing + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "go.opentelemetry.io/otel/trace" +) + +const ( + CerberusTracerName = "cerberus" + JaegerTracing = "jaeger" +) + +var cerberusTracer trace.Tracer + +func init() { + cerberusTracer = otel.Tracer(CerberusTracerName) +} + +func SetTracingProvider(provider string) error { + if provider == JaegerTracing { + exporter, err := jaeger.New(jaeger.WithAgentEndpoint()) + if err != nil { + return nil, err + } + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exporter), + tracesdk.WithSampler(tracesdk.ParentBased(tracesdk.TraceIDRatioBased(samplerRatio))), + // Record information about this application in a Resource. + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(serviceName), + )), + ) + return tp, nil + + } + return fmt.Errorf("invalid-tracing-provider") +} + +func StartSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { + return cerberusTracer.Start(ctx, spanName) +} + +func Tracer() *trace.Tracer { + return &cerberusTracer +}