Merge branch 'main' into gcp-histogram

CHANGELOG.next.asciidoc

@@ -398,6 +398,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Bump aerospike-client-go to version v7.7.1 and add support for basic auth in Aerospike module {pull}41233[41233]
 - Only watch metadata for ReplicaSets in metricbeat k8s module {pull}41289[41289]
 - Add support for region/zone for Vertex AI service in GCP module {pull}41551[41551]
+- Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626]
 
 *Metricbeat*
 

metricbeat/cmd/root.go

@@ -43,9 +43,6 @@ const (
 	Name = "metricbeat"
 )
 
-// RootCmd to handle beats cli
-var RootCmd *cmd.BeatsRootCmd
-
 // withECSVersion is a modifier that adds ecs.version to events.
 var withECSVersion = processing.WithFields(mapstr.M{
 	"ecs": mapstr.M{
@@ -60,7 +57,7 @@ func MetricbeatSettings(moduleNameSpace string) instance.Settings {
 	if moduleNameSpace == "" {
 		moduleNameSpace = "module"
 	}
-	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
+	runFlags := pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs"))
 	cfgfile.AddAllowedBackwardsCompatibleFlag("system.hostfs")
 	return instance.Settings{
@@ -82,7 +79,3 @@ func Initialize(settings instance.Settings) *cmd.BeatsRootCmd {
 	rootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator()))
 	return rootCmd
 }
-
-func init() {
-	RootCmd = Initialize(MetricbeatSettings(""))
-}

metricbeat/docs/modules/gcp.asciidoc

@@ -346,6 +346,7 @@ metricbeat.modules:
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 1m
+  location_label: "resource.labels.zone"
   metrics:
     - aligner: ALIGN_NONE
       service: compute

metricbeat/main.go

@@ -32,7 +32,7 @@ import (
 )
 
 func main() {
-	if err := cmd.RootCmd.Execute(); err != nil {
+	if err := cmd.Initialize(cmd.MetricbeatSettings("")).Execute(); err != nil {
 		os.Exit(1)
 	}
 }

metricbeat/main_test.go

@@ -21,32 +21,40 @@ package main
 
 import (
 	"flag"
+	"os"
 	"testing"
 
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
+	cmd "github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/tests/system/template"
-	"github.com/elastic/beats/v7/metricbeat/cmd"
+	mbcmd "github.com/elastic/beats/v7/metricbeat/cmd"
 )
 
-var systemTest *bool
+var (
+	systemTest *bool
+	mbCommand  *cmd.BeatsRootCmd
+)
 
 func init() {
 	testing.Init()
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
-	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	mbCommand = mbcmd.Initialize(mbcmd.MetricbeatSettings(""))
+	mbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
 	cfgfile.AddAllowedBackwardsCompatibleFlag("systemTest")
-	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
+	mbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 	cfgfile.AddAllowedBackwardsCompatibleFlag("test.coverprofile")
 }
 
 // Test started when the test binary is started. Only calls main.
 func TestSystem(t *testing.T) {
 	cfgfile.ConvertFlagsForBackwardsCompatibility()
 	if *systemTest {
-		main()
+		if err := mbCommand.Execute(); err != nil {
+			os.Exit(1)
+		}
 	}
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, cmd.Name, false)
+	template.TestTemplate(t, mbCommand.Name(), false)
 }

x-pack/agentbeat/main.go

@@ -42,7 +42,7 @@ into a single agentbeat binary.`,
 		prepareCommand(auditbeat.RootCmd),
 		prepareCommand(filebeat.Filebeat()),
 		prepareCommand(heartbeat.RootCmd),
-		prepareCommand(metricbeat.RootCmd),
+		prepareCommand(metricbeat.Initialize()),
 		prepareCommand(osquerybeat.RootCmd),
 		prepareCommand(packetbeat.RootCmd),
 	)

x-pack/libbeat/management/tests/mbtest/metricbeat_v2_test.go

@@ -34,8 +34,9 @@ var expectedMBStreams = &proto.UnitExpectedConfig{
 }
 
 func TestSingleMetricbeatMetricsetWithProcessors(t *testing.T) {
-	tests.InitBeatsForTest(t, cmd.RootCmd)
-	var mbStreams = []*proto.Stream{
+	mbCmd := cmd.Initialize()
+	tests.InitBeatsForTest(t, mbCmd)
+	mbStreams := []*proto.Stream{
 		{
 			Id: "system/metrics-system.cpu-default-system",
 			DataStream: &proto.DataStream{
@@ -79,7 +80,7 @@ func TestSingleMetricbeatMetricsetWithProcessors(t *testing.T) {
 
 	go func() {
 		t.Logf("Running beats...")
-		err := cmd.RootCmd.Execute()
+		err := mbCmd.Execute()
 		require.NoError(t, err)
 	}()
 

x-pack/libbeat/management/tests/mbtest/system/process_integration_test.go

@@ -37,7 +37,8 @@ func TestProcessStatusReporter(t *testing.T) {
 	unitOutID := mock.NewID()
 	token := mock.NewID()
 
-	tests.InitBeatsForTest(t, cmd.RootCmd)
+	mbCmd := cmd.Initialize()
+	tests.InitBeatsForTest(t, mbCmd)
 
 	filename := fmt.Sprintf("test-%d", time.Now().Unix())
 	outPath := filepath.Join(t.TempDir(), filename)
@@ -122,7 +123,7 @@ func TestProcessStatusReporter(t *testing.T) {
 
 	go func() {
 		t.Logf("Running beats...")
-		err := cmd.RootCmd.Execute()
+		err := mbCmd.Execute()
 		require.NoError(t, err)
 	}()
 

x-pack/metricbeat/cmd/root.go

@@ -33,30 +33,28 @@ const (
 	Name = "metricbeat"
 )
 
-// RootCmd to handle beats cli
-var RootCmd *cmd.BeatsRootCmd
-
 // withECSVersion is a modifier that adds ecs.version to events.
 var withECSVersion = processing.WithFields(mapstr.M{
 	"ecs": mapstr.M{
 		"version": ecs.Version,
 	},
 })
 
-func init() {
+func Initialize() *cmd.BeatsRootCmd {
 	globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors())
 	if err != nil { // these are hard-coded, shouldn't fail
 		panic(fmt.Errorf("error creating global processors: %w", err))
 	}
 	settings := mbcmd.MetricbeatSettings("")
 	settings.ElasticLicensed = true
 	settings.Processing = processing.MakeDefaultSupport(true, globalProcs, withECSVersion, processing.WithHost, processing.WithAgentMeta())
-	RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings)
-	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager))
-	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator()))
-	RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
+	rootCmd := cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings)
+	rootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager))
+	rootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator()))
+	rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
 		management.ConfigTransform.SetTransform(metricbeatCfg)
 	}
+	return rootCmd
 }
 
 func defaultProcessors() []mapstr.M {

x-pack/metricbeat/main.go

@@ -19,7 +19,7 @@ import (
 )
 
 func main() {
-	if err := cmd.RootCmd.Execute(); err != nil {
+	if err := cmd.Initialize().Execute(); err != nil {
 		os.Exit(1)
 	}
 }

x-pack/metricbeat/main_test.go

@@ -6,32 +6,40 @@ package main
 // This file is mandatory as otherwise the metricbeat.test binary is not generated correctly.
 import (
 	"flag"
+	"os"
 	"testing"
 
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
+	cmd "github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/tests/system/template"
-	"github.com/elastic/beats/v7/x-pack/metricbeat/cmd"
+	mbcmd "github.com/elastic/beats/v7/x-pack/metricbeat/cmd"
 )
 
-var systemTest *bool
+var (
+	systemTest *bool
+	mbCommand  *cmd.BeatsRootCmd
+)
 
 func init() {
 	testing.Init()
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
-	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
+	mbCommand = mbcmd.Initialize()
+	mbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
 	cfgfile.AddAllowedBackwardsCompatibleFlag("systemTest")
-	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
+	mbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
 	cfgfile.AddAllowedBackwardsCompatibleFlag("test.coverprofile")
 }
 
 // Test started when the test binary is started. Only calls main.
 func TestSystem(t *testing.T) {
 	cfgfile.ConvertFlagsForBackwardsCompatibility()
 	if *systemTest {
-		main()
+		if err := mbCommand.Execute(); err != nil {
+			os.Exit(1)
+		}
 	}
 }
 
 func TestTemplate(t *testing.T) {
-	template.TestTemplate(t, cmd.Name, true)
+	template.TestTemplate(t, mbCommand.Name(), true)
 }

x-pack/metricbeat/metricbeat.reference.yml

@@ -618,6 +618,7 @@ metricbeat.modules:
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 1m
+  location_label: "resource.labels.zone"
   metrics:
     - aligner: ALIGN_NONE
       service: compute

x-pack/metricbeat/module/gcp/_meta/config.yml

@@ -34,6 +34,7 @@
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 1m
+  location_label: "resource.labels.zone"
   metrics:
     - aligner: ALIGN_NONE
       service: compute

x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc

@@ -45,6 +45,8 @@ services under "Google Cloud metrics", but does not work for other services
 (`kubernetes` aka GKE for example).
 This option allow to override the default and specify an arbitrary metric prefix.
 
+* *location_label*: Use this option to specify the resource label that identifies the location (such as zone or region) for a Google Cloud service when filtering metrics. For example, labels like `resource.label.location` or `resource.label.zone` are used by Google Cloud to represent the region or zone of a resource. This is an optional configuration for the user. 
+
 [float]
 === Example Configuration
 * `metrics` metricset is enabled to collect metrics from all zones under
@@ -134,3 +136,28 @@ metric prefix, as for GKE metrics the required prefix is `kubernetes.io/`
       metric_types:
         - "container/cpu/core_usage_time"
 ----
+
+* `metrics` metricset is enabled to collect metrics from region
+`us-east4` in `elastic-observability` project. The metric, number of replicas of the prediction model is
+collected from a new GCP service `aiplatform`. Since its a new service which is not supported by 
+default in this metricset, the user provides the servicelabel (resource.label.location), for which 
+user wants to filter the incoming data
+
++
+[source,yaml]
+----
+- module: gcp
+  metricsets:
+    - metrics
+  project_id: "elastic-observability"
+  credentials_json: "your JSON credentials"
+  exclude_labels: false
+  period: 1m
+  location_label: "resource.label.location" # This is an optional configuration
+  regions:
+  - us-east4
+  metrics:
+    - service: aiplatform
+      metric_types:
+          - "prediction/online/replicas"
+----

x-pack/metricbeat/module/gcp/metrics/metrics_requester.go

@@ -251,7 +251,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) string {
 	// NOTE: some GCP services are global, not regional or zonal. To these services we don't need
 	// to apply any additional filters.
 	if locationsConfigsAvailable && !isAGlobalService(serviceName) {
-		serviceLabel := getServiceLabelFor(serviceName)
+		serviceLabel := r.getServiceLabel(serviceName)
 		f = r.buildLocationFilter(serviceLabel, f)
 	}
 
@@ -261,6 +261,16 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) string {
 	return f
 }
 
+// getServiceLabel determines the service label to be used for the given service name. If a custom
+// location label is configured, it will be used. Otherwise, the default service label for the
+// given service name will be returned.
+func (r *metricsRequester) getServiceLabel(serviceName string) string {
+	if r.config.LocationLabel != "" {
+		return r.config.LocationLabel
+	}
+	return getServiceLabelFor(serviceName)
+}
+
 // Returns a GCP TimeInterval based on the ingestDelay and samplePeriod from ListMetricDescriptor
 func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod *durationpb.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) {
 	var startTime, endTime, currentTime time.Time

x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go

@@ -128,6 +128,13 @@ func TestGetFilterForMetric(t *testing.T) {
 			metricsRequester{config: config{Region: "foobar", Regions: []string{"foo", "bar"}}, logger: logger},
 			"metric.type=\"dummy\" AND resource.labels.zone = starts_with(\"foobar\")",
 		},
+		{
+			"aiplatform service with configured region and zone",
+			"aiplatform",
+			"",
+			metricsRequester{config: config{Region: "foo", Zone: "bar", LocationLabel: "resource.label.location"}, logger: logger},
+			"metric.type=\"dummy\" AND resource.label.location = starts_with(\"foo\")",
+		},
 	}
 
 	for _, c := range cases {

x-pack/metricbeat/module/gcp/metrics/metricset.go

@@ -103,6 +103,7 @@ type config struct {
 	Zone                string   `config:"zone"`
 	Region              string   `config:"region"`
 	Regions             []string `config:"regions"`
+	LocationLabel       string   `config:"location_label"`
 	ProjectID           string   `config:"project_id" validate:"required"`
 	ExcludeLabels       bool     `config:"exclude_labels"`
 	CredentialsFilePath string   `config:"credentials_file_path"`

x-pack/metricbeat/modules.d/gcp.yml.disabled

@@ -37,6 +37,7 @@
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 1m
+  location_label: "resource.labels.zone"
   metrics:
     - aligner: ALIGN_NONE
       service: compute