Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into modern-session-view
Browse files Browse the repository at this point in the history
  • Loading branch information
mjwolf committed Oct 8, 2024
2 parents 84cc436 + 6339a42 commit abf5341
Show file tree
Hide file tree
Showing 25 changed files with 1,267 additions and 43 deletions.
7 changes: 6 additions & 1 deletion CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Fix long filepaths in diagnostics exceeding max path limits on Windows. {pull}40909[40909]
- Add backup and delete for AWS S3 polling mode feature back. {pull}41071[41071]
- Fix a bug in Salesforce input to only handle responses with 200 status code {pull}41015[41015]

- Fixed failed job handling and removed false-positive error logs in the GCS input. {pull}41142[41142]

*Heartbeat*

Expand Down Expand Up @@ -202,6 +202,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Remove excessive info-level logs in cgroups setup {pull}40491[40491]
- Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467]
- Add AWS OwningAccount support for cross account monitoring {issue}40570[40570] {pull}40691[40691]
- Use namespace for GetListMetrics when exists in AWS {pull}41022[41022]
- Fix http server helper SSL config. {pull}39405[39405]

*Osquerybeat*
Expand Down Expand Up @@ -233,6 +234,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- When running under Elastic-Agent Kafka output allows dynamic topic in `topic` field {pull}40415[40415]
- The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions.
- Update to Go 1.22.7. {pull}41018[41018]
- Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942]


*Auditbeat*

Expand Down Expand Up @@ -317,6 +320,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Add support to CEL for reading host environment variables. {issue}40762[40762] {pull}40779[40779]
- Add CSV decoder to awss3 input. {pull}40896[40896]
- Change request trace logging to include headers instead of complete request. {pull}41072[41072]
- Improved GCS input documentation. {pull}41143[41143]
- Add CSV decoding capacity to azureblobstorage input {pull}40978[40978]

*Auditbeat*

Expand Down
4 changes: 2 additions & 2 deletions dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ shared:
- &docker_spec
<<: *binary_spec
extra_vars:
from: '--platform=linux/amd64 ubuntu:20.04'
from: '--platform=linux/amd64 ubuntu:24.04'
buildFrom: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base'
user: '{{ .BeatName }}'
linux_capabilities: ''
Expand All @@ -172,7 +172,7 @@ shared:
- &docker_arm_spec
<<: *docker_spec
extra_vars:
from: '--platform=linux/arm64 ubuntu:20.04'
from: '--platform=linux/arm64 ubuntu:24.04'
buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'

- &docker_ubi_spec
Expand Down
2 changes: 2 additions & 0 deletions dev-tools/packaging/templates/docker/Dockerfile.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ RUN for iter in {1..10}; do \
{{- end }}

{{- if contains .from "ubuntu" }}
RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu

RUN for iter in {1..10}; do \
apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl gawk libcap2-bin xz-utils && \
Expand Down
3 changes: 2 additions & 1 deletion metricbeat/docs/modules/aws.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,8 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
| IAM ListAccountAliases | 1 | Once on startup
| STS GetCallerIdentity | 1 | Once on startup
| EC2 DescribeRegions| 1 | Once on startup
| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
| CloudWatch ListMetrics without specifying namespace in configuration | Total number of results / ListMetrics max page size | Per region per collection period
| CloudWatch ListMetrics with specific namespaces in configuration | Total number of results / ListMetrics max page size * number of unique namespaces | Per region per collection period
| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per namespace per collection period
|===
`billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.
Expand Down
2 changes: 1 addition & 1 deletion metricbeat/module/kubernetes/kubernetes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ spec:
spec:
containers:
- name: kube-state-metrics
image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.7.0
image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.12.0
livenessProbe:
httpGet:
path: /healthz
Expand Down
14 changes: 10 additions & 4 deletions metricbeat/module/system/process/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
return nil, err
}

sys := base.Module().(resolve.Resolver)
sys, ok := base.Module().(resolve.Resolver)
if !ok {
return nil, fmt.Errorf("resolver cannot be cast from the module")
}
enableCgroups := false
if runtime.GOOS == "linux" {
if config.Cgroups == nil || *config.Cgroups {
Expand Down Expand Up @@ -131,14 +134,17 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
return err
} else {
proc, root, err := m.stats.GetOneRootEvent(m.setpid)
if err != nil {
if err != nil && !errors.Is(err, process.NonFatalErr{}) {
// return only if the error is fatal in nature
return fmt.Errorf("error fetching pid %d: %w", m.setpid, err)
} else if (err != nil && errors.Is(err, process.NonFatalErr{})) {
err = mb.PartialMetricsError{Err: err}
}
// if error is non-fatal, emit partial metrics.
r.Event(mb.Event{
MetricSetFields: proc,
RootFields: root,
})
return err
}

return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

//go:build !(linux && (amd64 || arm64) && cgo)
//go:build linux && !((amd64 || arm64) && cgo)

package modernprovider

Expand Down
55 changes: 55 additions & 0 deletions x-pack/filebeat/docs/inputs/input-azure-blob-storage.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,61 @@ Example : `10s` would mean we would like the polling to occur every 10 seconds.
This attribute can be specified both at the root level of the configuration as well at the container level. The container level values will always
take priority and override the root level values if both are specified.

[id="input-{type}-encoding"]
[float]
==== `encoding`

The file encoding to use for reading data that contains international
characters. This only applies to non-JSON logs. See <<_encoding_3>>.

[id="input-{type}-decoding"]
[float]
==== `decoding`

The file decoding option is used to specify a codec that will be used to
decode the file contents. This can apply to any file stream data.
An example config is shown below:

Currently supported codecs are given below:-

1. <<attrib-decoding-csv-azureblobstorage,CSV>>: This codec decodes RFC 4180 CSV data streams.

[id="attrib-decoding-csv-azureblobstorage"]
[float]
==== `the CSV codec`
The `CSV` codec is used to decode RFC 4180 CSV data streams.
Enabling the codec without other options will use the default codec options.

[source,yaml]
----
decoding.codec.csv.enabled: true
----

The CSV codec supports five sub attributes to control aspects of CSV decoding.
The `comma` attribute specifies the field separator character used by the CSV
format. If it is not specified, the comma character '`,`' is used. The `comment`
attribute specifies the character that should be interpreted as a comment mark.
If it is specified, lines starting with the character will be ignored. Both
`comma` and `comment` must be single characters. The `lazy_quotes` attribute
controls how quoting in fields is handled. If `lazy_quotes` is true, a quote may
appear in an unquoted field and a non-doubled quote may appear in a quoted field.
The `trim_leading_space` attribute specifies that leading white space should be
ignored, even if the `comma` character is white space. For complete details
of the preceding configuration attribute behaviors, see the CSV decoder
https://pkg.go.dev/encoding/csv#Reader[documentation] The `fields_names`
attribute can be used to specify the column names for the data. If it is
absent, the field names are obtained from the first non-comment line of
data. The number of fields must match the number of field names.

An example config is shown below:

[source,yaml]
----
decoding.codec.csv.enabled: true
decoding.codec.csv.comma: "\t"
decoding.codec.csv.comment: "#"
----

[id="attrib-file_selectors"]
[float]
==== `file_selectors`
Expand Down
19 changes: 12 additions & 7 deletions x-pack/filebeat/docs/inputs/input-gcs.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -213,17 +213,16 @@ This is a specific subfield of a bucket. It specifies the bucket name.

This attribute defines the maximum amount of time after which a bucket operation will give and stop if no response is recieved (example: reading a file / listing a file).
It can be defined in the following formats : `{{x}}s`, `{{x}}m`, `{{x}}h`, here `s = seconds`, `m = minutes` and `h = hours`. The value `{{x}}` can be anything we wish.
If no value is specified for this, by default its initialized to `50 seconds`. This attribute can be specified both at the root level of the configuration as well at the bucket level.
The bucket level values will always take priority and override the root level values if both are specified.
If no value is specified for this, by default its initialized to `50 seconds`. This attribute can be specified both at the root level of the configuration as well at the bucket level. The bucket level values will always take priority and override the root level values if both are specified. The value of `bucket_timeout` that should be used depends on the size of the files and the network speed. If the timeout is too low, the input will not be able to read the file completely and `context_deadline_exceeded` errors will be seen in the logs. If the timeout is too high, the input will wait for a long time for the file to be read, which can cause the input to be slow. The ratio between the `bucket_timeout` and `poll_interval` should be considered while setting both the values. A low `poll_interval` and a very high `bucket_timeout` can cause resource utilization issues as schedule ops will be spawned every poll iteration. If previous poll ops are still running, this could result in concurrently running ops and so could cause a bottleneck over time.

[id="attrib-max_workers-gcs"]
[float]
==== `max_workers`

This attribute defines the maximum number of workers (go routines / lightweight threads) are allocated in the worker pool (thread pool) for processing jobs
which read contents of file. More number of workers equals a greater amount of concurrency achieved. There is an upper cap of `5000` workers per bucket that
can be defined due to internal sdk constraints. This attribute can be specified both at the root level of the configuration as well at the bucket level.
The bucket level values will always take priority and override the root level values if both are specified.
This attribute defines the maximum number of workers (goroutines / lightweight threads) are allocated in the worker pool (thread pool) for processing jobs which read the contents of files. This attribute can be specified both at the root level of the configuration and at the bucket level. Bucket level values override the root level values if both are specified. Larger number of workers do not necessarily improve of throughput, and this should be carefully tuned based on the number of files, the size of the files being processed and resources available. Increasing `max_workers` to very high values may cause resource utilization problems and can lead to a bottleneck in processing. Usually a maximum cap of `2000` workers is recommended. A very low `max_worker` count will drastically increase the number of network calls required to fetch the objects, which can cause a bottleneck in processing.

NOTE: The value of `max_workers` is tied to the `batch_size` currently to ensure even distribution of workloads across all goroutines. This ensures that the input is able to process the files in an efficient manner. This `batch_size` determines how many objects will be fetched in one single call. The `max_workers` value should be set based on the number of files to be read, the resources available and the network speed. For example,`max_workers=3` would mean that every pagination request a total number of `3` gcs objects are fetched and distributed among `3 goroutines`, `max_workers=100` would mean `100` gcs objects are fetched in every pagination request and distributed among `100 goroutines`.


[id="attrib-poll-gcs"]
[float]
Expand All @@ -241,7 +240,9 @@ This attribute defines the maximum amount of time after which the internal sched
defined in the following formats : `{{x}}s`, `{{x}}m`, `{{x}}h`, here `s = seconds`, `m = minutes` and `h = hours`. The value `{{x}}` can be anything we wish.
Example : `10s` would mean we would like the polling to occur every 10 seconds. If no value is specified for this, by default its initialized to `300 seconds`.
This attribute can be specified both at the root level of the configuration as well at the bucket level. The bucket level values will always take priority
and override the root level values if both are specified.
and override the root level values if both are specified. The `poll_interval` should be set to a value that is equal to the `bucket_timeout` value. This would ensure that another schedule operation is not started before the current buckets have all been processed. If the `poll_interval` is set to a value that is less than the `bucket_timeout`, then the input will start another schedule operation before the current one has finished, which can cause a bottleneck over time. Having a lower `poll_interval` can make the input faster at the cost of more resource utilization.

NOTE: Some edge case scenarios could require different values for `poll_interval` and `bucket_timeout`. For example, if the files are very large and the network speed is slow, then the `bucket_timeout` value should be set to a higher value than the `poll_interval`. This would ensure that polling operation does not wait too long for the files to be read and moves to the next iteration while the current one is still being processed. This would ensure a higher throughput and better resource utilization.

[id="attrib-parse_json"]
[float]
Expand Down Expand Up @@ -276,6 +277,8 @@ filebeat.inputs:
- regex: '/Security-Logs/'
----

The `file_selectors` operation is performed within the agent locally, hence using this option will cause the agent to download all the files and then filter them. This can cause a bottleneck in processing if the number of files is very high. It is recommended to use this attribute only when the number of files is limited or ample resources are available.

[id="attrib-expand_event_list_from_field-gcs"]
[float]
==== `expand_event_list_from_field`
Expand Down Expand Up @@ -341,6 +344,8 @@ filebeat.inputs:
timestamp_epoch: 1630444800
----

The GCS APIs don't provide a direct way to filter files based on the timestamp, so the input will download all the files and then filter them based on the timestamp. This can cause a bottleneck in processing if the number of files are very high. It is recommended to use this attribute only when the number of files are limited or ample resources are available. This option scales vertically and not horizontally.

[id="bucket-overrides"]
*The sample configs below will explain the bucket level overriding of attributes a bit further :-*

Expand Down
9 changes: 9 additions & 0 deletions x-pack/filebeat/input/azureblobstorage/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"

"github.com/elastic/beats/v7/libbeat/common/match"
"github.com/elastic/beats/v7/libbeat/reader/parser"
)

// MaxWorkers, Poll, PollInterval, FileSelectors, TimeStampEpoch & ExpandEventListFromField can
Expand All @@ -25,6 +26,7 @@ type config struct {
PollInterval *time.Duration `config:"poll_interval"`
Containers []container `config:"containers" validate:"required"`
FileSelectors []fileSelectorConfig `config:"file_selectors"`
ReaderConfig readerConfig `config:",inline"`
TimeStampEpoch *int64 `config:"timestamp_epoch"`
ExpandEventListFromField string `config:"expand_event_list_from_field"`
}
Expand All @@ -36,6 +38,7 @@ type container struct {
Poll *bool `config:"poll"`
PollInterval *time.Duration `config:"poll_interval"`
FileSelectors []fileSelectorConfig `config:"file_selectors"`
ReaderConfig readerConfig `config:",inline"`
TimeStampEpoch *int64 `config:"timestamp_epoch"`
ExpandEventListFromField string `config:"expand_event_list_from_field"`
}
Expand All @@ -46,6 +49,12 @@ type fileSelectorConfig struct {
// TODO: Add support for reader config in future
}

// readerConfig defines the options for reading the content of an azure container.
type readerConfig struct {
Parsers parser.Config `config:",inline"`
Decoding decoderConfig `config:"decoding"`
}

type authConfig struct {
SharedCredentials *sharedKeyConfig `config:"shared_credentials"`
ConnectionString *connectionStringConfig `config:"connection_string"`
Expand Down
47 changes: 47 additions & 0 deletions x-pack/filebeat/input/azureblobstorage/decoding.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package azureblobstorage

import (
"fmt"
"io"
)

// decoder is an interface for decoding data from an io.Reader.
type decoder interface {
// decode reads and decodes data from an io reader based on the codec type.
// It returns the decoded data and an error if the data cannot be decoded.
decode() ([]byte, error)
// next advances the decoder to the next data item and returns true if there is more data to be decoded.
next() bool
// close closes the decoder and releases any resources associated with it.
// It returns an error if the decoder cannot be closed.

// more returns whether there are more records to read.
more() bool

close() error
}

// valueDecoder is a decoder that can decode directly to a JSON serialisable value.
type valueDecoder interface {
decoder

decodeValue() ([]byte, map[string]any, error)
}

// newDecoder creates a new decoder based on the codec type.
// It returns a decoder type and an error if the codec type is not supported.
// If the reader config codec option is not set, it returns a nil decoder and nil error.
func newDecoder(cfg decoderConfig, r io.Reader) (decoder, error) {
switch {
case cfg.Codec == nil:
return nil, nil
case cfg.Codec.CSV != nil:
return newCSVDecoder(cfg, r)
default:
return nil, fmt.Errorf("unsupported config value: %v", cfg)
}
}
54 changes: 54 additions & 0 deletions x-pack/filebeat/input/azureblobstorage/decoding_config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package azureblobstorage

import (
"fmt"
"unicode/utf8"
)

// decoderConfig contains the configuration options for instantiating a decoder.
type decoderConfig struct {
Codec *codecConfig `config:"codec"`
}

// codecConfig contains the configuration options for different codecs used by a decoder.
type codecConfig struct {
CSV *csvCodecConfig `config:"csv"`
}

// csvCodecConfig contains the configuration options for the CSV codec.
type csvCodecConfig struct {
Enabled bool `config:"enabled"`

// Fields is the set of field names. If it is present
// it is used to specify the object names of returned
// values and the FieldsPerRecord field in the csv.Reader.
// Otherwise, names are obtained from the first
// line of the CSV data.
Fields []string `config:"fields_names"`

// The fields below have the same meaning as the
// fields of the same name in csv.Reader.
Comma *configRune `config:"comma"`
Comment configRune `config:"comment"`
LazyQuotes bool `config:"lazy_quotes"`
TrimLeadingSpace bool `config:"trim_leading_space"`
}

type configRune rune

func (r *configRune) Unpack(s string) error {
if s == "" {
return nil
}
n := utf8.RuneCountInString(s)
if n != 1 {
return fmt.Errorf("single character option given more than one character: %q", s)
}
_r, _ := utf8.DecodeRuneInString(s)
*r = configRune(_r)
return nil
}
Loading

0 comments on commit abf5341

Please sign in to comment.