Merge main into feature/health-report-api

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
         "pipeline_slug": "logstash-pull-request-pipeline",
         "allow_org_users": true,
         "allowed_repo_permissions": ["admin", "write"],
-        "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"],
+        "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"],
         "set_commit_status": true,
         "build_on_commit": true,
         "build_on_comment": true,

.buildkite/snyk_report_pipeline.yml

@@ -9,4 +9,7 @@ steps:
   # reports main, previous (ex: 7.latest) and current (ex: 8.latest) release branches to Snyk
   - label: ":hammer: Report to Snyk"
     command:
-      - .buildkite/scripts/snyk/report.sh
+      - .buildkite/scripts/snyk/report.sh
+    retry:
+      automatic:
+        - limit: 3

ci/branches.json

@@ -7,9 +7,6 @@
       {
         "branch": "8.15"
       },
-      {
-        "branch": "8.14"
-      },
       {
         "branch": "7.17"
       }

ci/logstash_releases.json

@@ -2,11 +2,11 @@
   "releases": {
     "5.x": "5.6.16",
     "6.x": "6.8.23",
-    "7.x": "7.17.22",
+    "7.x": "7.17.23",
     "8.x": "8.14.3"
   },
   "snapshots": {
-    "7.x": "7.17.23-SNAPSHOT",
+    "7.x": "7.17.24-SNAPSHOT",
     "8.x": "8.14.4-SNAPSHOT",
     "main": "8.15.0-SNAPSHOT"
   }

docs/static/breaking-changes.asciidoc

@@ -12,7 +12,22 @@ Logstash 8.0.0 and later.
 
 NOTE: Migrating directly between non-consecutive major versions (6.x to
 8.x) is not recommended. 
-We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0, 
+We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0.
+
+[discrete]
+[[breaking-8.15]]
+=== 8.15
+
+*New {ls} SNMP integration plugin*
+
+Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. 
+When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new SNMP integration plugin.
+
+IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in the {logstash-ref}/plugins-integrations-snmp.html[SNMP integration plugin].
+If you need to maintain current mappings for the `input-snmptrap` plugin, you have some options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior].
+
+[discrete]
+=== Breaking changes in earlier releases
 
 See these topics for breaking changes in earlier releases:
 

docs/static/monitoring/monitoring-apis.asciidoc

@@ -499,10 +499,6 @@ Example response:
     "worker_concurrency" : {
       "current": 1.973,
       "lifetime": 1.721
-    },
-    "worker_utilization" : {
-      "current": 49.32,
-      "lifetime": 43.02
     }
   }
 }
@@ -540,19 +536,6 @@ Tuning a saturated pipeline to have more workers can often work to increase that
 
 A _process_ is also considered "saturated" when its top-level `worker_concurrency` flow metric approaches the _cumulative_ `pipeline.workers` across _all_ pipelines, and similarly can be addressed by tuning the <<pipeline-stats,individual pipelines>> that are saturated.
 
-| worker_utilization |
-
-This is a unitless metric that indicates the percentage of available worker time being used by all plugins in a given pipeline (`duration` / (`uptime` * `pipeline.workers`).
-It is useful for determining whether the pipeline has consistently-idle resources or is under resource contention.
-
-A _pipeline_ is considered "saturated" when its `worker_utilization` flow metric approaches 100, because it indicates that all of its workers are being kept busy.
-This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline.
-Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs.
-
-A _pipeline_ is considered "starved" when its `worker_utilization` flow metric approaches 0, because it indicates that none of its workers are being kept busy.
-This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy.
-Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines.
-
 | queue_backpressure |
 This is a unitless metric representing the cumulative time spent by all inputs blocked pushing events into their pipeline's queue, relative to wall-clock time (`queue_push_duration_in_millis` / millisecond).
 It is typically most useful when looking at the stats for an <<pipeline-stats,individual pipeline>>.
@@ -1002,6 +985,18 @@ A positive number indicates that the queue size-on-disk is growing, and a negati
 
 NOTE: The size of a PQ on disk includes both unacknowledged events and previously-acknowledged events from pages that contain one or more unprocessed events.
       This means it grows gradually as individual events are added, but shrinks in large chunks each time a whole page of processed events is reclaimed (read more: <<garbage-collection, PQ disk garbage collection>>).
+
+| worker_utilization |
+This is a unitless metric that indicates the percentage of available worker time being used by this individual plugin (`duration` / (`uptime` * `pipeline.workers`).
+It is useful for identifying which plugins in a pipeline are using the available worker resources.
+
+A _pipeline_ is considered "saturated" when `worker_utilization` approaches 100, because it indicates that all of its workers are being kept busy.
+This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline.
+Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs.
+
+A _pipeline_ is considered "starved" when `worker_utilization` approaches 0, because it indicates that none of its workers are being kept busy.
+This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy.
+Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines.
 |===
 
 [discrete]

docs/static/releasenotes.asciidoc

@@ -3,6 +3,7 @@
 
 This section summarizes the changes in the following releases:
 
+* <<logstash-8-15-0,Logstash 8.15.0>>
 * <<logstash-8-14-3,Logstash 8.14.3>>
 * <<logstash-8-14-2,Logstash 8.14.2>>
 * <<logstash-8-14-1,Logstash 8.14.1>>
@@ -64,6 +65,128 @@ This section summarizes the changes in the following releases:
 * <<logstash-8-0-0-alpha1,Logstash 8.0.0-alpha1>>
 
 
+[[logstash-8-15-0]]
+=== Logstash 8.15.0 Release Notes
+
+[[snmp-ga-8.15.0]]
+==== Announcing the new {ls} SNMP integration plugin
+
+The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default.
+This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later.
+
+The `logstash-integration-snmp` plugin combines the 
+`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses
+the capabilities of both. 
+This integrated plugin package provides better alignment in snmp processing, better resource management, 
+easier package maintenance, and a smaller installation footprint. 
+
+[[breaking-8.15.0]]
+===== Breaking change with release of `logstash-integration-snmp`
+
+Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. 
+When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new integration.
+
+IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`.
+If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior].
+
+[[featured-8.15.0]]
+==== New features and enhancements
+
+[[wolfi-8.15.0]]
+===== Wolfi-flavored Docker Image
+Starting with 8.15.0, {ls} provides an extra Docker image flavor based on the Wolfi Linux distribution: https://github.com/wolfi-dev. (https://github.com/elastic/logstash/pull/16189[#16189]). 
+To use the image append "-wolfi" to the image name:
+
+```
+docker run docker.elastic.co/logstash/logstash-wolfi:8.15.0
+```
+
+This new image flavor builds on top of a smaller and more secure base image, and is planned to become the default flavor in the next major version of {ls}.
+
+[[notable-8.15.0]]
+==== Performance improvements and notable issues fixed
+* Improved performance in pipeline-to-pipeline workloads by removing lock contention in event serialization https://github.com/elastic/logstash/pull/16194[#16194]
+* Fixed pqcheck handling of paths with spaces on Windows https://github.com/elastic/logstash/pull/16205[#16205]
+
+[[core-8.15.0]]
+==== Other changes to Logstash core
+
+* Changed rollover strategy in log4j2.properties to delete compressed files older than 7 days https://github.com/elastic/logstash/pull/16179[#16179]
+* Fixed handling and reporting of compilation error when a pipeline didn't fit the JVM's ThreadStackSize https://github.com/elastic/logstash/pull/16336[#16336]
+* Fixed missing deprecation logging in deprecated Logstash core settings https://github.com/elastic/logstash/pull/16348[#16348]
+* Fixed innacurate accounting of `queue_size_in_bytes` metric when using `clean_consumed` option in dead_letter_queue input plugin https://github.com/elastic/logstash/pull/16195[#16195]
+
+[[dependencies-8.15.0]]
+==== Updates to dependencies
+
+* Updated bundled JDK to 21 https://github.com/elastic/logstash/pull/16055[#16055]
+* Updated JRuby to 9.4.8.0 https://github.com/elastic/logstash/pull/16278[#16278]
+
+[[docs-8.15.0]]
+==== Documentation enhancements
+
+* Logstash {logstash-ref}/monitoring-with-ea.html[monitoring doc] improvements https://github.com/elastic/logstash/pull/16208[#16208] 
+* Add ecs and datastream requirement for {logstash-ref}/ea-integrations.html#es-tips[integration filter] https://github.com/elastic/logstash/pull/16268[#16268] 
+* Remove reference to puppet {ls} module https://github.com/elastic/logstash/pull/12356[#12356]
+* Add section to describe intended usage of {logstash-ref}/jvm-settings.html#reducing-off-heap-usage[`pipeline.buffer.type`] https://github.com/elastic/logstash/pull/16083[#16083] 
+* Reposition {logstash-ref}/node-stats-api.html#pipeline-stats[`worker-utilization`] stat for better placement and flow https://github.com/elastic/logstash/pull/16337[#16337]
+* Add {logstash-ref}/performance-troubleshooting.html[tuning guidance] based on Flow metrics https://github.com/elastic/logstash/pull/16289[#16289]
+
+
+[[plugins-8.15.0]]
+==== Plugin releases
+
+*De_dot Filter - 1.1.0*
+
+* Add support for recursively searching sub-fields with the new `recusive =>` config option https://github.com/logstash-plugins/logstash-filter-de_dot/pull/24[#24]
+
+*Elastic_integration Filter - 0.1.13*
+
+* Update default elasticsearch tree branch to 8.15 https://github.com/elastic/logstash-filter-elastic_integration/pull/156[#156]
+
+*Geoip Filter - 7.3.0*
+
+* Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases https://github.com/logstash-plugins/logstash-filter-geoip/pull/223[#223]
+* Updated MaxMind dependencies.
+
+*Http Filter - 1.6.0*
+
+* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-filter-http/pull/52[#52]
+
+*Http_poller Input - 5.6.0*
+
+* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-input-http_poller/pull/146[#146]
+
+*Aws Integration - 7.1.7*
+
+* Remove empty temporary dirs at plugin close https://github.com/logstash-plugins/logstash-integration-aws/pull/46[#46]
+
+*Kafka Integration - 11.5.0*
+
+* Add "auto_create_topics" option to allow disabling of topic auto creation https://github.com/logstash-plugins/logstash-integration-kafka/pull/172[#172]
+
+*Snmp Integration - 4.0.4*
+
+* Introduce Snmp Integration by default, replacing Snmp Input v1.3.3 and Snmptrap Input v3.1.0 https://github.com/logstash-plugins/logstash-integration-snmp/pull/67[#67]
+
+*Http Output - 5.7.0*
+
+* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-output-http/pull/144[#144]
+
+*Redis Output - 5.2.0*
+
+* Added support to SSL/TLS configurations https://github.com/logstash-plugins/logstash-output-redis/pull/69[#69]
+  * `ssl_enabled` 
+  * `ssl_certificate_authorities`
+  * `ssl_certificate`
+  * `ssl_key`
+  * `ssl_verification_mode`
+  * `ssl_supported_protocols`
+  * `ssl_cipher_suites`
+* Added basic support for SSL https://github.com/logstash-plugins/logstash-output-redis/pull/59[#59]
+* Fixed documentation of required settings https://github.com/logstash-plugins/logstash-output-redis/pull/61[#61]
+
+
 [[logstash-8-14-3]]
 === Logstash 8.14.3 Release Notes
 
@@ -237,7 +360,7 @@ No user-facing changes in Logstash core.
 [[logstash-8-13-1]]
 === Logstash 8.13.1 Release Notes
 
-No user-facing changes in Logstash core.
+* Inject variable reference instead of variable value in `logstash.yaml` to ensure keystore values take precedence over Environment Variables https://github.com/elastic/logstash/pull/16037[#16037]
 
 [[logstash-8-13-0]]
 === Logstash 8.13.0 Release Notes
@@ -2318,4 +2441,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit
 This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437]
 
 ==== Plugin releases
-Plugins align with release 7.14.0
+Plugins align with release 7.14.0

logstash-core/lib/logstash/config/mixin.rb

@@ -93,7 +93,7 @@ def config_init(params)
 
     # Resolve environment variables references
     params.each do |name, value|
-      params[name.to_s] = deep_replace(value)
+      params[name.to_s] = deep_replace(value, true)
     end
 
     # Intercept codecs that have not been instantiated

logstash-core/lib/logstash/java_pipeline.rb

@@ -603,6 +603,11 @@ def init_worker_loop
         "Worker loop initialization error",
         default_logging_keys(:error => e.message, :exception => e.class, :stacktrace => e.backtrace.join("\n")))
       nil
+    rescue Java::java.lang.StackOverflowError => se
+      @logger.error(
+        "Stack overflow error while compiling Pipeline. Please increase thread stack size using -Xss",
+        default_logging_keys())
+      nil
     end
   end
 

logstash-core/lib/logstash/runner.rb

@@ -321,6 +321,9 @@ def execute
     if JavaVersion::CURRENT < JavaVersion::JAVA_11
       logger.warn I18n.t("logstash.runner.java.version",
                                              :java_home => java.lang.System.getProperty("java.home"))
+    elsif JavaVersion::CURRENT < JavaVersion::JAVA_17
+      deprecation_logger.deprecated I18n.t("logstash.runner.java.version_17_minimum",
+                                           :java_home => java.lang.System.getProperty("java.home"))
     end
 
     logger.warn I18n.t("logstash.runner.java.home") if ENV["JAVA_HOME"]

logstash-core/lib/logstash/settings.rb

@@ -187,7 +187,7 @@ def reset
 
     def from_yaml(yaml_path, file_name = "logstash.yml")
       settings = read_yaml(::File.join(yaml_path, file_name))
-      self.merge(deep_replace(flatten_hash(settings)), true)
+      self.merge(deep_replace(flatten_hash(settings), true), true)
       self
     end
 
@@ -197,6 +197,13 @@ def post_process
           callback.call(self)
         end
       end
+
+      # because we cannot rely on deprecation logger being wired up when the setters
+      # are initially used, we re-emit setter-related deprecations after all post-processing
+      # hooks have been activated (and therefore after logging has been configured)
+      @settings.each_value do |setting|
+        setting.observe_post_process if setting.respond_to?(:observe_post_process)
+      end
     end
 
     def on_post_process(&block)
@@ -839,9 +846,7 @@ def initialize(canonical_proxy, alias_name)
       end
 
       def set(value)
-        deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set",
-                                             :deprecated_alias => name,
-                                             :canonical_name => canonical_proxy.name))
+        do_log_setter_deprecation
         super
       end
 
@@ -856,6 +861,18 @@ def validate_value
         # bypass deprecation warning
         wrapped.validate_value if set?
       end
+
+      def observe_post_process
+        do_log_setter_deprecation if set?
+      end
+
+      private
+
+      def do_log_setter_deprecation
+        deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set",
+                                             :deprecated_alias => name,
+                                             :canonical_name => canonical_proxy.name))
+      end
     end
 
     ##