From 83ae3f630a321e7273fbde7a82d20835369b1137 Mon Sep 17 00:00:00 2001 From: jonathannewman Date: Wed, 22 May 2024 14:13:21 -0700 Subject: [PATCH] (PE-38421) avoid stack overflow in event collection See https://stuartsierra.com/2015/04/26/clojure-donts-concat As was seen in the field, the event concatenation can cause a stack overflow for a large number of events. This modifies the loop for retrieving events to use `into` instead of `concat` which prevents the stack overflow by collecting the results into a new vector with every iteration. --- .../services/watcher/filesystem_watch_core.clj | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/clj/puppetlabs/trapperkeeper/services/watcher/filesystem_watch_core.clj b/src/clj/puppetlabs/trapperkeeper/services/watcher/filesystem_watch_core.clj index 6f05d82..c6fb60f 100644 --- a/src/clj/puppetlabs/trapperkeeper/services/watcher/filesystem_watch_core.clj +++ b/src/clj/puppetlabs/trapperkeeper/services/watcher/filesystem_watch_core.clj @@ -119,14 +119,16 @@ (map #(clojurize % (.watchable watch-key)) events))) (schema/defn retrieve-events :- [Event] - "Blocks until an event the watcher is concerned with has occured. Will then + "Blocks until an event the watcher is concerned with has occurred. Will then poll for a new event, waiting at least `window-min` for a new event to occur. Will continue polling for as long as there are new events that occur within `window-min`, or the `window-max` time limit has been exceeded." [watcher :- (schema/protocol Watcher)] (let [^WatchService watch-service (:watch-service watcher) watch-key (.take watch-service) - initial-events (watch-key->events watch-key) + ;; use `vec` to ensure the events sequence is correctly concatenated below. + ;; Without it, the events will be out of order. + initial-events (vec (watch-key->events watch-key)) time-limit (+ (System/currentTimeMillis) window-max) recursive @(:recursive watcher)] (when recursive @@ -140,14 +142,14 @@ (watch-new-directories! waiting-events watcher)) (.reset waiting-key) (if (< (System/currentTimeMillis) time-limit) - (recur (concat events waiting-events)) - (concat events waiting-events))) + (recur (into events waiting-events)) + (into events waiting-events))) events)) initial-events))) (schema/defn process-events! - "Process for side-effects any events that occured for watcher's watch-key" + "Process for side effects any events that occurred for watcher's watch-key" [watcher :- (schema/protocol Watcher) events :- [Event]] (let [callbacks @(:callbacks watcher)