(PE-36952) ensure infra-serials are up to date

This updates the code that initializes and updates the infra-serial
file to ensure that the content contains all possible valid matches
for a given certname. Expired certs are removed as well.

Some extra checks were added to prevent trying to read files when
they don't exist.

src/clj/puppetlabs/puppetserver/certificate_authority.clj

@@ -523,8 +523,6 @@
             {}
             auth-exts)))
 
-(defn seq-contains? [coll target] (some #(= target %) coll))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Writing various SSL objects safely
 ;; These versions all encode writing atomically and knowledge of file permissions
@@ -658,13 +656,21 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Inventory File
 
+(def inventory-date-formatter
+  (time-format/formatter "YYY-MM-dd'T'HH:mm:ssz"))
+
 (schema/defn format-date-time :- schema/Str
   "Formats a date-time into the format expected by the ruby puppet code."
   [date-time :- Date]
   (time-format/unparse
-    (time-format/formatter "YYY-MM-dd'T'HH:mm:ssz")
+    inventory-date-formatter
     (time-coerce/from-date date-time)))
 
+(schema/defn parse-date-time :- DateTime
+  "parses a date-time string into a DateTime instance"
+  [date-time :- schema/Str]
+  (time-format/parse inventory-date-formatter date-time))
+
 (def buffer-copy-size (* 64 1024))
 
 (schema/defn ^:always-validate
@@ -726,6 +732,13 @@
    (if (some? row-subject)
      (= (subs row-subject 1) cn-subject)
      false))
+(schema/defn is-not-expired? :- schema/Bool
+  [now :- DateTime
+   [_serial _not-before not-after _row-subject] :- [schema/Str]]
+  (if-let [not-after-date (parse-date-time not-after)]
+    (time/before? now not-after-date)
+    ;; lack of an end date means we can't tell if it is expired or not, so assume it isn't.
+    false))
 
 (defn extract-inventory-row-contents
   [row]
@@ -736,26 +749,36 @@
    certname :- schema/Str]
   (common/with-safe-read-lock inventory-lock inventory-lock-descriptor inventory-lock-timeout-seconds
     (log/trace (i18n/trs "Looking for \"{0}\" in inventory file {1}" certname cert-inventory))
-    (with-open [inventory-reader (io/reader cert-inventory)]
-      (let [inventory-rows (map extract-inventory-row-contents (line-seq inventory-reader))
-            cn-subject (utils/cn certname)]
-        (some? (some (partial is-subject-in-inventory-row? cn-subject) inventory-rows))))))
+    (if (fs/exists? cert-inventory)
+      (with-open [inventory-reader (io/reader cert-inventory)]
+        (let [inventory-rows (map extract-inventory-row-contents (line-seq inventory-reader))
+             cn-subject (utils/cn certname)]
+          (some? (some (partial is-subject-in-inventory-row? cn-subject) inventory-rows))))
+      (do
+        (log/debug "Unable to find inventory file {0}" cert-inventory)
+        false))))
 
 (schema/defn find-matching-valid-serial-numbers :- [BigInteger]
   [{:keys [cert-inventory inventory-lock inventory-lock-timeout-seconds]} :- CaSettings
    certname :- schema/Str]
   (common/with-safe-read-lock inventory-lock inventory-lock-descriptor inventory-lock-timeout-seconds
     (log/trace (i18n/trs "Looking for serial numbers for \"{0}\" in inventory file {1}" certname cert-inventory))
-    (with-open [inventory-reader (io/reader cert-inventory)]
-      (let [inventory-rows (map extract-inventory-row-contents (line-seq inventory-reader))
-            cn-subject (utils/cn certname)]
-        (doall
-          (->> inventory-rows
-               (filter (partial is-subject-in-inventory-row? cn-subject))
-               ;; serial number is first entry in the array
-               (map first)
-               ;; assume serials are base 16 strings, drop the `0x` as BigInteger won't deal with them.
-               (map #(BigInteger. ^String (subs % 2) 16))))))))
+    (if (fs/exists? cert-inventory)
+      (with-open [inventory-reader (io/reader cert-inventory)]
+        (let [inventory-rows (map extract-inventory-row-contents (line-seq inventory-reader))
+              cn-subject (utils/cn certname)
+              now (time/now)]
+          (doall
+            (->> inventory-rows
+                 (filter (partial is-subject-in-inventory-row? cn-subject))
+                 (filter (partial is-not-expired? now))
+                 ;; serial number is first entry in the array
+                 (map first)
+                 ;; assume serials are base 16 strings, drop the `0x` as BigInteger won't deal with them.
+                 (map #(BigInteger. ^String (subs % 2) 16))))))
+      (do
+        (log/debug (i18n/trs "Unable to find inventory file {0}" cert-inventory))
+        []))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Initialization
@@ -823,36 +846,49 @@
     [infra-file-reader :- Reader]
     (line-seq infra-file-reader))
 
-(defn- write-infra-serials-to-writer
-  [writer infra-nodes-path signeddir]
-  (try
+(schema/defn extract-active-infra-serials :- [BigInteger]
+  "Read the infra nodes file to determine which nodes are infrastructure nodes.
+  For each node, check the inventory file for any serial numbers for that node,
+  Also check the filesystem for a signed cert for that node.  Return a sorted unique
+  set of serial numbers for nodes in the infra file"
+  [{:keys [infra-nodes-path signeddir] :as settings} :- CaSettings]
+  (if (fs/exists? infra-nodes-path)
     (with-open [infra-nodes-reader (io/reader infra-nodes-path)]
       (let [infra-nodes (read-infra-nodes infra-nodes-reader)]
-        (doseq [infra-node infra-nodes]
-          (try
-            (let [infra-serial (-> (path-to-cert signeddir infra-node)
-                                   (utils/pem->cert)
-                                   (utils/get-serial))]
-              (.write writer (str infra-serial))
-              (.newLine writer))
-            (catch FileNotFoundException _
-              (log/warn
-               (i18n/trs
-                (str
-                 "Failed to find/load certificate for Puppet Infrastructure Node:"
-                 infra-node))))))))
-    (catch FileNotFoundException _
-      (log/warn (i18n/trs (str infra-nodes-path " does not exist"))))))
-
-(schema/defn generate-infra-serials
+        (loop [local-infra-nodes infra-nodes
+               result []]
+          (if-let [infra-node (first local-infra-nodes)]
+            (let [augmented-results (concat result (find-matching-valid-serial-numbers settings infra-node))
+                  cert-path (path-to-cert signeddir infra-node)]
+              (if (fs/exists? cert-path)
+                (let [serial (-> cert-path
+                                 utils/pem->cert
+                                 utils/get-serial)]
+                  (recur (rest local-infra-nodes)
+                         (conj augmented-results serial)))
+                (do
+                  (log/warn (i18n/trs "Failed to find certificate for Puppet Infrastructure Node: {0}" infra-node))
+                  (recur (rest local-infra-nodes)
+                         augmented-results))))
+            ;; ensure they are unique and ordered so the end result is stable
+            (sort (set result))))))
+    []))
+
+(schema/defn write-infra-serials-to-writer
+  [writer :- BufferedWriter
+   settings :- CaSettings]
+  (let [infra-serials (extract-active-infra-serials settings)]
+    (doseq [infra-serial infra-serials]
+      (.write writer (str infra-serial))
+      (.newLine writer))))
+
+(schema/defn generate-infra-serials!
   "Given a list of infra nodes it will create a file containing
    serial numbers of their certificates (listed on separate lines).
    It is expected have at least one entry (MoM)"
-  [{:keys [infra-nodes-path infra-node-serials-path signeddir]} :- CaSettings]
+  [{:keys [infra-node-serials-path] :as settings} :- CaSettings]
   (ks-file/atomic-write infra-node-serials-path
-                        #(write-infra-serials-to-writer %
-                                                        infra-nodes-path
-                                                        signeddir)
+                        #(write-infra-serials-to-writer % settings)
                         public-key-perms))
 
 (defn symlink-cadir
@@ -888,7 +924,7 @@
   (-> ca-settings :signeddir fs/file ks/mkdirs!)
   (initialize-serial-file! ca-settings)
   (-> ca-settings :infra-nodes-path fs/file fs/create)
-  (generate-infra-serials ca-settings)
+  (generate-infra-serials! ca-settings)
   (let [keypair     (utils/generate-key-pair (:keylength ca-settings))
         public-key  (utils/get-public-key keypair)
         private-key (utils/get-private-key keypair)
@@ -1835,7 +1871,7 @@
       (do
         (log/info (i18n/trs "CA already initialized for SSL"))
         (when (:enable-infra-crl settings)
-          (generate-infra-serials settings))
+          (generate-infra-serials! settings))
         (when (:manage-internal-file-permissions settings)
           (ensure-ca-file-perms! settings)))
       (let [{found true

src/clj/puppetlabs/services/ca/certificate_authority_service.clj

@@ -79,7 +79,7 @@
                            (= (.getCanonicalPath (:changed-path %))
                               infra-nodes-file))
                      events)
-           (ca/generate-infra-serials settings))))
+           (ca/generate-infra-serials! settings))))
       (register-status
         "ca"
         (status-core/get-artifact-version "puppetlabs" "puppetserver")