Merge branch 'prep-0.15.0-rc1' into async-krill-signer-1142

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 # Note: some of these values are also used when building Debian packages below.
 name = "krill"
-version = "0.14.0-rc3"
+version = "0.14.2"
 edition = "2018"
 rust-version = "1.65"
 authors = ["NLnet Labs <[email protected]>"]

README.md

@@ -25,51 +25,48 @@ in this [blog post](https://blog.nlnetlabs.nl/testing-the-waters-with-krill/).
 
 # Changelog
 
-## 0.14.0-rc3
+## 0.14.2 'Extra, Extra, Extra!'
 
-This RC adds support for the updated ASPA v1 profile (issue #1080).
+This release fixes an additional corner case in the migration code that affects
+certain installations that archived 'surplus' commands (issue: #1147). There is
+no need to upgrade to this version if you already upgraded to 0.14.0 or 0.14.1.
 
-Earlier, we were planning to add this in a later release, but as it turns out,
-it was easier than expected to support this transition, so we included it in the coming release.
+## 0.14.1 'Extra, Extra!'
 
-Please note that the upgrade will fail if you are already using ASPA and upgraded to 0.14.0-rc1 or rc2.
+This release fixes a bug in the migration code that affects certain installations
+that archived 'surplus' data (issue: #1147). There is no need to upgrade to
+this version if you already upgraded to 0.14.0.
 
-## 0.14.0-rc2
+## 0.14.0 'ASPA'
 
-This RC fixes the following issue, also present in 0.13.x:
-- Show delete ROA button when no BGP preview is available #1139
-
-## 0.14.0-rc1
-
-This is the release candidate for the coming 0.14.0 release. We invite all
-interested users to test this version, but please do not upgrade your
-production environment until 0.14.0 has been released.
+This release adds support for the updated ASPA v1 profile (issue #1080).
+Any existing ASPA objects will be re-issued automatically.
 
-This release introduces the following small features and fixes:
+In addition, the following small features and fixes were done:
+- Show delete ROA button when no BGP preview is available #1139
 - Add traditional and simplified Chinese translations #1075
-- Let the testbed automatically renew the TA manifest and CRL #1095
+- Let the testbed automatically renew the TA manifest and CRL #1095 (see below)
 - Show the delete icon for AS0 ROA when there is another existing announcement #1109
 
-But we spent the main effort in this release on improving the way that
-Krill stores its data. This will help to improve robustness today, and
-it paves the way for introducing support for Krill clustering using
-a database back-end in a future release. For now, these issues were
-done:
+The main effort in this release was spent on less user-visible
+improvements in the way that Krill stores its data. This will
+help to improve robustness today, and it paves the way for introducing
+support for Krill clustering using a database back-end in a future release.
+
+For now, these issues have been done:
 - Improve transactionality of changes (e.g. #1076-1078, #1085, #1108, #1090)
 - Remove no longer needed 'always_recover_data' function #1086 
 - Improve upgrade failed error: tell users to downgrade #1042
 - Crash Krill if the task scheduler encounters a fatal error. #1132
-- Add support for importing delegated child CAs #1133
 
-Note that this release still uses the now outdated ASPA object syntax. We plan
-to make another focused release to address this immediately after 0.14.0 is
-released. See issue #1080.
+You can find the full list of issues here:
+https://github.com/NLnetLabs/krill/projects/25
 
-Note that if you were running 0.13.1 as a testbed, you may have symlinked
-the "signer" directory to "ta_signer" to support a manual work around for
-re-signing the trust anchor CRL and manifest (issue #1095). If you did,
-then you may need to delete any surplus files and directories under
-"/var/lib/krill/data/ta_signer" other than the directory called "ta".
+Finally, regarding issue #1095. If you were running 0.13.1 as a testbed, you
+may have symlinked the "signer" directory to "ta_signer" to support a manual
+workaround for re-signing the trust anchor CRL and manifest. If you did, you
+may need to delete any surplus files and directories under "data/ta_signer"
+other than the directory called "ta".
 
 ## 0.13.1 'Scrollbars!'
 

defaults/krill-pubd.conf

@@ -6,22 +6,15 @@
 
 # Specify the storage used by Krill.
 #
-# Currently, Krill supports two flavours of storage: disk
-# or a Postgresql database. Support for other databases or
-# key value store implementations may be added in future.
+# Currently, Krill supports disk based storage only.
 #
-# If you use disk-backed storage, you MUST ensure that
-# only one Krill node using this storage is active at any
-# given moment. Transactionality is not guaranteed across nodes
-# when using this option.
-#
-# If a database back-end is used, it is possible to have multiple
-# concurrent active Krill instances use it. Database replication
-# can further improve availability as long as this is transparent to Krill.
+# You can use a shared disk and a standby Krill node for redundancy.
+# However, data corruption issues can occur if you run multiple active
+# Krill nodes using a shared disk. Therefore, you MUST ensure that only
+# one Krill node is active at any given time.
 #
 # Example configurations:
 # storage_uri = "/var/lib/krill/data/"
-# storage_uri = "postgres://postgres@localhost/postgres" 
 #
 # Default:
 # storage_uri = "./data"
@@ -258,4 +251,4 @@
 # audit or research.
 #
 # If set to true files will be archived in a directory under '$data_dir/repo/archive'
-# rrdp_files_archive = false
+# rrdp_files_archive = false

defaults/krill-testbed.conf

@@ -36,22 +36,15 @@
 
 # Specify the storage used by Krill.
 #
-# Currently, Krill supports two flavours of storage: disk
-# or a Postgresql database. Support for other databases or
-# key value store implementations may be added in future.
+# Currently, Krill supports disk based storage only.
 #
-# If you use disk-backed storage, you MUST ensure that
-# only one Krill node using this storage is active at any
-# given moment. Transactionality is not guaranteed across nodes
-# when using this option.
-#
-# If a database back-end is used, it is possible to have multiple
-# concurrent active Krill instances use it. Database replication
-# can further improve availability as long as this is transparent to Krill.
+# You can use a shared disk and a standby Krill node for redundancy.
+# However, data corruption issues can occur if you run multiple active
+# Krill nodes using a shared disk. Therefore, you MUST ensure that only
+# one Krill node is active at any given time.
 #
 # Example configurations:
 # storage_uri = "/var/lib/krill/data/"
-# storage_uri = "postgres://postgres@localhost/postgres" 
 #
 # Default:
 # storage_uri = "./data"

defaults/krill.conf

@@ -6,22 +6,15 @@
 
 # Specify the storage used by Krill.
 #
-# Currently, Krill supports two flavours of storage: disk
-# or a Postgresql database. Support for other databases or
-# key value store implementations may be added in future.
+# Currently, Krill supports disk based storage only.
 #
-# If you use disk-backed storage, you MUST ensure that
-# only one Krill node using this storage is active at any
-# given moment. Transactionality is not guaranteed across nodes
-# when using this option.
-#
-# If a database back-end is used, it is possible to have multiple
-# concurrent active Krill instances use it. Database replication
-# can further improve availability as long as this is transparent to Krill.
+# You can use a shared disk and a standby Krill node for redundancy.
+# However, data corruption issues can occur if you run multiple active
+# Krill nodes using a shared disk. Therefore, you MUST ensure that only
+# one Krill node is active at any given time.
 #
 # Example configurations:
 # storage_uri = "/var/lib/krill/data/"
-# storage_uri = "postgres://postgres@localhost/postgres" 
 #
 # Default:
 # storage_uri = "./data"
@@ -466,4 +459,4 @@
 # timing_child_certificate_valid_weeks = 52
 # timing_child_certificate_reissue_weeks_before = 4
 # timing_roa_valid_weeks = 52
-# timing_roa_reissue_weeks_before = 4
+# timing_roa_reissue_weeks_before = 4

src/commons/eventsourcing/kv.rs

@@ -131,9 +131,12 @@ impl KeyValueStore {
     /// If matching is not empty then the key must contain the given `&str`.
     pub fn keys(&self, scope: &Scope, matching: &str) -> Result<Vec<Key>, KeyValueError> {
         self.execute(scope, |kv| {
+            // kvx list_keys returns keys in sub-scopes
             kv.list_keys(scope).map(|keys| {
                 keys.into_iter()
-                    .filter(|key| matching.is_empty() || key.name().as_str().contains(matching))
+                    .filter(|key| {
+                        key.scope() == scope && (matching.is_empty() || key.name().as_str().contains(matching))
+                    })
                     .collect()
             })
         })

src/upgrades/mod.rs

@@ -363,6 +363,12 @@ pub trait UpgradeAggregateStorePre0_14 {
 
         // Migrate the event sourced data for each scope and create new snapshots
         for scope in self.deployed_store().scopes()? {
+            // We only need top-level scopes, not sub-scopes such as 'surplus' archive dirs
+            if scope.len() != 1 {
+                trace!("Skipping migration for sub-scope: {}", scope);
+                continue;
+            }
+
             // Getting the Handle should never fail, but if it does then we should bail out asap.
             let handle = MyHandle::from_str(&scope.to_string())
                 .map_err(|_| UpgradeError::Custom(format!("Found invalid handle '{}'", scope)))?;
@@ -508,6 +514,10 @@ pub trait UpgradeAggregateStorePre0_14 {
             UpgradeMode::PrepareToFinalise => {
                 let mut aspa_configs = AspaMigrationConfigs::default();
                 for scope in self.deployed_store().scopes()? {
+                    if scope.len() != 1 {
+                        continue;
+                    }
+
                     // Getting the Handle should never fail, but if it does then we should bail out asap.
                     let ca = MyHandle::from_str(&scope.to_string())
                         .map_err(|_| UpgradeError::Custom(format!("Found invalid handle '{}'", scope)))?;

test-resources/krill-init-multi-user.conf

@@ -6,22 +6,15 @@
 
 # Specify the storage used by Krill.
 #
-# Currently, Krill supports two flavours of storage: disk
-# or a Postgresql database. Support for other databases or
-# key value store implementations may be added in future.
+# Currently, Krill supports disk based storage only.
 #
-# If you use disk-backed storage, you MUST ensure that
-# only one Krill node using this storage is active at any
-# given moment. Transactionality is not guaranteed across nodes
-# when using this option.
-#
-# If a database back-end is used, it is possible to have multiple
-# concurrent active Krill instances use it. Database replication
-# can further improve availability as long as this is transparent to Krill.
+# You can use a shared disk and a standby Krill node for redundancy.
+# However, data corruption issues can occur if you run multiple active
+# Krill nodes using a shared disk. Therefore, you MUST ensure that only
+# one Krill node is active at any given time.
 #
 # Example configurations:
 # storage_uri = "/var/lib/krill/data/"
-# storage_uri = "postgres://postgres@localhost/postgres" 
 #
 # Default:
 # storage_uri = "./data"
@@ -469,6 +462,7 @@ service_uri = "https://localhost:3001/"
 # timing_roa_reissue_weeks_before = 4
 
 
+
 ######################################################################################
 #                                                                                    #
 #               ----==== WEB UI MULTI-USER LOGIN CONFIGURATION ====----              #

test-resources/krill-init.conf

@@ -6,22 +6,15 @@
 
 # Specify the storage used by Krill.
 #
-# Currently, Krill supports two flavours of storage: disk
-# or a Postgresql database. Support for other databases or
-# key value store implementations may be added in future.
+# Currently, Krill supports disk based storage only.
 #
-# If you use disk-backed storage, you MUST ensure that
-# only one Krill node using this storage is active at any
-# given moment. Transactionality is not guaranteed across nodes
-# when using this option.
-#
-# If a database back-end is used, it is possible to have multiple
-# concurrent active Krill instances use it. Database replication
-# can further improve availability as long as this is transparent to Krill.
+# You can use a shared disk and a standby Krill node for redundancy.
+# However, data corruption issues can occur if you run multiple active
+# Krill nodes using a shared disk. Therefore, you MUST ensure that only
+# one Krill node is active at any given time.
 #
 # Example configurations:
 # storage_uri = "/var/lib/krill/data/"
-# storage_uri = "postgres://postgres@localhost/postgres" 
 #
 # Default:
 # storage_uri = "./data"
@@ -466,4 +459,4 @@ service_uri = "https://localhost:3001/"
 # timing_child_certificate_valid_weeks = 52
 # timing_child_certificate_reissue_weeks_before = 4
 # timing_roa_valid_weeks = 52
-# timing_roa_reissue_weeks_before = 4
+# timing_roa_reissue_weeks_before = 4

test-resources/migrations/v0_10_3/cas/testbed-client/surplus/delta-0.json

@@ -0,0 +1,13 @@
+{
+  "id": "testbed-client",
+  "version": 0,
+  "details": {
+    "id": {
+      "cert": {
+        "public_key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsPRviOjJtpjDicev6VqRPAFcf0jRhhZgecjKheNkab2rjL7LQUB34QfiC9ICQ30wRx77K1p+pszFrfKESoyMY+Z323G7+RY3pBwHEQyvaTaPAqCGCCsnunffCsMgRjQkS1c2Oft9U3MwDu88aweEptBmFpfdO5e1HRYlQPwSUZkS2ZgXoggOZ3+ePFYczot+Je3GuLpaytiV7lD4n15veMrUPSm8OUBwfOrh7xgRHAFI5AVQQLw99oMOkS+EqyLfXy/dS3Y35n2lUvjYyAwY3mrwl+epRq34ugP/71Nn0BJ/ug3Apzt1w113iFv42tUQXnLfWajPTKry+s3rdWCVRwIDAQAB",
+        "base64": "MIIDNDCCAhygAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyg5OEEwNUQ2Mzc3MENFQ0FCRjRCNDk5QURBODgyRjgyNTYwNTdGREY3MB4XDTIzMTAxOTA4MDYyNloXDTM4MTAxOTA4MTEyNlowMzExMC8GA1UEAxMoOThBMDVENjM3NzBDRUNBQkY0QjQ5OUFEQTg4MkY4MjU2MDU3RkRGNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALD0b4joybaYw4nHr+lakTwBXH9I0YYWYHnIyoXjZGm9q4y+y0FAd+EH4gvSAkN9MEce+ytafqbMxa3yhEqMjGPmd9txu/kWN6QcBxEMr2k2jwKghggrJ7p33wrDIEY0JEtXNjn7fVNzMA7vPGsHhKbQZhaX3TuXtR0WJUD8ElGZEtmYF6IIDmd/njxWHM6LfiXtxri6WsrYle5Q+J9eb3jK1D0pvDlAcHzq4e8YERwBSOQFUEC8PfaDDpEvhKsi318v3Ut2N+Z9pVL42MgMGN5q8JfnqUat+LoD/+9TZ9ASf7oNwKc7dcNdd4hb+NrVEF5y31moz0yq8vrN63VglUcCAwEAAaNTMFEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmKBdY3cM7Kv0tJmtqIL4JWBX/fcwHwYDVR0jBBgwFoAUmKBdY3cM7Kv0tJmtqIL4JWBX/fcwDQYJKoZIhvcNAQELBQADggEBAKAcxeZbckWD4MVKjHDrgGyiDyDqRfDotPt86D2tacf/kxv9zvPg0/fLKcRfobJqHB1OuCDQWYxXuARFOkwNr+R/AdhmoFbL7DeqT54VL4UJFFic5HwHzo3FXFrUgjKlh7vnQMiHqDjTcjS7V2tdzb/6uIOMR75A41VGUQ3PGhEbhPf3AQ7pUgi/T9cs0EO2Eo6g1IkD1izhgV3QKz4S1fozWYNqm+YzxqGU/b4s2IVCCE3pmIDTSu8qKvRYVepDJtaavBVtf1A4VTKo0YFPPw8dLeE0dabmK1y+ftlktgraPsQ3B8WbOf7BD+DRpkzj7TEsSSxhPuBIc8AUS1qsiKA=",
+        "hash": "8db69f3830f42e89fbe8615fabd3501ddaa15d74df831a74d773c11716943e3a"
+      }
+    }
+  }
+}

test-resources/migrations/v0_11_0/cas/testbed-client/surplus/delta-0.json

@@ -0,0 +1,13 @@
+{
+  "id": "testbed-client",
+  "version": 0,
+  "details": {
+    "id": {
+      "cert": {
+        "public_key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsPRviOjJtpjDicev6VqRPAFcf0jRhhZgecjKheNkab2rjL7LQUB34QfiC9ICQ30wRx77K1p+pszFrfKESoyMY+Z323G7+RY3pBwHEQyvaTaPAqCGCCsnunffCsMgRjQkS1c2Oft9U3MwDu88aweEptBmFpfdO5e1HRYlQPwSUZkS2ZgXoggOZ3+ePFYczot+Je3GuLpaytiV7lD4n15veMrUPSm8OUBwfOrh7xgRHAFI5AVQQLw99oMOkS+EqyLfXy/dS3Y35n2lUvjYyAwY3mrwl+epRq34ugP/71Nn0BJ/ug3Apzt1w113iFv42tUQXnLfWajPTKry+s3rdWCVRwIDAQAB",
+        "base64": "MIIDNDCCAhygAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyg5OEEwNUQ2Mzc3MENFQ0FCRjRCNDk5QURBODgyRjgyNTYwNTdGREY3MB4XDTIzMTAxOTA4MDYyNloXDTM4MTAxOTA4MTEyNlowMzExMC8GA1UEAxMoOThBMDVENjM3NzBDRUNBQkY0QjQ5OUFEQTg4MkY4MjU2MDU3RkRGNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALD0b4joybaYw4nHr+lakTwBXH9I0YYWYHnIyoXjZGm9q4y+y0FAd+EH4gvSAkN9MEce+ytafqbMxa3yhEqMjGPmd9txu/kWN6QcBxEMr2k2jwKghggrJ7p33wrDIEY0JEtXNjn7fVNzMA7vPGsHhKbQZhaX3TuXtR0WJUD8ElGZEtmYF6IIDmd/njxWHM6LfiXtxri6WsrYle5Q+J9eb3jK1D0pvDlAcHzq4e8YERwBSOQFUEC8PfaDDpEvhKsi318v3Ut2N+Z9pVL42MgMGN5q8JfnqUat+LoD/+9TZ9ASf7oNwKc7dcNdd4hb+NrVEF5y31moz0yq8vrN63VglUcCAwEAAaNTMFEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUmKBdY3cM7Kv0tJmtqIL4JWBX/fcwHwYDVR0jBBgwFoAUmKBdY3cM7Kv0tJmtqIL4JWBX/fcwDQYJKoZIhvcNAQELBQADggEBAKAcxeZbckWD4MVKjHDrgGyiDyDqRfDotPt86D2tacf/kxv9zvPg0/fLKcRfobJqHB1OuCDQWYxXuARFOkwNr+R/AdhmoFbL7DeqT54VL4UJFFic5HwHzo3FXFrUgjKlh7vnQMiHqDjTcjS7V2tdzb/6uIOMR75A41VGUQ3PGhEbhPf3AQ7pUgi/T9cs0EO2Eo6g1IkD1izhgV3QKz4S1fozWYNqm+YzxqGU/b4s2IVCCE3pmIDTSu8qKvRYVepDJtaavBVtf1A4VTKo0YFPPw8dLeE0dabmK1y+ftlktgraPsQ3B8WbOf7BD+DRpkzj7TEsSSxhPuBIc8AUS1qsiKA=",
+        "hash": "8db69f3830f42e89fbe8615fabd3501ddaa15d74df831a74d773c11716943e3a"
+      }
+    }
+  }
+}

test-resources/migrations/v0_12_1_pubserver/pubd/0/surplus/delta-0.json

@@ -0,0 +1,13 @@
+{
+  "id": "0",
+  "version": 0,
+  "details": {
+    "id_cert": {
+      "public_key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy3jbLo+08KFwmqikIJBmSX9qYarI/CEXVBshKWfJ6EpWOc9+nghM7XzfVsDgHu+C0pipoUzFusShwHDgEe/tvSHOYhV9+tnS8Vh3mQ4MpD/mUr0LCcFhG8EI8G7YrAhZYLWygcd2I+kz905WhfGYqTjppJyW8Vo8CfYIatdTdLHTjKmVZww4H709fgQuauC20rElgdcw1XPz1c0ANYL++CK9lOJzmHjo88xkifRwoguQp3BTHTkFw3NWDHYYDO/CynKHJiJP0RcGyBho0OXQQ1Zm7s0tuHEuPdZdE3F38AhaamwMOoGIirufoM6qJ5se7/eRqmCtvty/UvUzLCFEdQIDAQAB",
+      "base64": "MIIDNDCCAhygAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyg2QThBOUFEMTA5MjdGOUUzRTMyRDg3MjczQTg1NUIzRDBDMzRDNjFGMB4XDTIyMDIyNTEyMjU1NloXDTM3MDIyNTEyMzA1NlowMzExMC8GA1UEAxMoNkE4QTlBRDEwOTI3RjlFM0UzMkQ4NzI3M0E4NTVCM0QwQzM0QzYxRjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMt42y6PtPChcJqopCCQZkl/amGqyPwhF1QbISlnyehKVjnPfp4ITO1831bA4B7vgtKYqaFMxbrEocBw4BHv7b0hzmIVffrZ0vFYd5kODKQ/5lK9CwnBYRvBCPBu2KwIWWC1soHHdiPpM/dOVoXxmKk46aSclvFaPAn2CGrXU3Sx04yplWcMOB+9PX4ELmrgttKxJYHXMNVz89XNADWC/vgivZTic5h46PPMZIn0cKILkKdwUx05BcNzVgx2GAzvwspyhyYiT9EXBsgYaNDl0ENWZu7NLbhxLj3WXRNxd/AIWmpsDDqBiIq7n6DOqiebHu/3kapgrb7cv1L1MywhRHUCAwEAAaNTMFEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaoqa0Qkn+ePjLYcnOoVbPQw0xh8wHwYDVR0jBBgwFoAUaoqa0Qkn+ePjLYcnOoVbPQw0xh8wDQYJKoZIhvcNAQELBQADggEBAIH8R3/14PK1qy1D7HOqN5FbYEO2brwSPKb0UdqNC1o8zLLJ5qGNCWoAvck+pkAJsxOPxgvzvYlwlUzuCToiXFUnDBYPXzI6u8kziJFchRIOFmaQxL+1LKhV66hgXi7r85+3nB0UMrfARzgUjXcZ9WIIQOwtSYq7yTcMHf8na7GJRelvH4B1lNht5AW6fQeRPBHleZZ3DQQc18mX2cPdm/uoegnTlTpM9dRhFTcZeezwkZvEkYG/i9rSMxVmFd1E4gN2e0CPsxdJxZ9oMNubyNToW146Y7v+M5OTibfrIqQKvsBC3BuTZRbXB9093HVmJTff33/zFYexfM3X740MhOo=",
+      "hash": "595d4b3832527e8e65198b36b9e837e630060ee9fd73220ec3fcf9a5bc423068"
+    },
+    "rrdp_base_uri": "https://testbed.krill.cloud/rrdp/",
+    "rsync_jail": "rsync://testbed.krill.cloud/repo/"
+  }
+}