From 051cd8c3c60fcecb81ece3f79b760e946dddedf1 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Thu, 29 Aug 2024 17:43:31 +0200 Subject: [PATCH 1/2] Add a pushsecrets policy and vault path for ESO syncing See the README for more details, but TLDR: you can use `secret/pushsecrets` to push secrets from any node to the vault. This secret can then be retrieved from either a different namespace or a different cluster node. Tested this with a pushsecret as follows: ``` apiVersion: external-secrets.io/v1alpha1 kind: PushSecret metadata: name: pushsecret namespace: hello-world spec: data: - conversionStrategy: None match: remoteRef: property: baz remoteKey: pushsecrets/testme secretKey: bar deletionPolicy: Delete refreshInterval: 10s secretStoreRefs: - kind: ClusterSecretStore name: vault-backend selector: secret: name: existing-secret updatePolicy: Replace ``` The above takes the property called `baz` of an existing secret called `existing-secret` in the `hello-world` namespace and pushes it to the `secret/pushsecrets/testme` vault path. Suggested-By: Chris Butler Closes: MBP-641 --- ansible/roles/vault_utils/README.md | 91 +++++++++++-------- ansible/roles/vault_utils/defaults/main.yml | 2 + .../vault_utils/tasks/vault_secrets_init.yaml | 24 ++++- .../vault_utils/tasks/vault_spokes_init.yaml | 32 ++++++- 4 files changed, 106 insertions(+), 43 deletions(-) diff --git a/ansible/roles/vault_utils/README.md b/ansible/roles/vault_utils/README.md index 6b851f2a..ba26c702 100644 --- a/ansible/roles/vault_utils/README.md +++ b/ansible/roles/vault_utils/README.md @@ -40,6 +40,17 @@ unseal_namespace: "imperative" This relies on [kubernetes.core](https://docs.ansible.com/ansible/latest/collections/kubernetes/core/k8s_module.html) +## Vault out of the box configuration + +This role configures four secret paths in vault: + +1. `secret/global` - Any secret under this path is accessible in read-only only to all clusters known to ACM (hub and spokes) +2. `secret/hub` - Any secret under this path is accessible in read-only only to the ACM hub cluster +3. `secret/` - Any secret under this path is accessible in read-only only to the spoke cluster +4. `secret/pushsecrets` - Any secret here can be accessed in read and write mode to all clusters known to ACM. This area can + be used with ESO's `PushSecrets` so you can push an existing secret from one namespace, to the vault under this path and + then it can be retrieved by an `ExternalSecret` either in a different namespace *or* from an entirely different cluster. + ## Values secret file format Currently this role supports two formats: version 1.0 (which is the assumed @@ -58,46 +69,6 @@ secret file. The values secret YAML files can be encrypted with `ansible-vault`. If the role detects they are encrypted, the password to decrypt them will be prompted when needed. -### Version 1.0 - -Here is a well-commented example of a version 1.0 file: - -```yaml ---- -# By default when a top-level 'version: 1.0' is missing it is assumed to be '1.0' -# NEVER COMMIT THESE VALUES TO GIT - -secrets: - # These secrets will be pushed in the vault at secret/hub/test The vault will - # have secret/hub/test with secret1 and secret2 as keys with their associated - # values (secrets) - test: - secret1: foo - secret2: bar - - # This ends up as the s3Secret attribute to the path secret/hub/aws - aws: - s3Secret: test-secret - -# This will create the vault key secret/hub/testfoo which will have two -# properties 'b64content' and 'content' which will be the base64-encoded -# content and the normal content respectively -files: - testfoo: ~/ca.crt -# These secrets will be pushed in the vault at secret/region1/test The vault will -# have secret/region1/test with secret1 and secret2 as keys with their associated -# values (secrets) -secrets.region1: - test: - secret1: foo1 - secret2: bar1 -# This will create the vault key secret/region2/testbar which will have two -# properties 'b64content' and 'content' which will be the base64-encoded -# content and the normal content respectively -files.region2: - testbar: ~/ca.crt -``` - ### Version 2.0 Here is a version 2.0 example file (specifying `version: 2.0` is mandatory in this case): @@ -210,6 +181,46 @@ secrets: ini_key: aws_secret_access_key ``` +### Version 1.0 + +Here is a well-commented example of a version 1.0 file: + +```yaml +--- +# By default when a top-level 'version: 1.0' is missing it is assumed to be '1.0' +# NEVER COMMIT THESE VALUES TO GIT + +secrets: + # These secrets will be pushed in the vault at secret/hub/test The vault will + # have secret/hub/test with secret1 and secret2 as keys with their associated + # values (secrets) + test: + secret1: foo + secret2: bar + + # This ends up as the s3Secret attribute to the path secret/hub/aws + aws: + s3Secret: test-secret + +# This will create the vault key secret/hub/testfoo which will have two +# properties 'b64content' and 'content' which will be the base64-encoded +# content and the normal content respectively +files: + testfoo: ~/ca.crt +# These secrets will be pushed in the vault at secret/region1/test The vault will +# have secret/region1/test with secret1 and secret2 as keys with their associated +# values (secrets) +secrets.region1: + test: + secret1: foo1 + secret2: bar1 +# This will create the vault key secret/region2/testbar which will have two +# properties 'b64content' and 'content' which will be the base64-encoded +# content and the normal content respectively +files.region2: + testbar: ~/ca.crt +``` + Internals --------- diff --git a/ansible/roles/vault_utils/defaults/main.yml b/ansible/roles/vault_utils/defaults/main.yml index 4d263223..7759db48 100644 --- a/ansible/roles/vault_utils/defaults/main.yml +++ b/ansible/roles/vault_utils/defaults/main.yml @@ -17,6 +17,8 @@ vault_spoke_capabilities: '[\\\"read\\\"]' vault_spoke_ttl: "15m" vault_global_policy: global vault_global_capabilities: '[\\\"read\\\"]' +vault_pushsecrets_policy: pushsecrets +vault_pushsecrets_capabilities: '[\\\"create\\\",\\\"read\\\",\\\"update\\\",\\\"delete\\\"]' external_secrets_ns: golang-external-secrets external_secrets_sa: golang-external-secrets external_secrets_secret: golang-external-secrets diff --git a/ansible/roles/vault_utils/tasks/vault_secrets_init.yaml b/ansible/roles/vault_utils/tasks/vault_secrets_init.yaml index 35327d58..8a098a7c 100644 --- a/ansible/roles/vault_utils/tasks/vault_secrets_init.yaml +++ b/ansible/roles/vault_utils/tasks/vault_secrets_init.yaml @@ -71,6 +71,28 @@ pod: "{{ vault_pod }}" command: "vault policy write {{ vault_global_policy }}-secret /tmp/policy-{{ vault_global_policy }}.hcl" +- name: Configure VP pushsecrets policy template + kubernetes.core.k8s_exec: + namespace: "{{ vault_ns }}" + pod: "{{ vault_pod }}" + command: > + bash -e -c "echo \"path \\\"secret/data/{{ vault_pushsecrets_policy }}/*\\\" { + capabilities = {{ vault_pushsecrets_capabilities }} }\" > /tmp/policy-{{ vault_pushsecrets_policy }}.hcl" + +- name: Add metadata path to the pushsecrets policy + kubernetes.core.k8s_exec: + namespace: "{{ vault_ns }}" + pod: "{{ vault_pod }}" + command: > + bash -e -c "echo \"path \\\"secret/metadata/{{ vault_pushsecrets_policy }}/*\\\" { + capabilities = {{ vault_pushsecrets_capabilities }} }\" >> /tmp/policy-{{ vault_pushsecrets_policy }}.hcl" + +- name: Configure VP pushsecrets policy + kubernetes.core.k8s_exec: + namespace: "{{ vault_ns }}" + pod: "{{ vault_pod }}" + command: "vault policy write {{ vault_pushsecrets_policy }}-secret /tmp/policy-{{ vault_pushsecrets_policy }}.hcl" + - name: Configure policy template for hub kubernetes.core.k8s_exec: namespace: "{{ vault_ns }}" @@ -93,4 +115,4 @@ vault write auth/"{{ vault_hub }}"/role/"{{ vault_hub }}"-role bound_service_account_names="{{ external_secrets_sa }}" bound_service_account_namespaces="{{ external_secrets_ns }}" - policies="default,{{ vault_global_policy }}-secret,{{ vault_hub }}-secret" ttl="{{ vault_hub_ttl }}" + policies="default,{{ vault_global_policy }}-secret,{{ vault_pushsecrets_policy }}-secret,{{ vault_hub }}-secret" ttl="{{ vault_hub_ttl }}" diff --git a/ansible/roles/vault_utils/tasks/vault_spokes_init.yaml b/ansible/roles/vault_utils/tasks/vault_spokes_init.yaml index 060378bc..bafe490b 100644 --- a/ansible/roles/vault_utils/tasks/vault_spokes_init.yaml +++ b/ansible/roles/vault_utils/tasks/vault_spokes_init.yaml @@ -157,7 +157,7 @@ loop_control: label: "{{ item.key }}" -- name: Configure policy template +- name: Configure spoke policy template kubernetes.core.k8s_exec: namespace: "{{ vault_ns }}" pod: "{{ vault_pod }}" @@ -171,6 +171,34 @@ loop_control: label: "{{ item.key }}" +- name: Configure spoke pushsecrets policy template + kubernetes.core.k8s_exec: + namespace: "{{ vault_ns }}" + pod: "{{ vault_pod }}" + command: > + bash -e -c "echo \"path \\\"secret/data/{{ vault_pushsecrets_policy }}/*\\\" { + capabilities = {{ vault_pushsecrets_capabilities }} }\" >> /tmp/policy-{{ item.value['vault_path'] }}.hcl" + loop: "{{ clusters_info | dict2items }}" + when: + - item.value['esoToken'] is defined + - item.key != "local-cluster" + loop_control: + label: "{{ item.key }}" + +- name: Configure spoke pushsecrets metadata policy template + kubernetes.core.k8s_exec: + namespace: "{{ vault_ns }}" + pod: "{{ vault_pod }}" + command: > + bash -e -c "echo \"path \\\"secret/metadata/{{ vault_pushsecrets_policy }}/*\\\" { + capabilities = {{ vault_pushsecrets_capabilities }} }\" >> /tmp/policy-{{ item.value['vault_path'] }}.hcl" + loop: "{{ clusters_info | dict2items }}" + when: + - item.value['esoToken'] is defined + - item.key != "local-cluster" + loop_control: + label: "{{ item.key }}" + - name: Configure policy for spokes kubernetes.core.k8s_exec: namespace: "{{ vault_ns }}" @@ -191,7 +219,7 @@ vault write auth/"{{ item.value['vault_path'] }}"/role/"{{ item.value['vault_path'] }}"-role bound_service_account_names="{{ external_secrets_sa }}" bound_service_account_namespaces="{{ external_secrets_ns }}" - policies="default,{{ vault_global_policy }}-secret,{{ item.value['vault_path'] }}-secret" ttl="{{ vault_spoke_ttl }}" + policies="default,{{ vault_global_policy }}-secret,{{ vault_pushsecrets_policy }}-secret,{{ item.value['vault_path'] }}-secret" ttl="{{ vault_spoke_ttl }}" loop: "{{ clusters_info | dict2items }}" when: - item.value['esoToken'] is defined From 9aef7783fd35c4e6c7f8f1950f6e1674736bf650 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Fri, 30 Aug 2024 12:09:41 +0200 Subject: [PATCH 2/2] Fix PyInk warnings --- ansible/plugins/filter/parse_acm_secrets.py | 1 + ansible/plugins/module_utils/load_secrets_v1.py | 1 + ansible/plugins/module_utils/load_secrets_v2.py | 1 + ansible/plugins/module_utils/parse_secrets_v2.py | 1 + ansible/plugins/modules/vault_load_parsed_secrets.py | 1 + ansible/tests/unit/test_ini_file.py | 1 + ansible/tests/unit/test_parse_secrets.py | 2 ++ ansible/tests/unit/test_vault_load_parsed_secrets.py | 1 + ansible/tests/unit/test_vault_load_secrets.py | 1 + ansible/tests/unit/test_vault_load_secrets_v2.py | 1 + 10 files changed, 11 insertions(+) diff --git a/ansible/plugins/filter/parse_acm_secrets.py b/ansible/plugins/filter/parse_acm_secrets.py index 0445d96d..1c5148e3 100644 --- a/ansible/plugins/filter/parse_acm_secrets.py +++ b/ansible/plugins/filter/parse_acm_secrets.py @@ -79,5 +79,6 @@ def parse_acm_secrets(secrets): class FilterModule: + def filters(self): return {"parse_acm_secrets": parse_acm_secrets} diff --git a/ansible/plugins/module_utils/load_secrets_v1.py b/ansible/plugins/module_utils/load_secrets_v1.py index 6478ac26..8b89d85a 100644 --- a/ansible/plugins/module_utils/load_secrets_v1.py +++ b/ansible/plugins/module_utils/load_secrets_v1.py @@ -26,6 +26,7 @@ class LoadSecretsV1: + def __init__( self, module, diff --git a/ansible/plugins/module_utils/load_secrets_v2.py b/ansible/plugins/module_utils/load_secrets_v2.py index 05a5917e..46cdcffa 100644 --- a/ansible/plugins/module_utils/load_secrets_v2.py +++ b/ansible/plugins/module_utils/load_secrets_v2.py @@ -40,6 +40,7 @@ class LoadSecretsV2: + def __init__(self, module, syaml, namespace, pod): self.module = module self.namespace = namespace diff --git a/ansible/plugins/module_utils/parse_secrets_v2.py b/ansible/plugins/module_utils/parse_secrets_v2.py index 512f75ef..f88579b6 100644 --- a/ansible/plugins/module_utils/parse_secrets_v2.py +++ b/ansible/plugins/module_utils/parse_secrets_v2.py @@ -42,6 +42,7 @@ class ParseSecretsV2: + def __init__(self, module, syaml, secrets_backing_store): self.module = module self.syaml = syaml diff --git a/ansible/plugins/modules/vault_load_parsed_secrets.py b/ansible/plugins/modules/vault_load_parsed_secrets.py index 0a6aa146..f5acdc86 100644 --- a/ansible/plugins/modules/vault_load_parsed_secrets.py +++ b/ansible/plugins/modules/vault_load_parsed_secrets.py @@ -82,6 +82,7 @@ class VaultSecretLoader: + def __init__( self, module, diff --git a/ansible/tests/unit/test_ini_file.py b/ansible/tests/unit/test_ini_file.py index e92280cb..6c30fdbb 100644 --- a/ansible/tests/unit/test_ini_file.py +++ b/ansible/tests/unit/test_ini_file.py @@ -29,6 +29,7 @@ class TestMyModule(unittest.TestCase): + def setUp(self): self.testdir_v2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), "v2") diff --git a/ansible/tests/unit/test_parse_secrets.py b/ansible/tests/unit/test_parse_secrets.py index 0cfef1b6..2dab5716 100644 --- a/ansible/tests/unit/test_parse_secrets.py +++ b/ansible/tests/unit/test_parse_secrets.py @@ -62,6 +62,7 @@ def set_module_args(args): class BytesEncoder(json.JSONEncoder): + def default(self, o): if isinstance(o, bytes): return base64.b64encode(o).decode("ascii") @@ -113,6 +114,7 @@ def fail_json(*args, **kwargs): @mock.patch("getpass.getpass") class TestMyModule(unittest.TestCase): + def create_inifile(self): self.inifile = open("/tmp/awscredentials", "w") config = configparser.ConfigParser() diff --git a/ansible/tests/unit/test_vault_load_parsed_secrets.py b/ansible/tests/unit/test_vault_load_parsed_secrets.py index 1a449739..66ec6b69 100644 --- a/ansible/tests/unit/test_vault_load_parsed_secrets.py +++ b/ansible/tests/unit/test_vault_load_parsed_secrets.py @@ -70,6 +70,7 @@ def fail_json(*args, **kwargs): class TestMyModule(unittest.TestCase): + def setUp(self): self.mock_module_helper = patch.multiple( basic.AnsibleModule, exit_json=exit_json, fail_json=fail_json diff --git a/ansible/tests/unit/test_vault_load_secrets.py b/ansible/tests/unit/test_vault_load_secrets.py index 12deeb3f..03d25d8c 100644 --- a/ansible/tests/unit/test_vault_load_secrets.py +++ b/ansible/tests/unit/test_vault_load_secrets.py @@ -74,6 +74,7 @@ def fail_json(*args, **kwargs): class TestMyModule(unittest.TestCase): + def setUp(self): self.mock_module_helper = patch.multiple( basic.AnsibleModule, exit_json=exit_json, fail_json=fail_json diff --git a/ansible/tests/unit/test_vault_load_secrets_v2.py b/ansible/tests/unit/test_vault_load_secrets_v2.py index d0e5881c..7b934320 100644 --- a/ansible/tests/unit/test_vault_load_secrets_v2.py +++ b/ansible/tests/unit/test_vault_load_secrets_v2.py @@ -77,6 +77,7 @@ def fail_json(*args, **kwargs): @mock.patch("getpass.getpass") class TestMyModule(unittest.TestCase): + def create_inifile(self): self.inifile = open("/tmp/awscredentials", "w") config = configparser.ConfigParser()