Release v1.3.5

[taylorbot]

Towards https://github.com/giantswarm/giantswarm/issues/31843 and https://github.com/giantswarm/giantswarm/issues/32254

[github-actions]

There were differences in the rendered Helm template, please check! ⚠️

Output

=== Differences when rendered with values file helm/cluster-aws/ci/test-auditd-values.yaml ===

(file level)
  - two documents removed:
    ---
    # Source: cluster-aws/charts/cluster/templates/apps/apps.yaml
    apiVersion: application.giantswarm.io/v1alpha1
    kind: App
    metadata:
      name: test-wc-minimal-observability-policies
      namespace: org-giantswarm
      labels:
        # deprecated: "app: cluster-aws" label is deprecated and it will be removed after upgrading
    # to Kubernetes 1.25. We still need it here because existing ClusterResourceSet selectors
    # need this label on the Cluster resource.
    app: cluster-aws
        app.kubernetes.io/name: cluster
        app.kubernetes.io/version: 1.7.0
        app.kubernetes.io/part-of: cluster-aws
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        helm.sh/chart: cluster-1.7.0
        application.giantswarm.io/team: turtles
        giantswarm.io/cluster: test-wc-minimal
        giantswarm.io/organization: test
        giantswarm.io/service-priority: lowest
        cluster.x-k8s.io/cluster-name: test-wc-minimal
        cluster.x-k8s.io/watch-filter: capi
        release.giantswarm.io/version: 27.0.0-alpha.1
        giantswarm.io/managed-by: cluster
    spec:
      name: observability-policies
      version: N/A
      catalog: default
      namespace: kube-system
      kubeConfig:
        context:
          name: test-wc-minimal-admin@test-wc-minimal
        inCluster: false
        secret:
          name: test-wc-minimal-kubeconfig
          namespace: org-giantswarm
    # Source: cluster-aws/charts/cluster/templates/clusterapi/workers/kubeadmconfig.yaml
    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
    kind: KubeadmConfig
    metadata:
      annotations:
        machine-pool.giantswarm.io/name: test-wc-minimal-pool0
      labels:
        # deprecated: "app: cluster-aws" label is deprecated and it will be removed after upgrading
    # to Kubernetes 1.25. We still need it here because existing ClusterResourceSet selectors
    # need this label on the Cluster resource.
    app: cluster-aws
        app.kubernetes.io/name: cluster
        app.kubernetes.io/version: 1.7.0
        app.kubernetes.io/part-of: cluster-aws
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        helm.sh/chart: cluster-1.7.0
        application.giantswarm.io/team: turtles
        giantswarm.io/cluster: test-wc-minimal
        giantswarm.io/organization: test
        giantswarm.io/service-priority: lowest
        cluster.x-k8s.io/cluster-name: test-wc-minimal
        cluster.x-k8s.io/watch-filter: capi
        release.giantswarm.io/version: 27.0.0-alpha.1
        giantswarm.io/machine-pool: test-wc-minimal-pool0
      name: test-wc-minimal-pool0-fc8a3
      namespace: org-giantswarm
    spec:
      format: ignition
      ignition:
        containerLinuxConfig:
          additionalConfig: |
            systemd:
              units:      
              - name: os-hardening.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Apply os hardening
                  [Service]
                  Type=oneshot
                  ExecStartPre=-/bin/bash -c "gpasswd -d core rkt; gpasswd -d core docker; gpasswd -d core wheel"
                  ExecStartPre=/bin/bash -c "until [ -f '/etc/sysctl.d/hardening.conf' ]; do echo Waiting for sysctl file; sleep 1s;done;"
                  ExecStart=/usr/sbin/sysctl -p /etc/sysctl.d/hardening.conf
                  [Install]
                  WantedBy=multi-user.target
              - name: update-engine.service
                enabled: false
                mask: true
              - name: locksmithd.service
                enabled: false
                mask: true
              - name: sshkeys.service
                enabled: false
                mask: true
              - name: kubeadm.service
                dropins:
                - name: 10-flatcar.conf
                  contents: |
                    [Unit]
                    # kubeadm must run after coreos-metadata populated /run/metadata directory.
                    Requires=coreos-metadata.service
                    After=coreos-metadata.service
                    # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939.
                    After=containerd.service
                    # kubeadm requires having an IP
                    After=network-online.target
                    Wants=network-online.target
                    [Service]
                    # Ensure kubeadm service has access to kubeadm binary in /opt/bin on Flatcar.
                    Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/bin
                    # To make metadata environment variables available for pre-kubeadm commands.
                    EnvironmentFile=/run/metadata/*
              - name: containerd.service
                enabled: true
                contents: |
                dropins:
                - name: 10-change-cgroup.conf
                  contents: |
                    [Service]
                    CPUAccounting=true
                    MemoryAccounting=true
                    Slice=kubereserved.slice
              - name: audit-rules.service
                enabled: true
                dropins:
                - name: 10-wait-for-containerd.conf
                  contents: |
                    [Service]
                    ExecStartPre=/bin/bash -c "while [ ! -f /etc/audit/rules.d/containerd.rules ]; do echo 'Waiting for /etc/audit/rules.d/containerd.rules to be written' && sleep 1; done"
                    Restart=on-failure
              - name: teleport.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Teleport Service
                  After=network.target
                  [Service]
                  Type=simple
                  Restart=on-failure
                  ExecStart=/opt/bin/teleport start --roles=node --config=/etc/teleport.yaml --pid-file=/run/teleport.pid
                  ExecReload=/bin/kill -HUP $MAINPID
                  PIDFile=/run/teleport.pid
                  LimitNOFILE=524288
                  [Install]
                  WantedBy=multi-user.target      
              - name: kubelet-aws-config.service
                enabled: true
              - name: var-lib.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=lib volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/lib
                  Where=/var/lib
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
              - name: var-log.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=log volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/log
                  Where=/var/log
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
            storage:
              filesystems:      
              - name: lib
                mount:
                  device: /dev/xvdd
                  format: xfs
                  wipeFilesystem: true
                  label: lib
              - name: log
                mount:
                  device: /dev/xvde
                  format: xfs
                  wipeFilesystem: true
                  label: log
              directories:      
              - path: /var/lib/kubelet
                mode: 0750      
      joinConfiguration:
        nodeRegistration:
          name: ${COREOS_EC2_HOSTNAME}
          kubeletExtraArgs:
            cloud-provider: external
            healthz-bind-address: 0.0.0.0
            node-ip: ${COREOS_EC2_IPV4_LOCAL}
            node-labels: "ip=${COREOS_EC2_IPV4_LOCAL},role=worker,giantswarm.io/machine-pool=test-wc-minimal-pool0"
            v: 2
        patches:
          directory: /etc/kubernetes/patches
      preKubeadmCommands:
      - "envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp"
      - "mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml"
      - "systemctl restart containerd"
      files:
      - path: /etc/sysctl.d/hardening.conf
        permissions: 0644
        encoding: base64
        content: ZnMuaW5vdGlmeS5tYXhfdXNlcl93YXRjaGVzID0gMTYzODQKZnMuaW5vdGlmeS5tYXhfdXNlcl9pbnN0YW5jZXMgPSA4MTkyCmtlcm5lbC5rcHRyX3Jlc3RyaWN0ID0gMgprZXJuZWwuc3lzcnEgPSAwCm5ldC5pcHY0LmNvbmYuYWxsLmxvZ19tYXJ0aWFucyA9IDEKbmV0LmlwdjQuY29uZi5hbGwuc2VuZF9yZWRpcmVjdHMgPSAwCm5ldC5pcHY0LmNvbmYuZGVmYXVsdC5hY2NlcHRfcmVkaXJlY3RzID0gMApuZXQuaXB2NC5jb25mLmRlZmF1bHQubG9nX21hcnRpYW5zID0gMQpuZXQuaXB2NC50Y3BfdGltZXN0YW1wcyA9IDAKbmV0LmlwdjYuY29uZi5hbGwuYWNjZXB0X3JlZGlyZWN0cyA9IDAKbmV0LmlwdjYuY29uZi5kZWZhdWx0LmFjY2VwdF9yZWRpcmVjdHMgPSAwCiMgSW5jcmVhc2VkIG1tYXBmcyBiZWNhdXNlIHNvbWUgYXBwbGljYXRpb25zLCBsaWtlIEVTLCBuZWVkIGhpZ2hlciBsaW1pdCB0byBzdG9yZSBkYXRhIHByb3Blcmx5CnZtLm1heF9tYXBfY291bnQgPSAyNjIxNDQKIyBSZXNlcnZlZCB0byBhdm9pZCBjb25mbGljdHMgd2l0aCBrdWJlLWFwaXNlcnZlciwgd2hpY2ggYWxsb2NhdGVzIHdpdGhpbiB0aGlzIHJhbmdlCm5ldC5pcHY0LmlwX2xvY2FsX3Jlc2VydmVkX3BvcnRzPTMwMDAwLTMyNzY3Cm5ldC5pcHY0LmNvbmYuYWxsLnJwX2ZpbHRlciA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2lnbm9yZSA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2Fubm91bmNlID0gMgoKIyBUaGVzZSBhcmUgcmVxdWlyZWQgZm9yIHRoZSBrdWJlbGV0ICctLXByb3RlY3Qta2VybmVsLWRlZmF1bHRzJyBmbGFnCiMgU2VlIGh0dHBzOi8vZ2l0aHViLmNvbS9naWFudHN3YXJtL2dpYW50c3dhcm0vaXNzdWVzLzEzNTg3CnZtLm92ZXJjb21taXRfbWVtb3J5PTEKa2VybmVsLnBhbmljPTEwCmtlcm5lbC5wYW5pY19vbl9vb3BzPTEK
      - path: /etc/selinux/config
        permissions: 0644
        encoding: base64
        content: IyBUaGlzIGZpbGUgY29udHJvbHMgdGhlIHN0YXRlIG9mIFNFTGludXggb24gdGhlIHN5c3RlbSBvbiBib290LgoKIyBTRUxJTlVYIGNhbiB0YWtlIG9uZSBvZiB0aGVzZSB0aHJlZSB2YWx1ZXM6CiMgICAgICAgZW5mb3JjaW5nIC0gU0VMaW51eCBzZWN1cml0eSBwb2xpY3kgaXMgZW5mb3JjZWQuCiMgICAgICAgcGVybWlzc2l2ZSAtIFNFTGludXggcHJpbnRzIHdhcm5pbmdzIGluc3RlYWQgb2YgZW5mb3JjaW5nLgojICAgICAgIGRpc2FibGVkIC0gTm8gU0VMaW51eCBwb2xpY3kgaXMgbG9hZGVkLgpTRUxJTlVYPXBlcm1pc3NpdmUKCiMgU0VMSU5VWFRZUEUgY2FuIHRha2Ugb25lIG9mIHRoZXNlIGZvdXIgdmFsdWVzOgojICAgICAgIHRhcmdldGVkIC0gT25seSB0YXJnZXRlZCBuZXR3b3JrIGRhZW1vbnMgYXJlIHByb3RlY3RlZC4KIyAgICAgICBzdHJpY3QgICAtIEZ1bGwgU0VMaW51eCBwcm90ZWN0aW9uLgojICAgICAgIG1scyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1MZXZlbCBTZWN1cml0eQojICAgICAgIG1jcyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1DYXRlZ29yeSBTZWN1cml0eQojICAgICAgICAgICAgICAgICAgKG1scywgYnV0IG9ubHkgb25lIHNlbnNpdGl2aXR5IGxldmVsKQpTRUxJTlVYVFlQRT1tY3MK
      - path: /etc/systemd/timesyncd.conf
        permissions: 0644
        encoding: base64
        content: W1RpbWVdCk5UUD0xNjkuMjU0LjE2OS4xMjMK
      - path: /etc/kubernetes/patches/kubeletconfiguration.yaml
        permissions: 0644
        encoding: base64
        content: YXBpVmVyc2lvbjoga3ViZWxldC5jb25maWcuazhzLmlvL3YxYmV0YTEKa2luZDogS3ViZWxldENvbmZpZ3VyYXRpb24Kc2h1dGRvd25HcmFjZVBlcmlvZDogMzAwcwpzaHV0ZG93bkdyYWNlUGVyaW9kQ3JpdGljYWxQb2RzOiA2MHMKa2VybmVsTWVtY2dOb3RpZmljYXRpb246IHRydWUKZXZpY3Rpb25Tb2Z0OgogIG1lbW9yeS5hdmFpbGFibGU6ICI1MDBNaSIKZXZpY3Rpb25IYXJkOgogIG1lbW9yeS5hdmFpbGFibGU6ICIyMDBNaSIKICBpbWFnZWZzLmF2YWlsYWJsZTogIjE1JSIKZXZpY3Rpb25Tb2Z0R3JhY2VQZXJpb2Q6CiAgbWVtb3J5LmF2YWlsYWJsZTogIjVzIgpldmljdGlvbk1heFBvZEdyYWNlUGVyaW9kOiA2MAprdWJlUmVzZXJ2ZWQ6CiAgY3B1OiAzNTBtCiAgbWVtb3J5OiAxMjgwTWkKICBlcGhlbWVyYWwtc3RvcmFnZTogMTAyNE1pCmt1YmVSZXNlcnZlZENncm91cDogL2t1YmVyZXNlcnZlZC5zbGljZQpwcm90ZWN0S2VybmVsRGVmYXVsdHM6IHRydWUKc3lzdGVtUmVzZXJ2ZWQ6CiAgY3B1OiAyNTBtCiAgbWVtb3J5OiAzODRNaQpzeXN0ZW1SZXNlcnZlZENncm91cDogL3N5c3RlbS5zbGljZQp0bHNDaXBoZXJTdWl0ZXM6Ci0gVExTX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19BRVNfMjU2X0dDTV9TSEEzODQKLSBUTFNfQ0hBQ0hBMjBfUE9MWTEzMDVfU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQ0hBQ0hBMjBfUE9MWTEzMDUKLSBUTFNfRUNESEVfRUNEU0FfV0lUSF9DSEFDSEEyMF9QT0xZMTMwNV9TSEEyNTYKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX1JTQV9XSVRIX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19FQ0RIRV9SU0FfV0lUSF9BRVNfMjU2X0NCQ19TSEEKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1X1NIQTI1NgotIFRMU19SU0FfV0lUSF9BRVNfMTI4X0NCQ19TSEEKLSBUTFNfUlNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX1JTQV9XSVRIX0FFU18yNTZfQ0JDX1NIQQotIFRMU19SU0FfV0lUSF9BRVNfMjU2X0dDTV9TSEEzODQKc2VyaWFsaXplSW1hZ2VQdWxsczogZmFsc2UKc3RyZWFtaW5nQ29ubmVjdGlvbklkbGVUaW1lb3V0OiAxaAphbGxvd2VkVW5zYWZlU3lzY3RsczoKLSAibmV0LioiCg==
      - path: /etc/systemd/logind.conf.d/zzz-kubelet-graceful-shutdown.conf
        permissions: 0700
        encoding: base64
        content: W0xvZ2luXQojIGRlbGF5CkluaGliaXREZWxheU1heFNlYz0zMDAK
      - path: /etc/teleport-join-token
        permissions: 0644
        contentFrom:
          secret:
            name: test-wc-minimal-teleport-join-token
            key: joinToken
      - path: /opt/teleport-node-role.sh
        permissions: 0755
        encoding: base64
        content: IyEvYmluL2Jhc2gKCmlmIHN5c3RlbWN0bCBpcy1hY3RpdmUgLXEga3ViZWxldC5zZXJ2aWNlOyB0aGVuCiAgICBpZiBbIC1lICIvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzL2t1YmUtYXBpc2VydmVyLnlhbWwiIF07IHRoZW4KICAgICAgICBlY2hvICJjb250cm9sLXBsYW5lIgogICAgZWxzZQogICAgICAgIGVjaG8gIndvcmtlciIKICAgIGZpCmVsc2UKICAgIGVjaG8gIiIKZmkK
      - path: /etc/teleport.yaml
        permissions: 0644
        encoding: base64
        content: dmVyc2lvbjogdjMKdGVsZXBvcnQ6CiAgZGF0YV9kaXI6IC92YXIvbGliL3RlbGVwb3J0CiAgam9pbl9wYXJhbXM6CiAgICB0b2tlbl9uYW1lOiAvZXRjL3RlbGVwb3J0LWpvaW4tdG9rZW4KICAgIG1ldGhvZDogdG9rZW4KICBwcm94eV9zZXJ2ZXI6IHRlbGVwb3J0LmdpYW50c3dhcm0uaW86NDQzCiAgbG9nOgogICAgb3V0cHV0OiBzdGRlcnIKYXV0aF9zZXJ2aWNlOgogIGVuYWJsZWQ6ICJubyIKc3NoX3NlcnZpY2U6CiAgZW5hYmxlZDogInllcyIKICBjb21tYW5kczoKICAtIG5hbWU6IG5vZGUKICAgIGNvbW1hbmQ6IFtob3N0bmFtZV0KICAgIHBlcmlvZDogMjRoMG0wcwogIC0gbmFtZTogYXJjaAogICAgY29tbWFuZDogW3VuYW1lLCAtbV0KICAgIHBlcmlvZDogMjRoMG0wcwogIC0gbmFtZTogcm9sZQogICAgY29tbWFuZDogWy9vcHQvdGVsZXBvcnQtbm9kZS1yb2xlLnNoXQogICAgcGVyaW9kOiAxbTBzCiAgbGFiZWxzOgogICAgaW5zOiB0ZXN0CiAgICBtYzogdGVzdAogICAgY2x1c3RlcjogdGVzdC13Yy1taW5pbWFsCiAgICBiYXNlRG9tYWluOiBleGFtcGxlLmNvbQpwcm94eV9zZXJ2aWNlOgogIGVuYWJsZWQ6ICJubyIK
      - path: /etc/audit/rules.d/99-default.rules
        permissions: 0640
        encoding: base64
        content: IyBPdmVycmlkZGVuIGJ5IEdpYW50IFN3YXJtLgotYSBleGl0LGFsd2F5cyAtRiBhcmNoPWI2NCAtUyBleGVjdmUgLWsgYXVkaXRpbmcKLWEgZXhpdCxhbHdheXMgLUYgYXJjaD1iMzIgLVMgZXhlY3ZlIC1rIGF1ZGl0aW5nCg==
      - contentFrom:
          secret:
            key: kubelet-aws-config.sh
            name: test-wc-minimal-provider-specific-files-4
        path: /opt/bin/kubelet-aws-config.sh
        permissions: 0755
      - contentFrom:
          secret:
            key: kubelet-aws-config.service
            name: test-wc-minimal-provider-specific-files-4
        path: /etc/systemd/system/kubelet-aws-config.service
        permissions: 0644
      - contentFrom:
          secret:
            key: 99-unmanaged-devices.network
            name: test-wc-minimal-provider-specific-files-4
        path: /etc/systemd/network/99-unmanaged-devices.network
        permissions: 0644
      - path: /etc/containerd/config.toml
        permissions: 0644
        contentFrom:
          secret:
            name: test-wc-minimal-pool0-containerd-dec40c1e
            key: config.toml
    
  
    ---
    # Source: cluster-aws/charts/cluster/templates/clusterapi/workers/kubeadmconfig.yaml
    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
    kind: KubeadmConfig
    metadata:
      name: test-wc-minimal-pool0-f2de8
      namespace: org-giantswarm
      annotations:
        machine-pool.giantswarm.io/name: test-wc-minimal-pool0
      labels:
        # deprecated: "app: cluster-aws" label is deprecated and it will be removed after upgrading
    # to Kubernetes 1.25. We still need it here because existing ClusterResourceSet selectors
    # need this label on the Cluster resource.
    app: cluster-aws
        app.kubernetes.io/name: cluster
        app.kubernetes.io/version: 1.0.3
        app.kubernetes.io/part-of: cluster-aws
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        helm.sh/chart: cluster-1.0.3
        application.giantswarm.io/team: turtles
        giantswarm.io/cluster: test-wc-minimal
        giantswarm.io/organization: test
        giantswarm.io/service-priority: lowest
        cluster.x-k8s.io/cluster-name: test-wc-minimal
        cluster.x-k8s.io/watch-filter: capi
        release.giantswarm.io/version: 27.0.0-alpha.1
        giantswarm.io/machine-pool: test-wc-minimal-pool0
    spec:
      format: ignition
      ignition:
        containerLinuxConfig:
          additionalConfig: |
            systemd:
              units:      
              - name: os-hardening.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Apply os hardening
                  [Service]
                  Type=oneshot
                  ExecStartPre=-/bin/bash -c "gpasswd -d core rkt; gpasswd -d core docker; gpasswd -d core wheel"
                  ExecStartPre=/bin/bash -c "until [ -f '/etc/sysctl.d/hardening.conf' ]; do echo Waiting for sysctl file; sleep 1s;done;"
                  ExecStart=/usr/sbin/sysctl -p /etc/sysctl.d/hardening.conf
                  [Install]
                  WantedBy=multi-user.target
              - name: update-engine.service
                enabled: false
                mask: true
              - name: locksmithd.service
                enabled: false
                mask: true
              - name: sshkeys.service
                enabled: false
                mask: true
              - name: teleport.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Teleport Service
                  After=network.target
                  [Service]
                  Type=simple
                  Restart=on-failure
                  ExecStart=/opt/bin/teleport start --roles=node --config=/etc/teleport.yaml --pid-file=/run/teleport.pid
                  ExecReload=/bin/kill -HUP $MAINPID
                  PIDFile=/run/teleport.pid
                  LimitNOFILE=524288
                  [Install]
                  WantedBy=multi-user.target
              - name: kubeadm.service
                dropins:
                - name: 10-flatcar.conf
                  contents: |
                    [Unit]
                    # kubeadm must run after coreos-metadata populated /run/metadata directory.
                    Requires=coreos-metadata.service
                    After=coreos-metadata.service
                    # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939.
                    After=containerd.service
                    # kubeadm requires having an IP
                    After=network-online.target
                    Wants=network-online.target
                    [Service]
                    # Ensure kubeadm service has access to kubeadm binary in /opt/bin on Flatcar.
                    Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/bin
                    # To make metadata environment variables available for pre-kubeadm commands.
                    EnvironmentFile=/run/metadata/*
              - name: containerd.service
                enabled: true
                contents: |
                dropins:
                - name: 10-change-cgroup.conf
                  contents: |
                    [Service]
                    CPUAccounting=true
                    MemoryAccounting=true
                    Slice=kubereserved.slice
              - name: audit-rules.service
                enabled: true
                dropins:
                - name: 10-wait-for-containerd.conf
                  contents: |
                    [Service]
                    ExecStartPre=/bin/bash -c "while [ ! -f /etc/audit/rules.d/containerd.rules ]; do echo 'Waiting for /etc/audit/rules.d/containerd.rules to be written' && sleep 1; done"
                    Restart=on-failure      
              - name: kubelet-aws-config.service
                enabled: true
              - name: var-lib.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=lib volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/lib
                  Where=/var/lib
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
              - name: var-log.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=log volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/log
                  Where=/var/log
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
            storage:
              filesystems:      
              - name: lib
                mount:
                  device: /dev/xvdd
                  format: xfs
                  wipeFilesystem: true
                  label: lib
              - name: log
                mount:
                  device: /dev/xvde
                  format: xfs
                  wipeFilesystem: true
                  label: log
              directories:      
              - path: /var/lib/kubelet
                mode: 0750      
            
      joinConfiguration:
        nodeRegistration:
          name: ${COREOS_EC2_HOSTNAME}
          kubeletExtraArgs:
            cloud-provider: external
            healthz-bind-address: 0.0.0.0
            node-ip: ${COREOS_EC2_IPV4_LOCAL}
            node-labels: "ip=${COREOS_EC2_IPV4_LOCAL},role=worker,giantswarm.io/machine-pool=test-wc-minimal-pool0,"
            v: 2
        patches:
          directory: /etc/kubernetes/patches
      preKubeadmCommands:
      - "envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp"
      - "mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml"
      - "systemctl restart containerd"
      files:
      - path: /etc/sysctl.d/hardening.conf
        permissions: 0644
        encoding: base64
        content: ZnMuaW5vdGlmeS5tYXhfdXNlcl93YXRjaGVzID0gMTYzODQKZnMuaW5vdGlmeS5tYXhfdXNlcl9pbnN0YW5jZXMgPSA4MTkyCmtlcm5lbC5rcHRyX3Jlc3RyaWN0ID0gMgprZXJuZWwuc3lzcnEgPSAwCm5ldC5pcHY0LmNvbmYuYWxsLmxvZ19tYXJ0aWFucyA9IDEKbmV0LmlwdjQuY29uZi5hbGwuc2VuZF9yZWRpcmVjdHMgPSAwCm5ldC5pcHY0LmNvbmYuZGVmYXVsdC5hY2NlcHRfcmVkaXJlY3RzID0gMApuZXQuaXB2NC5jb25mLmRlZmF1bHQubG9nX21hcnRpYW5zID0gMQpuZXQuaXB2NC50Y3BfdGltZXN0YW1wcyA9IDAKbmV0LmlwdjYuY29uZi5hbGwuYWNjZXB0X3JlZGlyZWN0cyA9IDAKbmV0LmlwdjYuY29uZi5kZWZhdWx0LmFjY2VwdF9yZWRpcmVjdHMgPSAwCiMgSW5jcmVhc2VkIG1tYXBmcyBiZWNhdXNlIHNvbWUgYXBwbGljYXRpb25zLCBsaWtlIEVTLCBuZWVkIGhpZ2hlciBsaW1pdCB0byBzdG9yZSBkYXRhIHByb3Blcmx5CnZtLm1heF9tYXBfY291bnQgPSAyNjIxNDQKIyBSZXNlcnZlZCB0byBhdm9pZCBjb25mbGljdHMgd2l0aCBrdWJlLWFwaXNlcnZlciwgd2hpY2ggYWxsb2NhdGVzIHdpdGhpbiB0aGlzIHJhbmdlCm5ldC5pcHY0LmlwX2xvY2FsX3Jlc2VydmVkX3BvcnRzPTMwMDAwLTMyNzY3Cm5ldC5pcHY0LmNvbmYuYWxsLnJwX2ZpbHRlciA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2lnbm9yZSA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2Fubm91bmNlID0gMgoKIyBUaGVzZSBhcmUgcmVxdWlyZWQgZm9yIHRoZSBrdWJlbGV0ICctLXByb3RlY3Qta2VybmVsLWRlZmF1bHRzJyBmbGFnCiMgU2VlIGh0dHBzOi8vZ2l0aHViLmNvbS9naWFudHN3YXJtL2dpYW50c3dhcm0vaXNzdWVzLzEzNTg3CnZtLm92ZXJjb21taXRfbWVtb3J5PTEKa2VybmVsLnBhbmljPTEwCmtlcm5lbC5wYW5pY19vbl9vb3BzPTEK
      - path: /etc/selinux/config
        permissions: 0644
        encoding: base64
        content: IyBUaGlzIGZpbGUgY29udHJvbHMgdGhlIHN0YXRlIG9mIFNFTGludXggb24gdGhlIHN5c3RlbSBvbiBib290LgoKIyBTRUxJTlVYIGNhbiB0YWtlIG9uZSBvZiB0aGVzZSB0aHJlZSB2YWx1ZXM6CiMgICAgICAgZW5mb3JjaW5nIC0gU0VMaW51eCBzZWN1cml0eSBwb2xpY3kgaXMgZW5mb3JjZWQuCiMgICAgICAgcGVybWlzc2l2ZSAtIFNFTGludXggcHJpbnRzIHdhcm5pbmdzIGluc3RlYWQgb2YgZW5mb3JjaW5nLgojICAgICAgIGRpc2FibGVkIC0gTm8gU0VMaW51eCBwb2xpY3kgaXMgbG9hZGVkLgpTRUxJTlVYPXBlcm1pc3NpdmUKCiMgU0VMSU5VWFRZUEUgY2FuIHRha2Ugb25lIG9mIHRoZXNlIGZvdXIgdmFsdWVzOgojICAgICAgIHRhcmdldGVkIC0gT25seSB0YXJnZXRlZCBuZXR3b3JrIGRhZW1vbnMgYXJlIHByb3RlY3RlZC4KIyAgICAgICBzdHJpY3QgICAtIEZ1bGwgU0VMaW51eCBwcm90ZWN0aW9uLgojICAgICAgIG1scyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1MZXZlbCBTZWN1cml0eQojICAgICAgIG1jcyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1DYXRlZ29yeSBTZWN1cml0eQojICAgICAgICAgICAgICAgICAgKG1scywgYnV0IG9ubHkgb25lIHNlbnNpdGl2aXR5IGxldmVsKQpTRUxJTlVYVFlQRT1tY3MK
      - path: /etc/systemd/timesyncd.conf
        permissions: 0644
        encoding: base64
        content: W1RpbWVdCk5UUD0xNjkuMjU0LjE2OS4xMjMK
      - path: /etc/kubernetes/patches/kubeletconfiguration.yaml
        permissions: 0644
        encoding: base64
        content: YXBpVmVyc2lvbjoga3ViZWxldC5jb25maWcuazhzLmlvL3YxYmV0YTEKa2luZDogS3ViZWxldENvbmZpZ3VyYXRpb24Kc2h1dGRvd25HcmFjZVBlcmlvZDogMzAwcwpzaHV0ZG93bkdyYWNlUGVyaW9kQ3JpdGljYWxQb2RzOiA2MHMKa2VybmVsTWVtY2dOb3RpZmljYXRpb246IHRydWUKZXZpY3Rpb25Tb2Z0OgogIG1lbW9yeS5hdmFpbGFibGU6ICI1MDBNaSIKZXZpY3Rpb25IYXJkOgogIG1lbW9yeS5hdmFpbGFibGU6ICIyMDBNaSIKICBpbWFnZWZzLmF2YWlsYWJsZTogIjE1JSIKZXZpY3Rpb25Tb2Z0R3JhY2VQZXJpb2Q6CiAgbWVtb3J5LmF2YWlsYWJsZTogIjVzIgpldmljdGlvbk1heFBvZEdyYWNlUGVyaW9kOiA2MAprdWJlUmVzZXJ2ZWQ6CiAgY3B1OiAzNTBtCiAgbWVtb3J5OiAxMjgwTWkKICBlcGhlbWVyYWwtc3RvcmFnZTogMTAyNE1pCmt1YmVSZXNlcnZlZENncm91cDogL2t1YmVyZXNlcnZlZC5zbGljZQpwcm90ZWN0S2VybmVsRGVmYXVsdHM6IHRydWUKc3lzdGVtUmVzZXJ2ZWQ6CiAgY3B1OiAyNTBtCiAgbWVtb3J5OiAzODRNaQpzeXN0ZW1SZXNlcnZlZENncm91cDogL3N5c3RlbS5zbGljZQp0bHNDaXBoZXJTdWl0ZXM6Ci0gVExTX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19BRVNfMjU2X0dDTV9TSEEzODQKLSBUTFNfQ0hBQ0hBMjBfUE9MWTEzMDVfU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQ0hBQ0hBMjBfUE9MWTEzMDUKLSBUTFNfRUNESEVfRUNEU0FfV0lUSF9DSEFDSEEyMF9QT0xZMTMwNV9TSEEyNTYKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX1JTQV9XSVRIX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19FQ0RIRV9SU0FfV0lUSF9BRVNfMjU2X0NCQ19TSEEKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1X1NIQTI1NgotIFRMU19SU0FfV0lUSF9BRVNfMTI4X0NCQ19TSEEKLSBUTFNfUlNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX1JTQV9XSVRIX0FFU18yNTZfQ0JDX1NIQQotIFRMU19SU0FfV0lUSF9BRVNfMjU2X0dDTV9TSEEzODQKc2VyaWFsaXplSW1hZ2VQdWxsczogZmFsc2UKc3RyZWFtaW5nQ29ubmVjdGlvbklkbGVUaW1lb3V0OiAxaAphbGxvd2VkVW5zYWZlU3lzY3RsczoKLSAibmV0LioiCg==
      - path: /etc/systemd/logind.conf.d/zzz-kubelet-graceful-shutdown.conf
        permissions: 0700
        encoding: base64
        content: W0xvZ2luXQojIGRlbGF5CkluaGliaXREZWxheU1heFNlYz0zMDAK
      - path: /etc/teleport-join-token
        permissions: 0644
        contentFrom:
          secret:
            name: test-wc-minimal-teleport-join-token
            key: joinToken
      - path: /opt/teleport-node-role.sh
        permissions: 0755
        encoding: base64
        content: IyEvYmluL2Jhc2gKCmlmIHN5c3RlbWN0bCBpcy1hY3RpdmUgLXEga3ViZWxldC5zZXJ2aWNlOyB0aGVuCiAgICBpZiBbIC1lICIvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzL2t1YmUtYXBpc2VydmVyLnlhbWwiIF07IHRoZW4KICAgICAgICBlY2hvICJjb250cm9sLXBsYW5lIgogICAgZWxzZQogICAgICAgIGVjaG8gIndvcmtlciIKICAgIGZpCmVsc2UKICAgIGVjaG8gIiIKZmkK
      - path: /etc/teleport.yaml
        permissions: 0644
        encoding: base64
        content: dmVyc2lvbjogdjMKdGVsZXBvcnQ6CiAgZGF0YV9kaXI6IC92YXIvbGliL3RlbGVwb3J0CiAgam9pbl9wYXJhbXM6CiAgICB0b2tlbl9uYW1lOiAvZXRjL3RlbGVwb3J0LWpvaW4tdG9rZW4KICAgIG1ldGhvZDogdG9rZW4KICBwcm94eV9zZXJ2ZXI6IHRlbGVwb3J0LmdpYW50c3dhcm0uaW86NDQzCiAgbG9nOgogICAgb3V0cHV0OiBzdGRlcnIKYXV0aF9zZXJ2aWNlOgogIGVuYWJsZWQ6ICJubyIKc3NoX3NlcnZpY2U6CiAgZW5hYmxlZDogInllcyIKICBjb21tYW5kczoKICAtIG5hbWU6IG5vZGUKICAgIGNvbW1hbmQ6IFtob3N0bmFtZV0KICAgIHBlcmlvZDogMjRoMG0wcwogIC0gbmFtZTogYXJjaAogICAgY29tbWFuZDogW3VuYW1lLCAtbV0KICAgIHBlcmlvZDogMjRoMG0wcwogIC0gbmFtZTogcm9sZQogICAgY29tbWFuZDogWy9vcHQvdGVsZXBvcnQtbm9kZS1yb2xlLnNoXQogICAgcGVyaW9kOiAxbTBzCiAgbGFiZWxzOgogICAgaW5zOiB0ZXN0CiAgICBtYzogdGVzdAogICAgY2x1c3RlcjogdGVzdC13Yy1taW5pbWFsCiAgICBiYXNlRG9tYWluOiBleGFtcGxlLmNvbQpwcm94eV9zZXJ2aWNlOgogIGVuYWJsZWQ6ICJubyIK
      - path: /etc/audit/rules.d/99-default.rules
        permissions: 0640
        encoding: base64
        content: IyBPdmVycmlkZGVuIGJ5IEdpYW50IFN3YXJtLgotYSBleGl0LGFsd2F5cyAtRiBhcmNoPWI2NCAtUyBleGVjdmUgLWsgYXVkaXRpbmcKLWEgZXhpdCxhbHdheXMgLUYgYXJjaD1iMzIgLVMgZXhlY3ZlIC1rIGF1ZGl0aW5nCg==
      - contentFrom:
          secret:
            name: test-wc-minimal-provider-specific-files-4
            key: kubelet-aws-config.sh
        path: /opt/bin/kubelet-aws-config.sh
        permissions: 0755
      - contentFrom:
          secret:
            name: test-wc-minimal-provider-specific-files-4
            key: kubelet-aws-config.service
        path: /etc/systemd/system/kubelet-aws-config.service
        permissions: 0644
      - contentFrom:
          secret:
            name: test-wc-minimal-provider-specific-files-4
            key: 99-unmanaged-devices.network
        path: /etc/systemd/network/99-unmanaged-devices.network
        permissions: 0644
      - path: /etc/containerd/config.toml
        permissions: 0644
        contentFrom:
          secret:
            name: test-wc-minimal-pool0-containerd-dec40c1e
            key: config.toml
    
  

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-cert-manager-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-cert-manager-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-cluster-autoscaler-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-cluster-autoscaler-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-etcd-k8s-res-count-exporter-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-etcd-k8s-res-count-exporter-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-external-dns-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-external-dns-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-metrics-server-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-metrics-server-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-net-exporter-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-net-exporter-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (v1/ConfigMap/org-giantswarm/test-wc-minimal-security-bundle-user-values)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-security-bundle-user-values)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-aws-ebs-csi-driver-servicemonitors-user-values)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-aws-nth-bundle-user-values)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-aws-pod-identity-webhook-user-values)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/helm.sh/chart  (v1/ConfigMap/org-giantswarm/test-wc-minimal-irsa-servicemonitors-user-values)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/helm.sh/chart  (infrastructure.cluster.x-k8s.io/v1beta2/AWSCluster/org-giantswarm/test-wc-minimal)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/app.kubernetes.io/version  (infrastructure.cluster.x-k8s.io/v1beta2/AWSCluster/org-giantswarm/test-wc-minimal)
  ± value change
    - 2.4.0
    + 1.3.5

/metadata/labels/helm.sh/chart  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachinePool/org-giantswarm/test-wc-minimal-pool0)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/app.kubernetes.io/version  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachinePool/org-giantswarm/test-wc-minimal-pool0)
  ± value change
    - 2.4.0
    + 1.3.5

/spec/refreshPreferences  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachinePool/org-giantswarm/test-wc-minimal-pool0)
  - one map entry removed:
    maxHealthyPercentage:

/metadata/labels/helm.sh/chart  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachineTemplate/org-giantswarm/test-wc-minimal-control-plane-2073bd83)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/app.kubernetes.io/version  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachineTemplate/org-giantswarm/test-wc-minimal-control-plane-2073bd83)
  ± value change
    - 2.4.0
    + 1.3.5

/spec/template/metadata/labels/helm.sh/chart  (infrastructure.cluster.x-k8s.io/v1beta2/AWSMachineTemplate/org-giantswarm/test-wc-minimal-control-plane-2073bd83)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-capi-node-labeler)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-capi-node-labeler)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cert-exporter)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cert-exporter)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cert-manager)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cert-manager)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-chart-operator-extensions)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-chart-operator-extensions)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cilium-servicemonitors)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cilium-servicemonitors)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cluster-autoscaler)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-cluster-autoscaler)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-etcd-k8s-res-count-exporter)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-etcd-k8s-res-count-exporter)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-external-dns)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-external-dns)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-k8s-audit-metrics)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-k8s-audit-metrics)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-k8s-dns-node-cache)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-k8s-dns-node-cache)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-metrics-server)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-metrics-server)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-net-exporter)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-net-exporter)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-node-exporter)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-node-exporter)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-observability-bundle)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-observability-bundle)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-prometheus-blackbox-exporter)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-prometheus-blackbox-exporter)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-security-bundle)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-security-bundle)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-teleport-kube-agent)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-teleport-kube-agent)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-vertical-pod-autoscaler)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-vertical-pod-autoscaler)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-aws-ebs-csi-driver-smons)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/spec/catalog  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-aws-ebs-csi-driver-smons)
  ± type change from <nil> to string
    - <nil>
    + default

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-aws-nth-bundle)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-aws-pod-identity-webhook)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/spec/catalog  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-aws-pod-identity-webhook)
  ± type change from <nil> to string
    - <nil>
    + default

/metadata/labels/helm.sh/chart  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-irsa-servicemonitors)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/spec/catalog  (application.giantswarm.io/v1alpha1/App/org-giantswarm/test-wc-minimal-irsa-servicemonitors)
  ± type change from <nil> to string
    - <nil>
    + default

/metadata/labels/app.kubernetes.io/version  (cluster.x-k8s.io/v1beta1/Cluster/org-giantswarm/test-wc-minimal)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (cluster.x-k8s.io/v1beta1/Cluster/org-giantswarm/test-wc-minimal)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cilium)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cilium)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/spec/values/defaultPolicies  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cilium)
  + three map entries added:
    enabled: false
    remove: true
    tolerations:
    - effect: NoSchedule
      operator: Exists
    - effect: NoExecute
      operator: Exists
    - key: CriticalAddonsOnly
      operator: Exists

/spec/values/extraPolicies  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cilium)
  + three map entries added:
    allowEgressToCoreDNS:
      enabled: false
    allowEgressToProxy:
      enabled: false
      httpProxy: null
      httpsProxy: null
    remove: true

/spec/values/kubeProxyReplacement  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cilium)
  ± value change
    - true
    + strict

/metadata/labels/app.kubernetes.io/version  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-coredns)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-coredns)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-network-policies)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-network-policies)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-vertical-pod-autoscaler-crd)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-vertical-pod-autoscaler-crd)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-aws-ebs-csi-driver)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/spec/chart/spec/sourceRef/name  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-aws-ebs-csi-driver)
  ± value change
    - test-wc-minimal-
    + test-wc-minimal-default

/metadata/labels/helm.sh/chart  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cloud-provider-aws)
  ± value change
    - cluster-aws-2.4.0
    + cluster-aws-1.3.5

/spec/chart/spec/sourceRef/name  (helm.toolkit.fluxcd.io/v2beta1/HelmRelease/org-giantswarm/test-wc-minimal-cloud-provider-aws)
  ± value change
    - test-wc-minimal-
    + test-wc-minimal-default

/metadata/labels/app.kubernetes.io/version  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-default)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-default)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-default-test)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-default-test)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-cluster)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-cluster)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-cluster-test)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (source.toolkit.fluxcd.io/v1beta2/HelmRepository/org-giantswarm/test-wc-minimal-cluster-test)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (controlplane.cluster.x-k8s.io/v1beta1/KubeadmControlPlane/org-giantswarm/test-wc-minimal)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (controlplane.cluster.x-k8s.io/v1beta1/KubeadmControlPlane/org-giantswarm/test-wc-minimal)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/spec/machineTemplate/metadata/labels/app.kubernetes.io/version  (controlplane.cluster.x-k8s.io/v1beta1/KubeadmControlPlane/org-giantswarm/test-wc-minimal)
  ± value change
    - 1.7.0
    + 1.0.3

/spec/machineTemplate/metadata/labels/helm.sh/chart  (controlplane.cluster.x-k8s.io/v1beta1/KubeadmControlPlane/org-giantswarm/test-wc-minimal)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/spec/kubeadmConfigSpec/ignition/containerLinuxConfig/additionalConfig  (controlplane.cluster.x-k8s.io/v1beta1/KubeadmControlPlane/org-giantswarm/test-wc-minimal)
  ± value change in multiline text (two inserts, one deletion)
    +   - name: teleport.service
    +     enabled: true
    +     contents: |
    +       [Unit]
    +       Description=Teleport Service
    +       After=network.target
    +       [Service]
    +       Type=simple
    +       Restart=on-failure
    +       ExecStart=/opt/bin/teleport start --roles=node --config=/etc/teleport.yaml --pid-file=/run/teleport.pid
    +       ExecReload=/bin/kill -HUP $MAINPID
    +       PIDFile=/run/teleport.pid
    +       LimitNOFILE=524288
    +       [Install]
    +       WantedBy=multi-user.target
    -         Restart=on-failure
    -   - name: teleport.service
    -     enabled: true
    -     contents: |
    -       [Unit]
    -       Description=Teleport Service
    -       After=network.target
    -       [Service]
    -       Type=simple
    -       Restart=on-failure
    -       ExecStart=/opt/bin/teleport start --roles=node --config=/etc/teleport.yaml --pid-file=/run/teleport.pid
    -       ExecReload=/bin/kill -HUP $MAINPID
    -       PIDFile=/run/teleport.pid
    -       LimitNOFILE=524288
    -       [Install]
    -       WantedBy=multi-user.target      
    +         Restart=on-failure      
  
  

/metadata/labels/app.kubernetes.io/version  (cluster.x-k8s.io/v1beta1/MachineHealthCheck/org-giantswarm/test-wc-minimal-control-plane)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (cluster.x-k8s.io/v1beta1/MachineHealthCheck/org-giantswarm/test-wc-minimal-control-plane)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (cluster.x-k8s.io/v1beta1/MachinePool/org-giantswarm/test-wc-minimal-pool0)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (cluster.x-k8s.io/v1beta1/MachinePool/org-giantswarm/test-wc-minimal-pool0)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/spec/template/spec/bootstrap/configRef/name  (cluster.x-k8s.io/v1beta1/MachinePool/org-giantswarm/test-wc-minimal-pool0)
  ± value change
    - test-wc-minimal-pool0-fc8a3
    + test-wc-minimal-pool0-f2de8

/metadata/labels/app.kubernetes.io/version  (v1/ServiceAccount/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (v1/ServiceAccount/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (rbac.authorization.k8s.io/v1/Role/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (rbac.authorization.k8s.io/v1/Role/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (rbac.authorization.k8s.io/v1/RoleBinding/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (rbac.authorization.k8s.io/v1/RoleBinding/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/metadata/labels/app.kubernetes.io/version  (batch/v1/Job/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - 1.7.0
    + 1.0.3

/metadata/labels/helm.sh/chart  (batch/v1/Job/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3

/spec/template/metadata/labels/app.kubernetes.io/version  (batch/v1/Job/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - 1.7.0
    + 1.0.3

/spec/template/metadata/labels/helm.sh/chart  (batch/v1/Job/org-giantswarm/test-wc-minimal-helmreleases-cleanup)
  ± value change
    - cluster-1.7.0
    + cluster-1.0.3



=== Differences when rendered with values file helm/cluster-aws/ci/test-eni-mode-values.yaml ===

(file level)
  - two documents removed:
    ---
    # Source: cluster-aws/charts/cluster/templates/apps/apps.yaml
    apiVersion: application.giantswarm.io/v1alpha1
    kind: App
    metadata:
      name: test-wc-observability-policies
      namespace: org-giantswarm
      labels:
        # deprecated: "app: cluster-aws" label is deprecated and it will be removed after upgrading
    # to Kubernetes 1.25. We still need it here because existing ClusterResourceSet selectors
    # need this label on the Cluster resource.
    app: cluster-aws
        app.kubernetes.io/name: cluster
        app.kubernetes.io/version: 1.7.0
        app.kubernetes.io/part-of: cluster-aws
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        helm.sh/chart: cluster-1.7.0
        application.giantswarm.io/team: turtles
        giantswarm.io/cluster: test-wc
        giantswarm.io/organization: test
        giantswarm.io/service-priority: highest
        cluster.x-k8s.io/cluster-name: test-wc
        cluster.x-k8s.io/watch-filter: capi
        release.giantswarm.io/version: 27.0.0-alpha.1
        giantswarm.io/managed-by: cluster
    spec:
      name: observability-policies
      version: N/A
      catalog: default
      namespace: kube-system
      kubeConfig:
        context:
          name: test-wc-admin@test-wc
        inCluster: false
        secret:
          name: test-wc-kubeconfig
          namespace: org-giantswarm
    # Source: cluster-aws/charts/cluster/templates/clusterapi/workers/kubeadmconfig.yaml
    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
    kind: KubeadmConfig
    metadata:
      annotations:
        machine-pool.giantswarm.io/name: test-wc-pool0
      labels:
        # deprecated: "app: cluster-aws" label is deprecated and it will be removed after upgrading
    # to Kubernetes 1.25. We still need it here because existing ClusterResourceSet selectors
    # need this label on the Cluster resource.
    app: cluster-aws
        app.kubernetes.io/name: cluster
        app.kubernetes.io/version: 1.7.0
        app.kubernetes.io/part-of: cluster-aws
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        helm.sh/chart: cluster-1.7.0
        application.giantswarm.io/team: turtles
        giantswarm.io/cluster: test-wc
        giantswarm.io/organization: test
        giantswarm.io/service-priority: highest
        cluster.x-k8s.io/cluster-name: test-wc
        cluster.x-k8s.io/watch-filter: capi
        release.giantswarm.io/version: 27.0.0-alpha.1
        giantswarm.io/machine-pool: test-wc-pool0
      name: test-wc-pool0-3f8fc
      namespace: org-giantswarm
    spec:
      format: ignition
      ignition:
        containerLinuxConfig:
          additionalConfig: |
            systemd:
              units:      
              - name: os-hardening.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Apply os hardening
                  [Service]
                  Type=oneshot
                  ExecStartPre=-/bin/bash -c "gpasswd -d core rkt; gpasswd -d core docker; gpasswd -d core wheel"
                  ExecStartPre=/bin/bash -c "until [ -f '/etc/sysctl.d/hardening.conf' ]; do echo Waiting for sysctl file; sleep 1s;done;"
                  ExecStart=/usr/sbin/sysctl -p /etc/sysctl.d/hardening.conf
                  [Install]
                  WantedBy=multi-user.target
              - name: update-engine.service
                enabled: false
                mask: true
              - name: locksmithd.service
                enabled: false
                mask: true
              - name: sshkeys.service
                enabled: false
                mask: true
              - name: kubeadm.service
                dropins:
                - name: 10-flatcar.conf
                  contents: |
                    [Unit]
                    # kubeadm must run after coreos-metadata populated /run/metadata directory.
                    Requires=coreos-metadata.service
                    After=coreos-metadata.service
                    # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939.
                    After=containerd.service
                    # kubeadm requires having an IP
                    After=network-online.target
                    Wants=network-online.target
                    [Service]
                    # Ensure kubeadm service has access to kubeadm binary in /opt/bin on Flatcar.
                    Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/bin
                    # To make metadata environment variables available for pre-kubeadm commands.
                    EnvironmentFile=/run/metadata/*
              - name: containerd.service
                enabled: true
                contents: |
                dropins:
                - name: 10-change-cgroup.conf
                  contents: |
                    [Service]
                    CPUAccounting=true
                    MemoryAccounting=true
                    Slice=kubereserved.slice
              - name: auditd.service
                enabled: false
              - name: teleport.service
                enabled: true
                contents: |
                  [Unit]
                  Description=Teleport Service
                  After=network.target
                  [Service]
                  Type=simple
                  Restart=on-failure
                  ExecStart=/opt/bin/teleport start --roles=node --config=/etc/teleport.yaml --pid-file=/run/teleport.pid
                  ExecReload=/bin/kill -HUP $MAINPID
                  PIDFile=/run/teleport.pid
                  LimitNOFILE=524288
                  [Install]
                  WantedBy=multi-user.target      
              - name: kubelet-aws-config.service
                enabled: true
              - name: var-lib.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=lib volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/lib
                  Where=/var/lib
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
              - name: var-log.mount
                enabled: true
                contents: |
                  [Unit]
                  Description=log volume
                  DefaultDependencies=no
                  [Mount]
                  What=/dev/disk/by-label/log
                  Where=/var/log
                  Type=xfs
                  [Install]
                  WantedBy=local-fs-pre.target
            storage:
              filesystems:      
              - name: lib
                mount:
                  device: /dev/xvdd
                  format: xfs
                  wipeFilesystem: true
                  label: lib
              - name: log
                mount:
                  device: /dev/xvde
                  format: xfs
                  wipeFilesystem: true
                  label: log
              directories:      
              - path: /var/lib/kubelet
                mode: 0750      
      joinConfiguration:
        nodeRegistration:
          name: ${COREOS_EC2_HOSTNAME}
          kubeletExtraArgs:
            cloud-provider: external
            healthz-bind-address: 0.0.0.0
            node-ip: ${COREOS_EC2_IPV4_LOCAL}
            node-labels: "ip=${COREOS_EC2_IPV4_LOCAL},role=worker,giantswarm.io/machine-pool=test-wc-pool0"
            v: 2
        patches:
          directory: /etc/kubernetes/patches
      preKubeadmCommands:
      - "envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp"
      - "mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml"
      - "systemctl restart containerd"
      files:
      - path: /etc/sysctl.d/hardening.conf
        permissions: 0644
        encoding: base64
        content: ZnMuaW5vdGlmeS5tYXhfdXNlcl93YXRjaGVzID0gMTYzODQKZnMuaW5vdGlmeS5tYXhfdXNlcl9pbnN0YW5jZXMgPSA4MTkyCmtlcm5lbC5rcHRyX3Jlc3RyaWN0ID0gMgprZXJuZWwuc3lzcnEgPSAwCm5ldC5pcHY0LmNvbmYuYWxsLmxvZ19tYXJ0aWFucyA9IDEKbmV0LmlwdjQuY29uZi5hbGwuc2VuZF9yZWRpcmVjdHMgPSAwCm5ldC5pcHY0LmNvbmYuZGVmYXVsdC5hY2NlcHRfcmVkaXJlY3RzID0gMApuZXQuaXB2NC5jb25mLmRlZmF1bHQubG9nX21hcnRpYW5zID0gMQpuZXQuaXB2NC50Y3BfdGltZXN0YW1wcyA9IDAKbmV0LmlwdjYuY29uZi5hbGwuYWNjZXB0X3JlZGlyZWN0cyA9IDAKbmV0LmlwdjYuY29uZi5kZWZhdWx0LmFjY2VwdF9yZWRpcmVjdHMgPSAwCiMgSW5jcmVhc2VkIG1tYXBmcyBiZWNhdXNlIHNvbWUgYXBwbGljYXRpb25zLCBsaWtlIEVTLCBuZWVkIGhpZ2hlciBsaW1pdCB0byBzdG9yZSBkYXRhIHByb3Blcmx5CnZtLm1heF9tYXBfY291bnQgPSAyNjIxNDQKIyBSZXNlcnZlZCB0byBhdm9pZCBjb25mbGljdHMgd2l0aCBrdWJlLWFwaXNlcnZlciwgd2hpY2ggYWxsb2NhdGVzIHdpdGhpbiB0aGlzIHJhbmdlCm5ldC5pcHY0LmlwX2xvY2FsX3Jlc2VydmVkX3BvcnRzPTMwMDAwLTMyNzY3Cm5ldC5pcHY0LmNvbmYuYWxsLnJwX2ZpbHRlciA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2lnbm9yZSA9IDEKbmV0LmlwdjQuY29uZi5hbGwuYXJwX2Fubm91bmNlID0gMgoKIyBUaGVzZSBhcmUgcmVxdWlyZWQgZm9yIHRoZSBrdWJlbGV0ICctLXByb3RlY3Qta2VybmVsLWRlZmF1bHRzJyBmbGFnCiMgU2VlIGh0dHBzOi8vZ2l0aHViLmNvbS9naWFudHN3YXJtL2dpYW50c3dhcm0vaXNzdWVzLzEzNTg3CnZtLm92ZXJjb21taXRfbWVtb3J5PTEKa2VybmVsLnBhbmljPTEwCmtlcm5lbC5wYW5pY19vbl9vb3BzPTEK
      - path: /etc/selinux/config
        permissions: 0644
        encoding: base64
        content: IyBUaGlzIGZpbGUgY29udHJvbHMgdGhlIHN0YXRlIG9mIFNFTGludXggb24gdGhlIHN5c3RlbSBvbiBib290LgoKIyBTRUxJTlVYIGNhbiB0YWtlIG9uZSBvZiB0aGVzZSB0aHJlZSB2YWx1ZXM6CiMgICAgICAgZW5mb3JjaW5nIC0gU0VMaW51eCBzZWN1cml0eSBwb2xpY3kgaXMgZW5mb3JjZWQuCiMgICAgICAgcGVybWlzc2l2ZSAtIFNFTGludXggcHJpbnRzIHdhcm5pbmdzIGluc3RlYWQgb2YgZW5mb3JjaW5nLgojICAgICAgIGRpc2FibGVkIC0gTm8gU0VMaW51eCBwb2xpY3kgaXMgbG9hZGVkLgpTRUxJTlVYPXBlcm1pc3NpdmUKCiMgU0VMSU5VWFRZUEUgY2FuIHRha2Ugb25lIG9mIHRoZXNlIGZvdXIgdmFsdWVzOgojICAgICAgIHRhcmdldGVkIC0gT25seSB0YXJnZXRlZCBuZXR3b3JrIGRhZW1vbnMgYXJlIHByb3RlY3RlZC4KIyAgICAgICBzdHJpY3QgICAtIEZ1bGwgU0VMaW51eCBwcm90ZWN0aW9uLgojICAgICAgIG1scyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1MZXZlbCBTZWN1cml0eQojICAgICAgIG1jcyAgICAgIC0gRnVsbCBTRUxpbnV4IHByb3RlY3Rpb24gd2l0aCBNdWx0aS1DYXRlZ29yeSBTZWN1cml0eQojICAgICAgICAgICAgICAgICAgKG1scywgYnV0IG9ubHkgb25lIHNlbnNpdGl2aXR5IGxldmVsKQpTRUxJTlVYVFlQRT1tY3MK
      - path: /etc/systemd/timesyncd.conf
        permissions: 0644
        encoding: base64
        content: W1RpbWVdCk5UUD0xNjkuMjU0LjE2OS4xMjMK
      - path: /etc/kubernetes/patches/kubeletconfiguration.yaml
        permissions: 0644
        encoding: base64
        content: YXBpVmVyc2lvbjoga3ViZWxldC5jb25maWcuazhzLmlvL3YxYmV0YTEKa2luZDogS3ViZWxldENvbmZpZ3VyYXRpb24Kc2h1dGRvd25HcmFjZVBlcmlvZDogMzAwcwpzaHV0ZG93bkdyYWNlUGVyaW9kQ3JpdGljYWxQb2RzOiA2MHMKa2VybmVsTWVtY2dOb3RpZmljYXRpb246IHRydWUKZXZpY3Rpb25Tb2Z0OgogIG1lbW9yeS5hdmFpbGFibGU6ICI1MDBNaSIKZXZpY3Rpb25IYXJkOgogIG1lbW9yeS5hdmFpbGFibGU6ICIyMDBNaSIKICBpbWFnZWZzLmF2YWlsYWJsZTogIjE1JSIKZXZpY3Rpb25Tb2Z0R3JhY2VQZXJpb2Q6CiAgbWVtb3J5LmF2YWlsYWJsZTogIjVzIgpldmljdGlvbk1heFBvZEdyYWNlUGVyaW9kOiA2MAprdWJlUmVzZXJ2ZWQ6CiAgY3B1OiAzNTBtCiAgbWVtb3J5OiAxMjgwTWkKICBlcGhlbWVyYWwtc3RvcmFnZTogMTAyNE1pCmt1YmVSZXNlcnZlZENncm91cDogL2t1YmVyZXNlcnZlZC5zbGljZQpwcm90ZWN0S2VybmVsRGVmYXVsdHM6IHRydWUKc3lzdGVtUmVzZXJ2ZWQ6CiAgY3B1OiAyNTBtCiAgbWVtb3J5OiAzODRNaQpzeXN0ZW1SZXNlcnZlZENncm91cDogL3N5c3RlbS5zbGljZQp0bHNDaXBoZXJTdWl0ZXM6Ci0gVExTX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19BRVNfMjU2X0dDTV9TSEEzODQKLSBUTFNfQ0hBQ0hBMjBfUE9MWTEzMDVfU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9DQkNfU0hBCi0gVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX0VDRFNBX1dJVEhfQ0hBQ0hBMjBfUE9MWTEzMDUKLSBUTFNfRUNESEVfRUNEU0FfV0lUSF9DSEFDSEEyMF9QT0xZMTMwNV9TSEEyNTYKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzEyOF9DQkNfU0hBCi0gVExTX0VDREhFX1JTQV9XSVRIX0FFU18xMjhfR0NNX1NIQTI1NgotIFRMU19FQ0RIRV9SU0FfV0lUSF9BRVNfMjU2X0NCQ19TSEEKLSBUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzI1Nl9HQ01fU0hBMzg0Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1Ci0gVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1X1NIQTI1NgotIFRMU19SU0FfV0lUSF9BRVNfMTI4X0NCQ19TSEEKLSBUTFNfUlNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2Ci0gVExTX1JTQV9XSVRIX0FFU18yNTZfQ0JDX1NIQQotIFRMU19SU0FfV0lUSF9BRVNfMjU2X0dDTV9TSEEzODQKc2VyaWFsaXplSW1hZ2VQdWxsczogZmFsc2UKc3RyZWFtaW5nQ29ubmVjdGlvbklkbGVUaW1lb3V0OiAxaAphbGxvd2VkVW5zYWZlU3lzY3RsczoKLSAibmV0LioiCg==
      - path: /etc/systemd/logind.conf.d/zzz-kubelet-graceful-shutdown.conf
        permissions: 0700
        encoding: base64
        content: W0xvZ2luXQojIGRlbGF5CkluaGliaXREZWxheU1heFNlYz0zMDAK
      - path: /etc/teleport-join-token
        permissions: 0644
        contentFrom:
          secret:
            name: test-wc-teleport-join-token
            key: joinToken
      - path: /opt/teleport-node-role.sh
        permissions: 0755
        encoding: base64
        content: IyEvYmluL2Jhc2gKCmlmIHN5c3RlbWN0bCBpcy1hY3RpdmUgLXEga3ViZWxldC5zZXJ2aWNlOyB0aGVuCiAgICBpZiBbIC1lICIvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzL2t1YmUtYXBpc2VydmVyLnlhbWwiIF07IHRoZW4KICAgICAgICBlY2hvICJjb250cm9sLXBsYW5lIgogICAgZWxzZQogICAgICAgIGVjaG8gIndvcmtlciIKICAgIGZpCmVsc2UKICAgIGVjaG8gIiIKZmkK
      - path: /etc/teleport.yaml
        permissions: 0644
        encoding: base64
        content: dmVyc2lvbjogdjMKdGVsZXBvcnQ6CiAgZGF0YV9kaXI6IC92YXIvbGliL3RlbGVwb3J0CiAgam9pbl9wYXJhbXM6CiAgICB0b2tlbl9uYW1lOiAvZXRjL3RlbGVwb3J0LWpvaW4tdG9rZW4KICAgIG1ldGhvZDogdG9rZW4KICBwcm94eV9zZXJ2ZXI6IHRlbGVwb3J0LmdpYW50c3dhcm0uaW86NDQz...*[Comment body truncated]*

[AndiDog]

Tested via giantswarm/releases#1518