diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index dbd8380c3..c3e779b38 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -23,6 +23,7 @@ - [Machine Health Checks](./topics/health-checking.md) - [Autoscaling](./topics/autoscaling.md) - [VPC](./topics/vpc.md) + - [Firewalling](./topics/firewalling.md) - [Development](./developers/development.md) - [Releasing](./developers/releasing.md) - [Testing](./developers/testing.md) diff --git a/docs/src/topics/firewalling.md b/docs/src/topics/firewalling.md new file mode 100644 index 000000000..abd92d764 --- /dev/null +++ b/docs/src/topics/firewalling.md @@ -0,0 +1,73 @@ +# Firewalling + +This guide covers how Cilium can be set up to act as a [host firewall](https://docs.cilium.io/en/latest/security/host-firewall/) on CAPL clusters. + +## Default Configuration +By default, the following policies are set to audit mode(without any enforcement) on CAPL clusters + +* [Kubeadm](./flavors/default.md) cluster allow rules + + | Ports | Use-case | Allowed clients | + |-----------|--------------------------|-----------------------| + | 6443 | API Server Traffic | World | + | 2379-2380 | Etcd Traffic | World | + | * | In Cluster Communication | Intra Cluster Traffic | + + +* [k3s](./flavors/k3s.md) cluster allow rules + + | Ports | Use-case | Allowed clients | + |-------|--------------------------|-------------------------------| + | 6443 | API Server Traffic | World | + | * | In Cluster Communication | Intra Cluster and VPC Traffic | + +* [RKE2](./flavors/rke2.md) cluster allow rules + + | Ports | Use-case | Allowed clients | + |-------|--------------------------|-------------------------------| + | 6443 | API Server Traffic | World | + | * | In Cluster Communication | Intra Cluster and VPC Traffic | + +## Enabling Firewall Enforcement +In order to turn the cilium network policy from audit to enforce mode use the environment variable `FW_AUDIT_ONLY=false` +when generating the cluster. This will set the [policy-audit-mode](https://docs.cilium.io/en/latest/security/policy-creation/#creating-policies-from-verdicts) +on the cilium deployment + +## Adding Additional Rules +Additional rules can be added to the `default-policy` +```yaml +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: "default-external-policy" +spec: + description: "allow cluster intra cluster traffic along api server traffic" + nodeSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - fromEntities: + - world + toPorts: + - ports: + - port: "22" # added for SSH Access to the nodes + - port: "6443" +``` +Alternatively, additional rules can be added by creating a new policy +```yaml +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: "ssh-access-policy" +spec: + description: "allows ssh access to nodes" + nodeSelector: {} + ingress: + - fromEntities: + - world + toPorts: + - ports: + - port: "22" +``` diff --git a/templates/addons/cilium-network-policies/ciliumNetworkPolicies.yaml b/templates/addons/cilium-network-policies/ciliumNetworkPolicies.yaml new file mode 100644 index 000000000..1400a242c --- /dev/null +++ b/templates/addons/cilium-network-policies/ciliumNetworkPolicies.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-cilium-policy +data: + cilium-policy.yaml: |- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-cluster-policy" + spec: + description: "allow cluster intra cluster traffic" + endpointSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - 192.168.128.0/17 + --- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-external-policy" + spec: + description: "allow etcd & api server traffic" + nodeSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - fromEntities: + - world + toPorts: + - ports: + - port: "6443" + - port: "2379" + - port: "2380" +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-cilium-policy +spec: + clusterSelector: + matchLabels: + cluster: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-cilium-policy + strategy: Reconcile diff --git a/templates/addons/cilium-network-policies/kustomization.yaml b/templates/addons/cilium-network-policies/kustomization.yaml new file mode 100644 index 000000000..8c9406bf4 --- /dev/null +++ b/templates/addons/cilium-network-policies/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ciliumNetworkPolicies.yaml diff --git a/templates/addons/cilium/cilium.yaml b/templates/addons/cilium/cilium.yaml index c611d3f0c..4db9f5e17 100644 --- a/templates/addons/cilium/cilium.yaml +++ b/templates/addons/cilium/cilium.yaml @@ -9,7 +9,7 @@ spec: repoURL: https://helm.cilium.io/ chartName: cilium namespace: kube-system - version: ${CILIUM_VERSION:=1.15.0} + version: ${CILIUM_VERSION:=1.15.4} options: waitForJobs: true wait: true @@ -23,6 +23,11 @@ spec: tunnelProtocol: "" enableIPv4Masquerade: true egressMasqueradeInterfaces: eth0 + policyAuditMode: ${FW_AUDIT_ONLY:=true} + hostFirewall: + enabled: true + extraConfig: + allow-localhost: policy k8sServiceHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }} k8sServicePort: {{ .InfraCluster.spec.controlPlaneEndpoint.port }} extraArgs: diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index b43a23aa0..9bfc48821 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -5,6 +5,7 @@ resources: - kubeadmConfigTemplate.yaml - kubeadmControlPlane.yaml - ../../addons/cilium + - ../../addons/cilium-network-policies - ../../addons/csi-driver-linode - ../../addons/ccm-linode - ../../addons/cluster-resource-set diff --git a/templates/flavors/k3s/ciliumNetworkPolicies.yaml b/templates/flavors/k3s/ciliumNetworkPolicies.yaml new file mode 100644 index 000000000..c0591b7b3 --- /dev/null +++ b/templates/flavors/k3s/ciliumNetworkPolicies.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-cilium-policy +data: + cilium-policy.yaml: |- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-cluster-policy" + spec: + description: "allow cluster intra cluster traffic" + endpointSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - 192.168.128.0/17 + --- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-external-policy" + spec: + description: "allow etcd & api server traffic" + nodeSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - fromEntities: + - world + toPorts: + - ports: + - port: "6443" +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-cilium-policy +spec: + clusterSelector: + matchLabels: + cluster: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-cilium-policy + strategy: Reconcile + diff --git a/templates/flavors/k3s/k3sControlPlane.yaml b/templates/flavors/k3s/k3sControlPlane.yaml index 1f9b7d55f..a7badd8c8 100644 --- a/templates/flavors/k3s/k3sControlPlane.yaml +++ b/templates/flavors/k3s/k3sControlPlane.yaml @@ -10,12 +10,14 @@ spec: name: ${CLUSTER_NAME}-control-plane kthreesConfigSpec: files: - - content: | + - path: /etc/rancher/k3s/config.yaml.d/capi-config.yaml + owner: root:root + content: | flannel-backend: none disable-network-policy: true + - path: /var/lib/rancher/k3s/server/manifests/ccm-linode.yaml owner: root:root - path: /etc/rancher/k3s/config.yaml.d/capi-config.yaml - - content: |- + content: |- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -36,17 +38,8 @@ spec: name: "linode-token-region" nodeSelector: node-role.kubernetes.io/control-plane: "true" + - path: /var/lib/rancher/k3s/server/manifests/cilium.yaml owner: root:root - path: /var/lib/rancher/k3s/server/manifests/ccm-linode.yaml - - contentFrom: - secret: - key: linode-token-region.yaml - name: linode-${CLUSTER_NAME}-crs-0 - owner: root:root - path: /var/lib/rancher/k3s/server/manifests/linode-token-region.yaml - - path: /var/lib/rancher/k3s/server/manifests/k3s-cilium-config.yaml - owner: root:root - permissions: "0640" content: |- apiVersion: helm.cattle.io/v1 kind: HelmChart @@ -55,11 +48,13 @@ spec: namespace: kube-system spec: targetNamespace: kube-system - version: ${CILIUM_VERSION:=1.15.0} + version: ${CILIUM_VERSION:=1.15.4} chart: cilium repo: https://helm.cilium.io/ bootstrap: true valuesContent: |- + bgpControlPlane: + enabled: true routingMode: native kubeProxyReplacement: true ipv4NativeRoutingCIDR: 10.0.0.0/8 @@ -68,9 +63,16 @@ spec: egressMasqueradeInterfaces: eth0 k8sServiceHost: 10.0.0.2 k8sServicePort: 6443 + policyAuditMode: ${FW_AUDIT_ONLY:=true} + hostFirewall: + enabled: true + extraConfig: + allow-localhost: policy + k8sServiceHost: 10.0.0.2 + k8sServicePort: 6443 extraArgs: - - --direct-routing-device=eth1 - - --nodeport-addresses=0.0.0.0/0 + - --direct-routing-device=eth1 + - --nodeport-addresses=0.0.0.0/0 ipam: mode: kubernetes ipv4: @@ -79,6 +81,17 @@ spec: enabled: false k8s: requireIPv4PodCIDR: true + hubble: + relay: + enabled: true + ui: + enabled: true + - path: /var/lib/rancher/k3s/server/manifests/linode-token-region.yaml + owner: root:root + contentFrom: + secret: + key: linode-token-region.yaml + name: linode-${CLUSTER_NAME}-crs-0 serverConfig: disableComponents: - servicelb diff --git a/templates/flavors/k3s/kustomization.yaml b/templates/flavors/k3s/kustomization.yaml index 76801b133..c542350e3 100644 --- a/templates/flavors/k3s/kustomization.yaml +++ b/templates/flavors/k3s/kustomization.yaml @@ -4,6 +4,7 @@ resources: - ../base - k3sControlPlane.yaml - k3sConfigTemplate.yaml + - ciliumNetworkPolicies.yaml - secret.yaml patches: - target: diff --git a/templates/flavors/rke2/ciliumNetworkPolicies.yaml b/templates/flavors/rke2/ciliumNetworkPolicies.yaml new file mode 100644 index 000000000..17ba7778e --- /dev/null +++ b/templates/flavors/rke2/ciliumNetworkPolicies.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-cilium-policy +data: + cilium-policy.yaml: |- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-cluster-policy" + spec: + description: "allow cluster intra cluster traffic" + endpointSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - 192.168.128.0/17 + --- + apiVersion: "cilium.io/v2" + kind: CiliumClusterwideNetworkPolicy + metadata: + name: "default-external-policy" + spec: + description: "allow etcd & api server traffic" + nodeSelector: {} + ingress: + - fromEntities: + - cluster + - fromCIDR: + - 10.0.0.0/8 + - fromEntities: + - world + toPorts: + - ports: + - port: "6443" +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-cilium-policy +spec: + clusterSelector: + matchLabels: + cluster: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-cilium-policy + strategy: Reconcile diff --git a/templates/flavors/rke2/kustomization.yaml b/templates/flavors/rke2/kustomization.yaml index c82618b80..212b243d5 100644 --- a/templates/flavors/rke2/kustomization.yaml +++ b/templates/flavors/rke2/kustomization.yaml @@ -2,10 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base + - ../../addons/cilium + - ciliumNetworkPolicies.yaml - rke2ControlPlane.yaml - rke2ConfigTemplate.yaml - secret.yaml - - ../../addons/cilium patches: - target: group: cluster.x-k8s.io diff --git a/templates/flavors/rke2/rke2ControlPlane.yaml b/templates/flavors/rke2/rke2ControlPlane.yaml index b4cf9203a..a5c6565c3 100644 --- a/templates/flavors/rke2/rke2ControlPlane.yaml +++ b/templates/flavors/rke2/rke2ControlPlane.yaml @@ -9,7 +9,9 @@ spec: kind: LinodeMachineTemplate name: ${CLUSTER_NAME}-control-plane files: - - content: |- + - path: /var/lib/rancher/rke2/server/manifests/ccm-linode.yaml + owner: root:root + content: |- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -30,8 +32,6 @@ spec: name: "linode-token-region" nodeSelector: node-role.kubernetes.io/control-plane: "true" - owner: root:root - path: /var/lib/rancher/rke2/server/manifests/ccm-linode.yaml - contentFrom: secret: key: linode-token-region.yaml