From 3667d282f6a965635928fb2819c0edd61746fc73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Nasiadka?= Date: Tue, 16 Feb 2021 11:17:21 +0100 Subject: [PATCH 01/17] Support extra_network and extra_subnet labels This allows users to add extra network interface to cluster nodes, e.g. storage network. Story: 2002448 Task: 21983 Co-Authored-By: Bharat Kunwar Change-Id: I10a6a4d72e9ec635f2c73d9fe64a8d136228f532 (cherry picked from commit bea8c723c37eff84201040ab053288873d00fcd3) (cherry picked from commit 8260f1156abba3f54963410746a5342c14cde83d) (cherry picked from commit d6c0a2d25ea684d9cc6751d02f2d2df7c7fd31fb) (cherry picked from commit 6227555f4bf4b5318ea5d2d6ecae13bde2d344c7) --- doc/source/user/index.rst | 23 ++++++- .../drivers/heat/k8s_fedora_template_def.py | 3 +- .../templates/kubecluster.yaml | 28 +++++++- .../templates/kubemaster.yaml | 64 +++++++++++++++++- .../templates/kubeminion.yaml | 65 ++++++++++++++++++- .../unit/drivers/test_template_definition.py | 13 ++++ ...ra-network-interface-68bc169cb467a13e.yaml | 6 ++ 7 files changed, 195 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index b58ac46980..7cb42e1195 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -484,6 +484,12 @@ the table are linked to more details elsewhere in the user guide. +---------------------------------------+--------------------+---------------+ | `octavia_lb_healthcheck`_ | see bellow | true | +---------------------------------------+--------------------+---------------+ +| `extra_network`_ | see below | "" | ++---------------------------------------+--------------------+---------------+ +| `extra_subnet`_ | see below | "" | ++---------------------------------------+--------------------+---------------+ +| `extra_security_group`_ | see below | see below | ++---------------------------------------+--------------------+---------------+ .. _cluster: @@ -1617,6 +1623,22 @@ _`octavia_lb_healthcheck` If true, enable Octavia load balancer healthcheck Default: true +_`extra_network` + Optional additional network name or UUID to add to cluster nodes. + When not specified, additional networks are not added. Optionally specify + 'extra_subnet' if you wish to use a specific subnet on the network. + Default: "" + +_`extra_subnet` + Optional additional subnet name or UUID to add to cluster nodes. + Only used when 'extra_network' is defined. + Default: "" + +_`extra_security_group` + Optional additional group name or UUID to add to network port. + Only used when 'extra_network' is defined. + Default: cluster node default security group. + Supported versions ------------------ @@ -2463,7 +2485,6 @@ _`calico_tag` Victoria default: v3.13.1 Wallaby default: v3.13.1 - Besides, the Calico network driver needs kube_tag with v1.9.3 or later, because Calico needs extra mounts for the kubelet container. See `commit `_ diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index a4ec6250ab..1c26d2a2da 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -127,7 +127,8 @@ def get_params(self, context, cluster_template, cluster, **kwargs): 'min_node_count', 'max_node_count', 'npd_enabled', 'ostree_remote', 'ostree_commit', 'use_podman', 'kube_image_digest', - 'metrics_scraper_tag'] + 'metrics_scraper_tag', + 'extra_network', 'extra_subnet', 'extra_security_group'] labels = self._get_relevant_labels(cluster, kwargs) diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 15bfd9af25..d7ca3972b8 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1008,13 +1008,32 @@ parameters: description: > Post install manifest URL used to setup some cloud provider/vendor specific configs - default: "" + default: '' master_lb_allowed_cidrs: type: comma_delimited_list description: The allowed CIDR list for master load balancer default: [] + extra_network: + type: string + description: > + Additional network to bind nodes to + default: '' + + extra_subnet: + type: string + description: > + Subnet for additional network + default: '' + + extra_security_group: + type: string + description: > + Additional security group name + default: '' + + resources: ###################################################################### @@ -1407,6 +1426,9 @@ resources: containerd_tarball_sha256: {get_param: containerd_tarball_sha256} post_install_manifest_url: {get_param: post_install_manifest_url} metrics_scraper_tag: {get_param: metrics_scraper_tag} + extra_network: {get_param: extra_network} + extra_subnet: {get_param: extra_subnet} + extra_security_group: {get_param: extra_security_group} kube_cluster_config: condition: create_cluster_resources @@ -1589,6 +1611,10 @@ resources: containerd_tarball_sha256: {get_param: containerd_tarball_sha256} kube_service_account_key: {get_param: kube_service_account_key} kube_service_account_private_key: {get_param: kube_service_account_private_key} + extra_network: {get_param: extra_network} + extra_subnet: {get_param: extra_subnet} + extra_security_group: {get_param: extra_security_group} + outputs: api_address: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index 917f010db8..c0381a9e9a 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -714,6 +714,21 @@ parameters: description: > Tag of metrics-scraper for kubernetes dashboard. + extra_network: + type: string + description: > + Additional network name to bind ports to instances + + extra_subnet: + type: string + description: > + Additional subnet name + + extra_security_group: + type: string + description: > + Additional security group name + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -723,6 +738,24 @@ conditions: - get_param: boot_volume_size - 0 + extra_network_defined: + not: + equals: + - get_param: extra_network + - '' + + extra_subnet_defined: + not: + equals: + - get_param: extra_subnet + - '' + + extra_security_group_defined: + not: + equals: + - get_param: extra_security_group + - '' + resources: ###################################################################### # @@ -985,7 +1018,12 @@ resources: software_config_transport: POLL_SERVER_HEAT user_data: {get_resource: agent_config} networks: - - port: {get_resource: kube_master_eth0} + list_concat: + - - port: {get_resource: kube_master_eth0} + - if: + - "extra_network_defined" + - - port: {get_resource: kube_master_eth1} + - [] scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} @@ -999,7 +1037,12 @@ resources: software_config_transport: POLL_SERVER_HEAT user_data: {get_resource: agent_config} networks: - - port: {get_resource: kube_master_eth0} + list_concat: + - - port: {get_resource: kube_master_eth0} + - if: + - "extra_network_defined" + - - port: {get_resource: kube_master_eth1} + - [] scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} block_device_mapping_v2: @@ -1018,6 +1061,23 @@ resources: - ip_address: {get_param: pods_network_cidr} replacement_policy: AUTO + kube_master_eth1: + type: OS::Neutron::Port + condition: extra_network_defined + properties: + network: {get_param: extra_network} + security_groups: + - if: + - "extra_security_group_defined" + - {get_param: extra_security_group} + - {get_param: secgroup_kube_master_id} + fixed_ips: + if: + - "extra_subnet_defined" + - - subnet: {get_param: extra_subnet} + - [] + replacement_policy: AUTO + kube_master_floating: type: Magnum::Optional::KubeMaster::Neutron::FloatingIP properties: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml index 3666b2cebf..3683cb6f1f 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml @@ -387,6 +387,21 @@ parameters: The private key will be used to sign generated k8s service account tokens. + extra_network: + type: string + description: > + Additional network name to bind ports to instances + + extra_subnet: + type: string + description: > + Additional subnet name + + extra_security_group: + type: string + description: > + Additional seurity group name + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -396,6 +411,25 @@ conditions: - get_param: boot_volume_size - 0 + extra_network_defined: + not: + equals: + - get_param: extra_network + - '' + + extra_subnet_defined: + not: + equals: + - get_param: extra_subnet + - '' + + extra_security_group_defined: + not: + equals: + - get_param: extra_security_group + - '' + + resources: agent_config: @@ -563,7 +597,12 @@ resources: user_data_format: SOFTWARE_CONFIG software_config_transport: POLL_SERVER_HEAT networks: - - port: {get_resource: kube_minion_eth0} + list_concat: + - - port: {get_resource: kube_minion_eth0} + - if: + - "extra_network_defined" + - - port: {get_resource: kube_minion_eth1} + - [] scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} @@ -577,7 +616,12 @@ resources: user_data_format: SOFTWARE_CONFIG software_config_transport: POLL_SERVER_HEAT networks: - - port: {get_resource: kube_minion_eth0} + list_concat: + - - port: {get_resource: kube_minion_eth0} + - if: + - "extra_network_defined" + - - port: {get_resource: kube_minion_eth1} + - [] scheduler_hints: { group: { get_param: nodes_server_group_id }} availability_zone: {get_param: availability_zone} block_device_mapping_v2: @@ -596,6 +640,23 @@ resources: - ip_address: {get_param: pods_network_cidr} replacement_policy: AUTO + kube_minion_eth1: + type: OS::Neutron::Port + condition: extra_network_defined + properties: + network: {get_param: extra_network} + security_groups: + - if: + - "extra_security_group_defined" + - get_param: extra_security_group + - get_param: secgroup_kube_minion_id + fixed_ips: + if: + - "extra_subnet_defined" + - - subnet: {get_param: extra_subnet} + - [] + replacement_policy: AUTO + kube_minion_floating: type: Magnum::Optional::KubeMinion::Neutron::FloatingIP properties: diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 46eabd1e64..fc9bdbeec8 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -615,6 +615,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key, octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm') octavia_lb_healthcheck = mock_cluster.labels.get( 'octavia_lb_healthcheck') + extra_network = mock_cluster.labels.get('extra_network') + extra_subnet = mock_cluster.labels.get('extra_subnet') + extra_security_group = mock_cluster.labels.get('extra_security_group') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -742,6 +745,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key, 'octavia_provider': octavia_provider, 'octavia_lb_algorithm': octavia_lb_algorithm, 'octavia_lb_healthcheck': octavia_lb_healthcheck, + 'extra_network': extra_network, + 'extra_subnet': extra_subnet, + 'extra_security_group': extra_security_group, }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, @@ -1175,6 +1181,10 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key, master_lb_allowed_cidrs = mock_cluster.labels.get( 'master_lb_allowed_cidrs') + extra_network = mock_cluster.labels.get('extra_network') + extra_subnet = mock_cluster.labels.get('extra_subnet') + extra_security_group = mock_cluster.labels.get('extra_security_group') + octavia_provider = mock_cluster.labels.get('octavia_provider') octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm') @@ -1309,6 +1319,9 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key, 'octavia_provider': octavia_provider, 'octavia_lb_algorithm': octavia_lb_algorithm, 'octavia_lb_healthcheck': octavia_lb_healthcheck, + 'extra_network': extra_network, + 'extra_subnet': extra_subnet, + 'extra_security_group': extra_security_group, }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, diff --git a/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml b/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml new file mode 100644 index 0000000000..be8ca5a7c5 --- /dev/null +++ b/releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Support extra_network and extra_subnet labels to allow users to assign + extra network interface to cluster nodes, e.g. storage network. Also adds + possibility to set custom security group on network port. From 4abaf43c5ce8f3c6d04893fc8c03ec8795b6eed2 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Thu, 7 Jan 2021 14:20:42 +0000 Subject: [PATCH 02/17] Update magnum k8s monitoring infra * Prometheus-server now runs only on master nodes. * Update prometheus-operator helm chart and tag. * Update prometheus-adapter version. * Deprecation notice for prometheus_monitoring component. task: 41569 story: 2006765 Signed-off-by: Diogo Guerra Change-Id: I05e8c2be4e4c8e66a166b485ec7851875dca8b1c (cherry picked from commit 0934160d22b6fb33e2971ccb9c55426d2a3c205a) (cherry picked from commit 9a476968e49e4030115eafd4fbc3e777de39f004) (cherry picked from commit 30419089b01963a946bad3a20f137f0fcabd398a) (cherry picked from commit 5f786f0d7528a820310a23438f2cc4f996b32f48) --- doc/source/user/index.rst | 15 +- doc/source/user/monitoring.rst | 19 +-- .../kubernetes/helm/prometheus-adapter.sh | 5 +- .../kubernetes/helm/prometheus-operator.sh | 132 +++++++++--------- .../templates/kubecluster.yaml | 12 +- .../templates/kubecluster.yaml | 12 +- ...te-monitoring-charts-1067dc4a0f0060b6.yaml | 10 ++ 7 files changed, 114 insertions(+), 91 deletions(-) create mode 100644 releasenotes/notes/update-monitoring-charts-1067dc4a0f0060b6.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 7cb42e1195..2b946b49ec 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -1230,13 +1230,14 @@ _`container_infra_prefix` Images that might be needed if 'monitoring_enabled' is 'true': - * quay.io/prometheus/alertmanager:v0.20.0 - * docker.io/squareup/ghostunnel:v1.5.2 - * docker.io/jettech/kube-webhook-certgen:v1.0.0 - * quay.io/coreos/prometheus-operator:v0.37.0 - * quay.io/coreos/configmap-reload:v0.0.1 - * quay.io/coreos/prometheus-config-reloader:v0.37.0 - * quay.io/prometheus/prometheus:v2.15.2 + * quay.io/prometheus/alertmanager:v0.21.0 + * docker.io/jettech/kube-webhook-certgen:v1.5.0 + * quay.io/prometheus-operator/prometheus-operator:v0.44.0 + * docker.io/jimmidyson/configmap-reload:v0.4.0 + * quay.io/prometheus-operator/prometheus-config-reloader:v0.44.0 + * quay.io/prometheus/prometheus:v2.22.1 + * quay.io/prometheus/node-exporter:v1.0.1 + * docker.io/directxman12/k8s-prometheus-adapter:v0.8.2 Images that might be needed if 'cinder_csi_enabled' is 'true': diff --git a/doc/source/user/monitoring.rst b/doc/source/user/monitoring.rst index b580edf438..53ccff77e4 100644 --- a/doc/source/user/monitoring.rst +++ b/doc/source/user/monitoring.rst @@ -35,15 +35,15 @@ _`metrics_server_enabled` _`monitoring_enabled` Enable installation of cluster monitoring solution provided by the - stable/prometheus-operator helm chart. + prometheus-community/kube-prometheus-stack helm chart. To use this service tiller_enabled must be true when using helm_client_tag> ${HELM_CHART_DIR}/values.yaml prometheus-adapter: image: - repository: ${CONTAINER_INFRA_PREFIX:-docker.io/directxman12/}k8s-prometheus-adapter-${ARCH} + repository: ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/prometheus-adapter/}prometheus-adapter priorityClassName: "system-cluster-critical" prometheus: - url: http://web.tcp.prometheus-prometheus.kube-system.svc.cluster.local + url: http://web.tcp.magnum-kube-prometheus-sta-prometheus.kube-system.svc.cluster.local + path: /prometheus resources: requests: cpu: 150m diff --git a/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh b/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh index 724975b8e2..4e3f1d709c 100755 --- a/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh +++ b/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh @@ -2,7 +2,7 @@ set +x . /etc/sysconfig/heat-params set -ex -CHART_NAME="prometheus-operator" +CHART_NAME="kube-prometheus-stack" if [ "$(echo ${MONITORING_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then echo "Writing ${CHART_NAME} config" @@ -80,22 +80,18 @@ EOF PROTOCOL="http" INSECURE_SKIP_VERIFY="True" fi - # FIXME: Force protocol to http as we don't want to use the cluster certs - USE_HTTPS="False" if [ "$(echo ${VERIFY_CA} | tr '[:upper:]' '[:lower:]')" == "false" ]; then INSECURE_SKIP_VERIFY="True" fi cat << EOF >> ${HELM_CHART_DIR}/values.yaml -prometheus-operator: - - defaultRules: - rules: - #TODO: To enable this we need firstly take care of exposing certs - etcd: false +kube-prometheus-stack: alertmanager: + podDisruptionBudget: + enabled: true + #config: ingress: enabled: ${MONITORING_INGRESS_ENABLED} annotations: @@ -108,6 +104,7 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} - ${CLUSTER_ROOT_DOMAIN_NAME} paths: - /alertmanager${APP_INGRESS_PATH_APPEND} + pathType: ImplementationSpecific ## TLS configuration for Alertmanager Ingress ## Secret must be manually created in the namespace tls: [] @@ -118,8 +115,8 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} image: repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}alertmanager logFormat: json + routePrefix: /alertmanager externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/alertmanager - # routePrefix: /alertmanager # resources: # requests: # cpu: 100m @@ -127,15 +124,7 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} priorityClassName: "system-cluster-critical" grafana: - image: - repository: ${CONTAINER_INFRA_PREFIX:-grafana/}grafana #enabled: ${ENABLE_GRAFANA} - sidecar: - image: ${CONTAINER_INFRA_PREFIX:-kiwigrid/}k8s-sidecar:0.1.99 - resources: - requests: - cpu: 100m - memory: 128Mi adminPassword: ${GRAFANA_ADMIN_PASSWD} ingress: enabled: ${MONITORING_INGRESS_ENABLED} @@ -146,13 +135,24 @@ ${APP_INGRESS_ANNOTATIONS} ## Must be provided if Ingress is enable. hosts: - ${CLUSTER_ROOT_DOMAIN_NAME} - path: /grafana${APP_INGRESS_PATH_APPEND} + paths: + - /grafana${APP_INGRESS_PATH_APPEND} + pathType: ImplementationSpecific ## TLS configuration for grafana Ingress ## Secret must be manually created in the namespace tls: [] # - secretName: grafana-general-tls # hosts: # - grafana.example.com + sidecar: + image: + repository: ${CONTAINER_INFRA_PREFIX:-quay.io/kiwigrid/}k8s-sidecar + image: + repository: ${CONTAINER_INFRA_PREFIX:-grafana/}grafana + resources: + requests: + cpu: 100m + memory: 128Mi persistence: enabled: ${APP_GRAFANA_PERSISTENT_STORAGE} storageClassName: ${MONITORING_STORAGE_CLASS_NAME} @@ -162,21 +162,10 @@ ${APP_INGRESS_ANNOTATIONS} domain: ${CLUSTER_ROOT_DOMAIN_NAME} root_url: https://${CLUSTER_ROOT_DOMAIN_NAME}/grafana serve_from_sub_path: true - paths: - data: /var/lib/grafana/data - logs: /var/log/grafana - plugins: /var/lib/grafana/plugins - provisioning: /etc/grafana/provisioning - analytics: - check_for_updates: true log: mode: console log.console: format: json - grafana_net: - url: https://grafana.net - plugins: - - grafana-piechart-panel kubeApiServer: tlsConfig: @@ -198,9 +187,9 @@ ${APP_INGRESS_ANNOTATIONS} serviceMonitor: ## Enable scraping kube-controller-manager over https. ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - https: ${USE_HTTPS} + https: "True" # Skip TLS certificate validation when scraping - insecureSkipVerify: null + insecureSkipVerify: "True" # Name of the server to use when validating TLS certificate serverName: null @@ -242,19 +231,21 @@ ${APP_INGRESS_ANNOTATIONS} serviceMonitor: ## Enable scraping kube-scheduler over https. ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - https: ${USE_HTTPS} + https: "True" ## Skip TLS certificate validation when scraping - insecureSkipVerify: null + insecureSkipVerify: "True" ## Name of the server to use when validating TLS certificate serverName: null - # kubeProxy: - # ## If your kube proxy is not deployed as a pod, specify IPs it can be found on - # endpoints: [] # masters + minions - # serviceMonitor: - # ## Enable scraping kube-proxy over https. - # ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - # https: ${USE_HTTPS} + kubeProxy: + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + endpoints: ${KUBE_MASTERS_PRIVATE} # masters + minions + serviceMonitor: + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + https: "True" + ## Skip TLS certificate validation when scraping + insecureSkipVerify: "True" kube-state-metrics: priorityClassName: "system-cluster-critical" @@ -271,37 +262,34 @@ ${APP_INGRESS_ANNOTATIONS} limits: cpu: 20m memory: 20M - extraArgs: - - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) - - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ - sidecars: [] - ## - name: nvidia-dcgm-exporter - ## image: nvidia/dcgm-exporter:1.4.3 prometheusOperator: - priorityClassName: "system-cluster-critical" - tlsProxy: - image: - repository: ${CONTAINER_INFRA_PREFIX:-squareup/}ghostunnel admissionWebhooks: patch: image: repository: ${CONTAINER_INFRA_PREFIX:-jettech/}kube-webhook-certgen - priorityClassName: "system-cluster-critical" - - resources: {} - # requests: - # cpu: 5m - # memory: 10Mi + resources: + requests: + cpu: 2m + limits: + memory: 30M + # clusterDomain: ${CLUSTER_ROOT_DOMAIN_NAME} + priorityClassName: "system-cluster-critical" logFormat: json + logLevel: info + resources: + requests: + cpu: 2m + limits: + memory: 32M image: - repository: ${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}prometheus-operator - configmapReloadImage: - repository: ${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}configmap-reload + repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus-operator/}prometheus-operator + prometheusDefaultBaseImage: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}prometheus + alertmanagerDefaultBaseImage: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}alertmanager prometheusConfigReloaderImage: - repository: ${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}prometheus-config-reloader - hyperkubeImage: - repository: ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube + repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus-operator/}prometheus-config-reloader + thanosImage: + repository: ${CONTAINER_INFRA_PREFIX:-quay.io/thanos/}thanos prometheus: ingress: @@ -317,6 +305,7 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} - ${CLUSTER_ROOT_DOMAIN_NAME} paths: - /prometheus${APP_INGRESS_PATH_APPEND} + pathType: ImplementationSpecific ## TLS configuration for Prometheus Ingress ## Secret must be manually created in the namespace tls: [] @@ -332,11 +321,13 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} bearerTokenFile: prometheusSpec: scrapeInterval: ${MONITORING_INTERVAL_SECONDS}s - scrapeInterval: 30s evaluationInterval: 30s image: repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}prometheus - retention: 14d + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" externalLabels: cluster_uuid: ${CLUSTER_UUID} externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/prometheus @@ -352,7 +343,16 @@ ${APP_INGRESS_BASIC_AUTH_ANNOTATIONS} retention: ${MONITORING_RETENTION_DAYS}d retentionSize: ${MONITORING_RETENTION_SIZE_GB}GB logFormat: json - #routePrefix: /prometheus + routePrefix: /prometheus + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: magnum.openstack.org/role + operator: In + values: + - master resources: requests: cpu: ${PROMETHEUS_SERVER_CPU}m diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 00769d1c95..03845c0b8e 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -740,8 +740,8 @@ parameters: prometheus_operator_chart_tag: type: string - description: The stable/prometheus-operator chart version to use. - default: v8.12.13 + description: The prometheus-community/kube-prometheus-stack chart version to use. + default: 17.2.0 prometheus_adapter_enabled: type: boolean @@ -750,8 +750,8 @@ parameters: prometheus_adapter_chart_tag: type: string - description: The stable/prometheus-adapter chart version to use. - default: 1.4.0 + description: The prometheus-community/prometheus-adapter chart version to use. + default: 2.5.1 prometheus_adapter_configmap: type: string @@ -1051,6 +1051,10 @@ resources: - protocol: udp port_range_min: 8472 port_range_max: 8472 + # Prometheus Server + - protocol: tcp + port_range_min: 9090 + port_range_max: 9090 secgroup_kube_minion: condition: create_cluster_resources diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index d7ca3972b8..a01afe8f7f 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -772,8 +772,8 @@ parameters: prometheus_operator_chart_tag: type: string - description: The stable/prometheus-operator chart version to use. - default: v8.12.13 + description: The prometheus-community/kube-prometheus-stack chart version to use. + default: 33.0.0 prometheus_adapter_enabled: type: boolean @@ -782,8 +782,8 @@ parameters: prometheus_adapter_chart_tag: type: string - description: The stable/prometheus-adapter chart version to use. - default: 1.4.0 + description: The prometheus-community/prometheus-adapter chart version to use. + default: 3.0.2 prometheus_adapter_configmap: type: string @@ -1131,6 +1131,10 @@ resources: - protocol: udp port_range_min: 8472 port_range_max: 8472 + # Prometheus Server + - protocol: tcp + port_range_min: 9090 + port_range_max: 9090 secgroup_kube_minion: condition: create_cluster_resources diff --git a/releasenotes/notes/update-monitoring-charts-1067dc4a0f0060b6.yaml b/releasenotes/notes/update-monitoring-charts-1067dc4a0f0060b6.yaml new file mode 100644 index 0000000000..174f6b8a3b --- /dev/null +++ b/releasenotes/notes/update-monitoring-charts-1067dc4a0f0060b6.yaml @@ -0,0 +1,10 @@ +--- +upgrade: + - Prometheus-Adapter helm chart updated to 2.12.1 from 1.4.0. + - Prometheus-Operator helm chart updated to kube-prometheus-stack:17.2.0 + from prometheus-operator:v8.12.13. + - Prometheus-server now runs only on master nodes + +deprecations: + - Enabling monitoring using the prometheus_monitoring label is deprecated + and will be removed in the X cycle. From abc814f6062b8e7201475aab3b8f00f0ee8eeb3c Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Mon, 7 Mar 2022 14:49:23 +0100 Subject: [PATCH 03/17] upgrade coredns Story: 2009874 Change-Id: Ibaa3b150e024918b6fc963eb6ca62bd18690a3c4 (cherry picked from commit 15c2bc236da5db48c89fe2881ac01a9d55e3d970) (cherry picked from commit bf7fd641368c226b94660dc98ac6b094f6e75837) (cherry picked from commit d7a502e84d27caddf507f4cac4ff7118a4951104) (cherry picked from commit eaaa762935862c23a8b8062b77c84da47a14c540) --- .../templates/kubernetes/fragments/core-dns-service.sh | 7 +++++-- .../k8s_fedora_coreos_v1/templates/kubecluster.yaml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh index 12fb9875cf..0851a74f0e 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh @@ -74,8 +74,11 @@ data: Corefile: | .:53 { errors - log - health + log stdout + health { + lameduck 5s + } + ready kubernetes ${DNS_CLUSTER_DOMAIN} ${PORTAL_NETWORK_CIDR} ${PODS_NETWORK_CIDR} { pods verified fallthrough in-addr.arpa ip6.arpa diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index a01afe8f7f..4c77e8c849 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -495,7 +495,7 @@ parameters: coredns_tag: type: string description: tag for coredns - default: 1.6.6 + default: 1.8.7 flannel_tag: type: string From d0e1aa238c5a7a8e78f42f654eb93b87db3035bb Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Thu, 7 Jan 2021 14:20:42 +0000 Subject: [PATCH 04/17] Update magnum k8s monitoring infra * Prometheus-server now runs only on master nodes. * Update prometheus-operator helm chart and tag. * Update prometheus-adapter version. * Deprecation notice for prometheus_monitoring component. Since v1.20 (changes are gradually introduced in next versions) k8s components are exposing (by default) metrics on 127.0.0.1 and are changing expose port. * Fix metrics address and ports post 1.20 * Extended memory limits to 64M * Changed expose addr to 127.0.0.1 * Changed expose ports task: 41569 story: 2006765 Signed-off-by: Diogo Guerra Co-authored-by: Grzegorz Bialas Change-Id: I05e8c2be4e4c8e66a166b485ec7851875dca8b1c (cherry picked from commit c92f605c7b7af43b7c4049af0e8c835abb32d6be) (cherry picked from commit 32b22e546575ee8a96408742d14478b3a6ed04ab) (cherry picked from commit 5b56182a1fd1f864f6f9b0afcbb1d24f0a08bb00) (cherry picked from commit 25e3bdc5ac1dd4bbfb5b222b56910ecc122d4be0) --- .../fragments/configure-kubernetes-master.sh | 6 ++-- .../kubernetes/helm/prometheus-operator.sh | 35 +++++-------------- .../templates/kubecluster.yaml | 12 +++++++ 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index c4a1f8e6ec..623f9b9405 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -269,7 +269,7 @@ CERT_DIR=/etc/kubernetes/certs # kube-proxy config PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml -KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}" +KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME} --metrics-bind-address=0.0.0.0" cat > /etc/kubernetes/proxy << EOF KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}" EOF @@ -404,6 +404,8 @@ KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/a KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}" +KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --secure-port=10257" +KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics" KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt" @@ -426,7 +428,7 @@ sed -i ' /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"# ' /etc/kubernetes/controller-manager -sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler +sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics "#' /etc/kubernetes/scheduler $ssh_cmd mkdir -p /etc/kubernetes/manifests KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}" diff --git a/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh b/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh index 4e3f1d709c..c2a6b37a0a 100755 --- a/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh +++ b/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh @@ -180,18 +180,10 @@ ${APP_INGRESS_ANNOTATIONS} endpoints: ${KUBE_MASTERS_PRIVATE} ## If using kubeControllerManager.endpoints only the port and targetPort are used service: - port: 10252 - targetPort: 10252 + port: 10257 + targetPort: 10257 # selector: # component: kube-controller-manager - serviceMonitor: - ## Enable scraping kube-controller-manager over https. - ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - https: "True" - # Skip TLS certificate validation when scraping - insecureSkipVerify: "True" - # Name of the server to use when validating TLS certificate - serverName: null coreDns: enabled: true @@ -224,28 +216,17 @@ ${APP_INGRESS_ANNOTATIONS} endpoints: ${KUBE_MASTERS_PRIVATE} ## If using kubeScheduler.endpoints only the port and targetPort are used service: - port: 10251 - targetPort: 10251 + port: 10259 + targetPort: 10259 # selector: # component: kube-scheduler serviceMonitor: - ## Enable scraping kube-scheduler over https. - ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - https: "True" - ## Skip TLS certificate validation when scraping - insecureSkipVerify: "True" - ## Name of the server to use when validating TLS certificate - serverName: null + scheme: https + insecureSkipVerify: true kubeProxy: ## If your kube proxy is not deployed as a pod, specify IPs it can be found on endpoints: ${KUBE_MASTERS_PRIVATE} # masters + minions - serviceMonitor: - ## Enable scraping kube-proxy over https. - ## Requires proper certs (not self-signed) and delegated authentication/authorization checks - https: "True" - ## Skip TLS certificate validation when scraping - insecureSkipVerify: "True" kube-state-metrics: priorityClassName: "system-cluster-critical" @@ -272,7 +253,7 @@ ${APP_INGRESS_ANNOTATIONS} requests: cpu: 2m limits: - memory: 30M + memory: 64M # clusterDomain: ${CLUSTER_ROOT_DOMAIN_NAME} priorityClassName: "system-cluster-critical" logFormat: json @@ -281,7 +262,7 @@ ${APP_INGRESS_ANNOTATIONS} requests: cpu: 2m limits: - memory: 32M + memory: 64M image: repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus-operator/}prometheus-operator prometheusDefaultBaseImage: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}prometheus diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 4c77e8c849..2a909b6c1f 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1135,6 +1135,18 @@ resources: - protocol: tcp port_range_min: 9090 port_range_max: 9090 + - protocol: tcp + port_range_min: 10259 + port_range_max: 10259 + - protocol: tcp + port_range_min: 10257 + port_range_max: 10257 + - protocol: tcp + port_range_min: 10249 + port_range_max: 10249 + - protocol: tcp + port_range_min: 9153 + port_range_max: 9153 secgroup_kube_minion: condition: create_cluster_resources From 4cd6339b5e0a5ead26d2c19fed95770fff777a34 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 10 Jan 2023 14:35:01 +0100 Subject: [PATCH 05/17] autoscaler: Update manifest per upstream Change-Id: I5b4c55307acd822168645871bebe0a31eee0ee27 (cherry picked from commit e3561684dcc2508df69d271bc5d9666b48e3c86e) (cherry picked from commit a35188480b175654c72f24afccf31062070f610c) (cherry picked from commit 29c99e9cf14004f1b745d7dc4eab2800c9c00ddc) --- .../fragments/enable-auto-scaling.sh | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh index 3cf6a7e96d..6365035ba2 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-scaling.sh @@ -30,54 +30,50 @@ rules: resources: ["leases"] resourceNames: ["cluster-autoscaler"] verbs: ["get", "update", "patch", "delete"] - # TODO: remove in 1.18; CA uses lease objects for leader election since 1.17 - apiGroups: [""] - resources: ["endpoints"] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] - apiGroups: [""] resources: ["endpoints"] resourceNames: ["cluster-autoscaler"] - verbs: ["get", "update", "patch", "delete"] - # accessing & modifying cluster state (nodes & pods) + verbs: ["get", "update"] - apiGroups: [""] resources: ["nodes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] + verbs: ["watch", "list", "get", "update"] - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - # read-only access to cluster state - - apiGroups: [""] - resources: ["services", "replicationcontrollers", "persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: ["daemonsets", "replicasets"] - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["get", "list", "watch"] + resources: + - "namespaces" + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] - apiGroups: ["batch"] resources: ["jobs"] - verbs: ["get", "list", "watch"] + verbs: ["watch", "list", "get"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] - verbs: ["get", "list", "watch"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["daemonsets", "replicasets", "statefulsets"] + verbs: ["watch", "list", "get"] - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes"] - verbs: ["get", "list", "watch"] - # misc access - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] + resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities"] + verbs: ["watch", "list", "get"] - apiGroups: [""] resources: ["configmaps"] - verbs: ["create"] + verbs: ["create","list","watch"] - apiGroups: [""] resources: ["configmaps"] - resourceNames: ["cluster-autoscaler-status"] - verbs: ["get", "update", "patch", "delete"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From fceb8d4c8f77a710d694d608420216962e15f7c6 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Thu, 5 Jan 2023 09:27:13 +0100 Subject: [PATCH 06/17] coredns: Use /run/systemd/resolve/resolv.conf In FCOS36 and later coredns started failing with loops. /etc/resolv.conf will not work with coredns (see [1]). Tested on FC35/36/37. [1]: https://github.com/coredns/coredns/blob/master/plugin/loop/README.md#troubleshooting Change-Id: I3a0700046bade7bd0b3a1b319f66b8f675f0c8fb (cherry picked from commit 037a71aaf8664a7f6a8926dceb294592d73da591) (cherry picked from commit bf99035555ded913239bde5ad6bb52fcc9db0585) (cherry picked from commit 0f3ff1a742785411ddf7e8ba53ea33143663df84) --- .../templates/kubernetes/fragments/core-dns-service.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh index 0851a74f0e..36d78b2c1c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh @@ -84,7 +84,7 @@ data: fallthrough in-addr.arpa ip6.arpa } prometheus :9153 - forward . /etc/resolv.conf + forward . /run/systemd/resolve/resolv.conf cache 30 loop reload @@ -144,6 +144,9 @@ spec: readOnly: true - name: tmp mountPath: /tmp + - name: resolvconf + mountPath: /run/systemd/resolve/resolv.conf + readOnly: true ports: - containerPort: 53 name: dns @@ -186,6 +189,10 @@ spec: items: - key: Corefile path: Corefile + - name: resolvconf + hostPath: + path: /run/systemd/resolve/resolv.conf + type: File --- apiVersion: v1 kind: Service From 47aa391b490ec5fc490b20900520216d469f2cb6 Mon Sep 17 00:00:00 2001 From: Dale Smith Date: Tue, 1 Aug 2023 15:16:29 +1200 Subject: [PATCH 07/17] Support k8s 1.27: Remove unsupported kubelet arg This argument has been defined for containerd clusters in Magnum, and is set to the default (and only valid) value of 'remote'. Kubelet warning in 1.26: * Flag --container-runtime has been deprecated, will be removed in 1.27 as the only valid value is 'remote' Kubelet error in 1.27: * E0801 03:10:26.723998 8889 run.go:74] "command failed" err="failed to parse kubelet flag: unknown flag: --container-runtime" Change-Id: I072fab1342593941414b86e28b8a76edf2b19a6f (cherry picked from commit dc2b3724f5e70c218a3454fb98b8e0abfe652067) --- .../kubernetes/fragments/configure-kubernetes-master.sh | 1 - .../kubernetes/fragments/configure-kubernetes-minion.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 623f9b9405..7c138255be 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -499,7 +499,6 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then KUBELET_ARGS="${KUBELET_ARGS} --runtime-cgroups=/system.slice/containerd.service" - KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" KUBELET_ARGS="${KUBELET_ARGS} --runtime-request-timeout=15m" KUBELET_ARGS="${KUBELET_ARGS} --container-runtime-endpoint=unix:///run/containerd/containerd.sock" else diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 23bd33a650..e12407eb5a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -276,7 +276,6 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then KUBELET_ARGS="${KUBELET_ARGS} --runtime-cgroups=/system.slice/containerd.service" - KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" KUBELET_ARGS="${KUBELET_ARGS} --runtime-request-timeout=15m" KUBELET_ARGS="${KUBELET_ARGS} --container-runtime-endpoint=unix:///run/containerd/containerd.sock" else From 4faa9e031e9a4b2a915b3077b1ab13a1633c0758 Mon Sep 17 00:00:00 2001 From: Dale Smith Date: Mon, 23 Jan 2023 16:56:50 +1300 Subject: [PATCH 08/17] Support k8s 1.26: remove logtostderr klog args have been removed from kubernetes in 1.26, and deprecated since 1.23. https://github.com/kubernetes/kubernetes/pull/112120 The argument --logtostderr has defaulted to true for a long time, so this removal on older versions should have no impact. Change-Id: I64f934a9bbc39c5e054d8a83b3f6edee061469e6 (cherry picked from commit 16baf85716bd36bb00ef5b69616c9f0b8edf88fb) --- .../fragments/configure-kubernetes-master.sh | 11 +++++------ .../fragments/configure-kubernetes-minion.sh | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 7c138255be..c044d9d400 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -49,7 +49,6 @@ fi KUBE_MASTER_URI="https://127.0.0.1:$KUBE_API_PORT" mkdir -p /srv/magnum/kubernetes/ cat > /etc/kubernetes/config < /etc/kubernetes/kubelet < /etc/kubernetes/config < /etc/kubernetes/kubelet < Date: Fri, 10 Feb 2023 00:33:52 +1100 Subject: [PATCH 09/17] Remove PodSecurityPolicy PodSecurityPolicy has been removed in Kubernetes v1.25 [1]. To allow Magnum to support Kubernetes v1.25 and above, PodSecurityPolicy Admission Controller has has been removed. [1] https://kubernetes.io/docs/concepts/security/pod-security-policy/ Change-Id: I0fb0c372b484275b0677114193289469ee788b84 (cherry picked from commit 1b1c2122f04d3aa03929a2e3af6cd62a219c1d6c) --- .../kubernetes/fragments/flannel-service.sh | 51 ---------------- .../kube-apiserver-to-kubelet-role.sh | 61 ------------------- .../templates/kubecluster.yaml | 2 +- ...ve-podsecuritypolicy-5851f4009f1a166c.yaml | 13 ++++ 4 files changed, 14 insertions(+), 113 deletions(-) create mode 100644 releasenotes/notes/remove-podsecuritypolicy-5851f4009f1a166c.yaml diff --git a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh index 093933340a..abb2497761 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh @@ -14,62 +14,11 @@ if [ "$NETWORK_DRIVER" = "flannel" ]; then set +x cat << EOF > ${FLANNEL_DEPLOY} --- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default -spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unsed in CaaSP - rule: 'RunAsAny' ---- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - apiGroups: - "" resources: diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index c9759e6d3f..74a4146deb 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -78,67 +78,6 @@ EOF } kubectl apply --validate=false -f ${ADMIN_RBAC} -POD_SECURITY_POLICIES=/srv/magnum/kubernetes/podsecuritypolicies.yaml -# Pod Security Policies -[ -f ${POD_SECURITY_POLICIES} ] || { - echo "Writing File: $POD_SECURITY_POLICIES" - mkdir -p $(dirname ${POD_SECURITY_POLICIES}) - cat > ${POD_SECURITY_POLICIES} < List of admission control plugins to activate - default: "PodSecurityPolicy,NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,RuntimeClass" + default: "NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,RuntimeClass" kube_allow_priv: type: string diff --git a/releasenotes/notes/remove-podsecuritypolicy-5851f4009f1a166c.yaml b/releasenotes/notes/remove-podsecuritypolicy-5851f4009f1a166c.yaml new file mode 100644 index 0000000000..82b92d22ad --- /dev/null +++ b/releasenotes/notes/remove-podsecuritypolicy-5851f4009f1a166c.yaml @@ -0,0 +1,13 @@ +--- +deprecations: + - | + PodSecurityPolicy has been removed in Kubernetes v1.25 [1]. To allow Magnum + to support Kubernetes v1.25 and above, PodSecurityPolicy Admission + Controller has has been removed. + + This means that there is a behaviour change in Cluster Templates created + after this change, where new Clusters with such Cluster Templates will not + have PodSecurityPolicy. Please be aware of the subsequent impact on Helm + Charts, etc. + + [1] https://kubernetes.io/docs/concepts/security/pod-security-policy/ From b4ed3ecb01f0c7ca3636c1978a9be67ee96f13a7 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 13 Nov 2023 10:44:49 +0100 Subject: [PATCH 10/17] Fix containerd runtime for releases older than 1.27 (cherry picked from commit d589f0f32a4159a1da8e1aed768ceb2b3770924d) (cherry picked from commit 3f901a5422c28a8b56cc46c5d5b3ab1fa8a15289) --- .../kubernetes/fragments/configure-kubernetes-master.sh | 7 +++++++ .../kubernetes/fragments/configure-kubernetes-minion.sh | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index c044d9d400..1eed308b51 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -496,7 +496,14 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil # specified cgroup driver KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" + if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then + # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option + KUBELET_VERSION=$(podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" + if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then + KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" + fi KUBELET_ARGS="${KUBELET_ARGS} --runtime-cgroups=/system.slice/containerd.service" KUBELET_ARGS="${KUBELET_ARGS} --runtime-request-timeout=15m" KUBELET_ARGS="${KUBELET_ARGS} --container-runtime-endpoint=unix:///run/containerd/containerd.sock" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 7f00cc1320..5228815e26 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -274,6 +274,12 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil # specified cgroup driver KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then + # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option + KUBELET_VERSION=$(podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" + if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then + KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" + fi KUBELET_ARGS="${KUBELET_ARGS} --runtime-cgroups=/system.slice/containerd.service" KUBELET_ARGS="${KUBELET_ARGS} --runtime-request-timeout=15m" KUBELET_ARGS="${KUBELET_ARGS} --container-runtime-endpoint=unix:///run/containerd/containerd.sock" From 835329c9a44daf18ac4824c88ea4e9a0b1b3323d Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 14 Nov 2023 10:08:32 +0100 Subject: [PATCH 11/17] Use ssh_cmd to get kubelet version (cherry picked from commit cb52da1097c3c06ce91236c8e0370262078761b2) (cherry picked from commit 560855ccecfcc5c3e831f0ff24d84b6255996573) --- .../kubernetes/fragments/configure-kubernetes-master.sh | 2 +- .../kubernetes/fragments/configure-kubernetes-minion.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 1eed308b51..bfb555a1aa 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -499,7 +499,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option - KUBELET_VERSION=$(podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + KUBELET_VERSION=$($ssh_cmd podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 5228815e26..25ca6cd39e 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -275,7 +275,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option - KUBELET_VERSION=$(podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + KUBELET_VERSION=$($ssh_cmd podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" From 25d5472a551e7b5037aacda5d1ef82954e8228f3 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 14 Nov 2023 16:06:44 +0100 Subject: [PATCH 12/17] Run podman run --rm for kubelet version (cherry picked from commit c13a91e222711af9a76e31db8568907934789d8b) (cherry picked from commit 5cbc4715e8a69db2cfaf03cd4966f8fbd7764e80) --- .../kubernetes/fragments/configure-kubernetes-master.sh | 2 +- .../kubernetes/fragments/configure-kubernetes-minion.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index bfb555a1aa..7264947473 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -499,7 +499,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option - KUBELET_VERSION=$($ssh_cmd podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + KUBELET_VERSION=$($ssh_cmd podman run --rm ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 25ca6cd39e..bb2dfaa1c1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -275,7 +275,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}" if [ ${CONTAINER_RUNTIME} = "containerd" ] ; then # check kubelet version, 1.27.0 dropped docker shim and --container-runtime command line option - KUBELET_VERSION=$($ssh_cmd podman run ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') + KUBELET_VERSION=$($ssh_cmd podman run --rm ${CONTAINER_INFRA_PREFIX:-${HYPERKUBE_PREFIX}}hyperkube:${KUBE_TAG} kubelet --version | awk '{print $2}') CONTAINER_RUNTIME_REMOTE_DROPPED="v1.27.0" if [[ "${CONTAINER_RUNTIME_REMOTE_DROPPED}" != $(echo -e "${CONTAINER_RUNTIME_REMOTE_DROPPED}\n${KUBELET_VERSION}" | sort -V | head -n1) && "${KUBELET_VERSION}" != "devel" ]]; then KUBELET_ARGS="${KUBELET_ARGS} --container-runtime=remote" From 75491e8e3115242c0d2cafdf39017a37c587f6c1 Mon Sep 17 00:00:00 2001 From: Dale Smith Date: Mon, 16 Jan 2023 10:27:44 +1300 Subject: [PATCH 13/17] Support k8s 1.25 in Calico Manifest PodDisruptionBudget is `policy/v1` since 1.21. https://github.com/projectcalico/calico/issues/4570 Change-Id: I07786095a30ae15fe856fd3966fc073267d2ae9d (cherry picked from commit 5abcab4efd937e26a7126014f18aed060dd91a65) --- .../common/templates/kubernetes/fragments/calico-service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 08c204860f..da60ef529f 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -4441,7 +4441,7 @@ metadata: # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers From 688e9f7d234da404d0c617c6434b3666340c3cab Mon Sep 17 00:00:00 2001 From: Jakub Darmach Date: Thu, 8 Feb 2024 12:08:31 +0100 Subject: [PATCH 14/17] Removing legacy calico v3.3 Removed legacy Calico v3.3 manifest and conditional. - Change-Id: Ica52a670452e65c5ae012012895080c9c8d9abf7 (cherry picked from commit fde7f8e73ab7928a18027cd8edd041d7d0b50248) --- .../fragments/calico-service-v3-3-x.sh | 669 ------------------ .../templates/kubecluster.yaml | 11 +- .../drop-calico-v3-3-7d47eb04fcb392dc.yaml | 4 + 3 files changed, 5 insertions(+), 679 deletions(-) delete mode 100644 magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh create mode 100644 releasenotes/notes/drop-calico-v3-3-7d47eb04fcb392dc.yaml diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh deleted file mode 100644 index 5e8202e461..0000000000 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-3-x.sh +++ /dev/null @@ -1,669 +0,0 @@ -step="calico-service-v3-3-x" -printf "Starting to run ${step}\n" - -. /etc/sysconfig/heat-params - -if [ "$NETWORK_DRIVER" = "calico" ]; then - _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} - - CALICO_DEPLOY=/srv/magnum/kubernetes/manifests/calico-deploy.yaml - [ -f ${CALICO_DEPLOY} ] || { - echo "Writing File: $CALICO_DEPLOY" - mkdir -p $(dirname ${CALICO_DEPLOY}) - cat << EOF > ${CALICO_DEPLOY} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: magnum:podsecuritypolicy:calico - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: magnum:podsecuritypolicy:privileged -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system ---- -# Calico Version v3.3.6 -# https://docs.projectcalico.org/v3.3/releases#v3.3.6 -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node -rules: - - apiGroups: [""] - resources: - - nodes/status - verbs: - - patch - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- -# Calico Version v3.3.6 -# https://docs.projectcalico.org/v3.3/releases#v3.3.6 -# This manifest includes the following component versions: -# calico/node:v3.3.6 -# calico/cni:v3.3.6 - -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas - # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is - # essential. - typha_service_name: "none" - # Configure the Calico backend to use. - calico_backend: "bird" - - # Configure the MTU to use - veth_mtu: "1440" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.0", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] - } - ---- - - -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. - -apiVersion: v1 -kind: Service -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha - ---- - -# This manifest creates a Deployment of Typha to back the above service. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 - revisionHistoryLimit: 2 - selector: - matchLabels: - k8s-app: calico-typha - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical - # add-on, ensuring it gets priority scheduling and that its resources are reserved - # if it ever gets evicted. - scheduler.alpha.kubernetes.io/critical-pod: '' - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - containers: - - image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}typha:${CALICO_TAG}" - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - exec: - command: - - calico-typha - - check - - liveness - periodSeconds: 30 - initialDelaySeconds: 30 - readinessProbe: - exec: - command: - - calico-typha - - check - - readiness - periodSeconds: 10 - ---- - -# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha - ---- - -# This manifest installs the calico/node container, as well -# as the Calico CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - annotations: - # This, along with the CriticalAddonsOnly toleration below, - # marks the pod as a critical add-on, ensuring it gets - # priority scheduling and that its resources are reserved - # if it ever gets evicted. - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - containers: - # Runs calico/node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}node:${CALICO_TAG}" - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within '--cluster-cidr'. - - name: CALICO_IPV4POOL_CIDR - value: ${CALICO_IPV4POOL} - - name: CALICO_IPV4POOL_IPIP - value: "${CALICO_IPV4POOL_IPIP}" - - name: CALICO_IPV4POOL_NAT_OUTGOING - value: "true" - # Set noderef for node controller. - - name: CALICO_K8S_NODE_REF - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Disable file logging so 'kubectl logs' works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -bird-ready - - -felix-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: "${CONTAINER_INFRA_PREFIX:-docker.io/calico/}cni:${CALICO_TAG}" - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- - -# Create all the CustomResourceDefinitions needed for -# Calico policy and networking mode. - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: HostEndpoint - plural: hostendpoints - singular: hostendpoint - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy - -EOF - } - - until [ "ok" = "$(kubectl get --raw='/healthz')" ] - do - echo "Waiting for Kubernetes API..." - sleep 5 - done - - /usr/bin/kubectl apply -f ${CALICO_DEPLOY} --namespace=kube-system -fi - -printf "Finished running ${step}\n" diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index e97e8d605f..affefb0fb9 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -36,12 +36,6 @@ conditions: - create_cluster_resources - is_worker - calico_v3_3: - yaql: - expression: $.data.calico_tag.startsWith("v3.3.") - data: - calico_tag: {get_param: calico_tag} - parameters: # needs to become a list if we want to join master nodes? @@ -1459,10 +1453,7 @@ resources: - "source /etc/bashrc" - get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh - - if: - - calico_v3_3 - - get_file: ../../common/templates/kubernetes/fragments/calico-service-v3-3-x.sh - - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh + - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh - str_replace: diff --git a/releasenotes/notes/drop-calico-v3-3-7d47eb04fcb392dc.yaml b/releasenotes/notes/drop-calico-v3-3-7d47eb04fcb392dc.yaml new file mode 100644 index 0000000000..a83105f411 --- /dev/null +++ b/releasenotes/notes/drop-calico-v3-3-7d47eb04fcb392dc.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Support for deploying ``Calico v3.3`` has been dropped. From 4795099fdae91717b39d9f35081e6f625dc04b35 Mon Sep 17 00:00:00 2001 From: Jakub Darmach Date: Thu, 8 Feb 2024 13:19:33 +0100 Subject: [PATCH 15/17] Removing Tiller support Removed Tiller support from Helm modules install kubernetes fragment. Change-Id: I81db0055ae82c64218498ae3e2a4fcc802f8d0e4 (cherry picked from commit e3aaa89376c2b405d74c91995888c6370b25dc51) --- doc/source/user/index.rst | 28 +-- doc/source/user/monitoring.rst | 2 - .../fragments/enable-helm-tiller.sh | 237 ------------------ .../fragments/install-helm-modules.sh | 95 ++++--- .../fragments/write-heat-params-master.sh | 3 - .../drivers/heat/k8s_fedora_template_def.py | 3 - .../templates/kubecluster.yaml | 19 -- .../templates/kubemaster.yaml | 15 -- .../unit/drivers/test_template_definition.py | 18 -- .../notes/drop-tiller-5b98862961003df8.yaml | 8 + 10 files changed, 54 insertions(+), 374 deletions(-) delete mode 100644 magnum/drivers/common/templates/kubernetes/fragments/enable-helm-tiller.sh create mode 100644 releasenotes/notes/drop-tiller-5b98862961003df8.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 2b946b49ec..ce1ea24e0a 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -413,13 +413,6 @@ the table are linked to more details elsewhere in the user guide. +---------------------------------------+--------------------+---------------+ | `k8s_keystone_auth_tag`_ | see below | see below | +---------------------------------------+--------------------+---------------+ -| `tiller_enabled`_ | - true | false | -| | - false | | -+---------------------------------------+--------------------+---------------+ -| `tiller_tag`_ | see below | "" | -+---------------------------------------+--------------------+---------------+ -| `tiller_namespace`_ | see below | see below | -+---------------------------------------+--------------------+---------------+ | `helm_client_url`_ | see below | see below | +---------------------------------------+--------------------+---------------+ | `helm_client_sha256`_ | see below | see below | @@ -1456,22 +1449,6 @@ _`k8s_keystone_auth_tag` Train default: v1.14.0 Ussuri default: v1.18.0 -_`tiller_enabled` - If set to true, tiller will be deployed in the kube-system namespace. - Ussuri default: false - Train default: false - -_`tiller_tag` - This label allows users to override the default container tag for Tiller. - For additional tags, `refer to Tiller page - `_ and look for tags`_. You must use identical tiller_tag if - you wish to use Tiller (for helm_client_tag`_. Ussuri default: v3.2.1 _`master_lb_floating_ip_enabled` @@ -1715,8 +1691,6 @@ _`ingress_controller` Controller is configured. For more details about octavia-ingress-controller please refer to `cloud-provider-openstack document `_ - To use 'nginx' ingress controller, tiller_enabled must be true when using - helm_client_tag ${TILLER_RBAC} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${TILLER_NAMESPACE} ---- -# Tiller service account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: tiller - namespace: ${TILLER_NAMESPACE} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: tiller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: tiller - namespace: ${TILLER_NAMESPACE} -EOF - } - - [ -f ${TILLER_DEPLOYER} ] || { - echo "Writing File: $TILLER_DEPLOYER" - mkdir -p $(dirname ${TILLER_DEPLOYER}) - cat << EOF > ${TILLER_DEPLOYER} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - app: helm - name: tiller - name: tiller-deploy - namespace: ${TILLER_NAMESPACE} -spec: - replicas: 1 - strategy: {} - selector: - matchLabels: - app: helm - name: tiller - template: - metadata: - creationTimestamp: null - labels: - app: helm - name: tiller - spec: - automountServiceAccountToken: true - containers: - - env: - - name: TILLER_NAMESPACE - value: ${TILLER_NAMESPACE} - - name: TILLER_HISTORY_MAX - value: "0" - - name: TILLER_TLS_VERIFY - value: "1" - - name: TILLER_TLS_ENABLE - value: "1" - - name: TILLER_TLS_CERTS - value: /etc/certs - image: ${TILLER_IMAGE} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /liveness - port: 44135 - initialDelaySeconds: 1 - timeoutSeconds: 1 - name: tiller - ports: - - containerPort: 44134 - name: tiller - - containerPort: 44135 - name: http - readinessProbe: - httpGet: - path: /readiness - port: 44135 - initialDelaySeconds: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 25m - volumeMounts: - - mountPath: /etc/certs - name: tiller-certs - readOnly: true - serviceAccountName: tiller - tolerations: - # Make sure the pod can be scheduled on master kubelet. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - # run only on master nodes - nodeSelector: - node-role.kubernetes.io/master: "" - volumes: - - name: tiller-certs - secret: - secretName: tiller-secret -status: {} - ---- -apiVersion: v1 -kind: Service -metadata: - creationTimestamp: null - labels: - app: helm - name: tiller - name: tiller-deploy - namespace: ${TILLER_NAMESPACE} -spec: - ports: - - name: tiller - port: 44134 - targetPort: tiller - selector: - app: helm - name: tiller - type: ClusterIP -status: - loadBalancer: {} - ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - creationTimestamp: null - labels: - app: helm - name: tiller - name: tiller-secret - namespace: ${TILLER_NAMESPACE} -data: - ca.crt: $(cat "${CERTS_DIR}/ca.cert.pem" | base64 --wrap=0) - tls.crt: $(cat "${CERTS_DIR}/tiller.cert.pem" | base64 --wrap=0) - tls.key: $(cat "${CERTS_DIR}/tiller.key.pem" | base64 --wrap=0) ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - creationTimestamp: null - labels: - app: helm - name: tiller-ca-key - name: tiller-ca-key - namespace: ${TILLER_NAMESPACE} -data: - ca.key.pem: $(cat "${CERTS_DIR}/ca.key.pem" | base64 --wrap=0) ---- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - creationTimestamp: null - labels: - app: helm - name: helm-client - name: helm-client-secret - namespace: ${TILLER_NAMESPACE} -data: - ca.pem: $(cat "${CERTS_DIR}/ca.cert.pem" | base64 --wrap=0) - cert.pem: $(cat "${CERTS_DIR}/helm.cert.pem" | base64 --wrap=0) - key.pem: $(cat "${CERTS_DIR}/helm.key.pem" | base64 --wrap=0) -EOF - } - - until [ "ok" = "$(kubectl get --raw='/healthz')" ] - do - echo "Waiting for Kubernetes API..." - sleep 5 - done - - kubectl apply -f ${TILLER_RBAC} - kubectl apply -f ${TILLER_DEPLOYER} -fi - -printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh index a0b3f4bc75..1bdc5aabc6 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh @@ -24,68 +24,63 @@ until [ "ok" = "$(kubectl get --raw='/healthz')" ]; do sleep 5 done -if [[ "$(echo ${TILLER_ENABLED} | tr '[:upper:]' '[:lower:]')" != "true" && "${HELM_CLIENT_TAG}" == v2.* ]]; then - echo "Use --labels tiller_enabled=True for helm_client_tag /srv/magnum/bin/helm - $ssh_cmd chmod +x /srv/magnum/bin/helm +source /etc/bashrc +$ssh_cmd tar xzvf /srv/magnum/helm-client.tar.gz linux-amd64/helm -O > /srv/magnum/bin/helm +$ssh_cmd chmod +x /srv/magnum/bin/helm - helm_install_cmd="helm upgrade --install magnum . --namespace kube-system --values values.yaml --render-subchart-notes" - helm_history_cmd="helm history magnum --namespace kube-system" - if [[ "${HELM_CLIENT_TAG}" == v2.* ]]; then - CERTS_DIR="/etc/kubernetes/helm/certs" - export HELM_HOME="/srv/magnum/kubernetes/helm/home" - export HELM_TLS_ENABLE="true" - export TILLER_NAMESPACE - mkdir -p "${HELM_HOME}" - ln -s ${CERTS_DIR}/helm.cert.pem ${HELM_HOME}/cert.pem - ln -s ${CERTS_DIR}/helm.key.pem ${HELM_HOME}/key.pem - ln -s ${CERTS_DIR}/ca.cert.pem ${HELM_HOME}/ca.pem +helm_install_cmd="helm upgrade --install magnum . --namespace kube-system --values values.yaml --render-subchart-notes" +helm_history_cmd="helm history magnum --namespace kube-system" +if [[ "${HELM_CLIENT_TAG}" == v2.* ]]; then + CERTS_DIR="/etc/kubernetes/helm/certs" + export HELM_HOME="/srv/magnum/kubernetes/helm/home" + export HELM_TLS_ENABLE="true" + mkdir -p "${HELM_HOME}" + ln -s ${CERTS_DIR}/helm.cert.pem ${HELM_HOME}/cert.pem + ln -s ${CERTS_DIR}/helm.key.pem ${HELM_HOME}/key.pem + ln -s ${CERTS_DIR}/ca.cert.pem ${HELM_HOME}/ca.pem - # HACK - Force wait because of bug https://github.com/helm/helm/issues/5170 - until helm init --client-only --wait; do - sleep 5s - done - helm_install_cmd="helm upgrade --install --name magnum . --namespace kube-system --values values.yaml --render-subchart-notes" - helm_history_cmd="helm history magnum" - fi + # HACK - Force wait because of bug https://github.com/helm/helm/issues/5170 + until helm init --client-only --wait; do + sleep 5s + done + helm_install_cmd="helm upgrade --install --name magnum . --namespace kube-system --values values.yaml --render-subchart-notes" + helm_history_cmd="helm history magnum" +fi - HELM_CHART_DIR="/srv/magnum/kubernetes/helm/magnum" - if [[ -d "${HELM_CHART_DIR}" ]]; then - pushd ${HELM_CHART_DIR} - cat << EOF > Chart.yaml +HELM_CHART_DIR="/srv/magnum/kubernetes/helm/magnum" +if [[ -d "${HELM_CHART_DIR}" ]]; then + pushd ${HELM_CHART_DIR} + cat << EOF > Chart.yaml apiVersion: v1 name: magnum version: 1.0.0 appVersion: v1.0.0 description: Magnum Helm Charts EOF - sed -i '1i\dependencies:' requirements.yaml + sed -i '1i\dependencies:' requirements.yaml - i=0 - until ($helm_history_cmd | grep magnum | grep deployed) || (helm dep update && $helm_install_cmd); do - i=$((i + 1)) - [ $i -lt 60 ] || break; - sleep 5 - done - popd - fi + i=0 + until ($helm_history_cmd | grep magnum | grep deployed) || (helm dep update && $helm_install_cmd); do + i=$((i + 1)) + [ $i -lt 60 ] || break; + sleep 5 + done + popd fi echo "END: ${step}" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index 0cd02bf95b..53034354bc 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -124,9 +124,6 @@ KEYSTONE_AUTH_ENABLED="$KEYSTONE_AUTH_ENABLED" K8S_KEYSTONE_AUTH_TAG="$K8S_KEYSTONE_AUTH_TAG" PROJECT_ID="$PROJECT_ID" EXTERNAL_NETWORK_ID="$EXTERNAL_NETWORK_ID" -TILLER_ENABLED="$TILLER_ENABLED" -TILLER_TAG="$TILLER_TAG" -TILLER_NAMESPACE="$TILLER_NAMESPACE" HELM_CLIENT_URL="$HELM_CLIENT_URL" HELM_CLIENT_SHA256="$HELM_CLIENT_SHA256" HELM_CLIENT_TAG="$HELM_CLIENT_TAG" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 1c26d2a2da..201485af64 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -112,9 +112,6 @@ def get_params(self, context, cluster_template, cluster, **kwargs): 'prometheus_adapter_chart_tag', 'prometheus_adapter_configmap', 'selinux_mode', - 'tiller_enabled', - 'tiller_tag', - 'tiller_namespace', 'helm_client_url', 'helm_client_sha256', 'helm_client_tag', 'traefik_ingress_controller_tag', diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index affefb0fb9..7719a4747f 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -789,21 +789,6 @@ parameters: description: > project id of current project - tiller_enabled: - type: boolean - description: Choose whether to install tiller or not. - default: false - - tiller_tag: - type: string - description: tag of tiller container - default: "v2.16.7" - - tiller_namespace: - type: string - description: namespace where tiller will be installed. - default: "magnum-tiller" - helm_client_url: type: string description: url of helm client tarball @@ -1399,9 +1384,6 @@ resources: prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag} prometheus_adapter_configmap: {get_param: prometheus_adapter_configmap} project_id: {get_param: project_id} - tiller_enabled: {get_param: tiller_enabled} - tiller_tag: {get_param: tiller_tag} - tiller_namespace: {get_param: tiller_namespace} helm_client_url: {get_param: helm_client_url} helm_client_sha256: {get_param: helm_client_sha256} helm_client_tag: {get_param: helm_client_tag} @@ -1455,7 +1437,6 @@ resources: - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh - - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh} params: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index c0381a9e9a..81ddf55584 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -545,18 +545,6 @@ parameters: description: > project id of current project - tiller_enabled: - type: string - description: Whether to enable tiller or not - - tiller_tag: - type: string - description: tag of tiller container - - tiller_namespace: - type: string - description: namespace where tiller will be installed - helm_client_url: type: string description: url of helm client tarball @@ -928,9 +916,6 @@ resources: "$PROMETHEUS_ADAPTER_CONFIGMAP": {get_param: prometheus_adapter_configmap} "$PROJECT_ID": {get_param: project_id} "$EXTERNAL_NETWORK_ID": {get_param: external_network} - "$TILLER_ENABLED": {get_param: tiller_enabled} - "$TILLER_TAG": {get_param: tiller_tag} - "$TILLER_NAMESPACE": {get_param: tiller_namespace} "$HELM_CLIENT_URL": {get_param: helm_client_url} "$HELM_CLIENT_SHA256": {get_param: helm_client_sha256} "$HELM_CLIENT_TAG": {get_param: helm_client_tag} diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index fc9bdbeec8..74dbe02d4a 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -550,12 +550,6 @@ def test_k8s_get_params(self, mock_generate_csr_and_key, prometheus_adapter_configmap = mock_cluster.labels.get( 'prometheus_adapter_configmap') project_id = mock_cluster.project_id - tiller_enabled = mock_cluster.labels.get( - 'tiller_enabled') - tiller_tag = mock_cluster.labels.get( - 'tiller_tag') - tiller_namespace = mock_cluster.labels.get( - 'tiller_namespace') helm_client_url = mock_cluster.labels.get( 'helm_client_url') helm_client_sha256 = mock_cluster.labels.get( @@ -699,9 +693,6 @@ def test_k8s_get_params(self, mock_generate_csr_and_key, 'prometheus_adapter_configmap': prometheus_adapter_configmap, 'project_id': project_id, 'external_network': external_network_id, - 'tiller_enabled': tiller_enabled, - 'tiller_tag': tiller_tag, - 'tiller_namespace': tiller_namespace, 'helm_client_url': helm_client_url, 'helm_client_sha256': helm_client_sha256, 'helm_client_tag': helm_client_tag, @@ -1119,12 +1110,6 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key, prometheus_adapter_configmap = mock_cluster.labels.get( 'prometheus_adapter_configmap') project_id = mock_cluster.project_id - tiller_enabled = mock_cluster.labels.get( - 'tiller_enabled') - tiller_tag = mock_cluster.labels.get( - 'tiller_tag') - tiller_namespace = mock_cluster.labels.get( - 'tiller_namespace') helm_client_url = mock_cluster.labels.get( 'helm_client_url') helm_client_sha256 = mock_cluster.labels.get( @@ -1273,9 +1258,6 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key, 'prometheus_adapter_configmap': prometheus_adapter_configmap, 'project_id': project_id, 'external_network': external_network_id, - 'tiller_enabled': tiller_enabled, - 'tiller_tag': tiller_tag, - 'tiller_namespace': tiller_namespace, 'helm_client_url': helm_client_url, 'helm_client_sha256': helm_client_sha256, 'helm_client_tag': helm_client_tag, diff --git a/releasenotes/notes/drop-tiller-5b98862961003df8.yaml b/releasenotes/notes/drop-tiller-5b98862961003df8.yaml new file mode 100644 index 0000000000..bef60274af --- /dev/null +++ b/releasenotes/notes/drop-tiller-5b98862961003df8.yaml @@ -0,0 +1,8 @@ +--- +upgrade: + - | + ``Tiller`` support has been dropped, following labels are not functional + anymore: + * ``tiller_enabled`` + * ``tiller_tag`` + * ``tiller_namespace`` From a7a7759c7c3cf0684f51016d5b4f205650735960 Mon Sep 17 00:00:00 2001 From: Jakub Darmach Date: Thu, 8 Feb 2024 14:49:00 +0100 Subject: [PATCH 16/17] Move Helm client install to separate script To deploy Calico with Tigera Operator helm charts it was necessary to move Helm client install to separate script for earlier execution. Change-Id: Iab738b4120c0ac823b247b04f0cd750de0147779 (cherry picked from commit b6e2aa8af766197fc585605b30e91e85f97bc401) --- .../fragments/install-helm-modules.sh | 21 ---------- .../kubernetes/fragments/install-helm.sh | 41 +++++++++++++++++++ .../templates/kubecluster.yaml | 1 + 3 files changed, 42 insertions(+), 21 deletions(-) create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/install-helm.sh diff --git a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh index 1bdc5aabc6..e9181d6e3d 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/install-helm-modules.sh @@ -17,32 +17,11 @@ if [ ! -z "$NO_PROXY" ]; then export NO_PROXY fi -ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost" - echo "Waiting for Kubernetes API..." until [ "ok" = "$(kubectl get --raw='/healthz')" ]; do sleep 5 done -if [ -z "${HELM_CLIENT_URL}" ] ; then - HELM_CLIENT_URL="https://get.helm.sh/helm-$HELM_CLIENT_TAG-linux-amd64.tar.gz" -fi -i=0 -until curl -o /srv/magnum/helm-client.tar.gz "${HELM_CLIENT_URL}"; do - i=$((i + 1)) - [ $i -lt 5 ] || break; - sleep 5 -done - -if ! echo "${HELM_CLIENT_SHA256} /srv/magnum/helm-client.tar.gz" | sha256sum -c - ; then - echo "ERROR helm-client.tar.gz computed checksum did NOT match, exiting." - exit 1 -fi - -source /etc/bashrc -$ssh_cmd tar xzvf /srv/magnum/helm-client.tar.gz linux-amd64/helm -O > /srv/magnum/bin/helm -$ssh_cmd chmod +x /srv/magnum/bin/helm - helm_install_cmd="helm upgrade --install magnum . --namespace kube-system --values values.yaml --render-subchart-notes" helm_history_cmd="helm history magnum --namespace kube-system" if [[ "${HELM_CLIENT_TAG}" == v2.* ]]; then diff --git a/magnum/drivers/common/templates/kubernetes/fragments/install-helm.sh b/magnum/drivers/common/templates/kubernetes/fragments/install-helm.sh new file mode 100644 index 0000000000..e73e30bc4f --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/install-helm.sh @@ -0,0 +1,41 @@ +step="install-helm" +echo "START: ${step}" + +set +x +. /etc/sysconfig/heat-params +set -ex + +if [ ! -z "$HTTP_PROXY" ]; then + export HTTP_PROXY +fi + +if [ ! -z "$HTTPS_PROXY" ]; then + export HTTPS_PROXY +fi + +if [ ! -z "$NO_PROXY" ]; then + export NO_PROXY +fi + +ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost" + +if [ -z "${HELM_CLIENT_URL}" ] ; then + HELM_CLIENT_URL="https://get.helm.sh/helm-$HELM_CLIENT_TAG-linux-amd64.tar.gz" +fi +i=0 +until curl -o /srv/magnum/helm-client.tar.gz "${HELM_CLIENT_URL}"; do + i=$((i + 1)) + [ $i -lt 5 ] || break; + sleep 5 +done + +if ! echo "${HELM_CLIENT_SHA256} /srv/magnum/helm-client.tar.gz" | sha256sum -c - ; then + echo "ERROR helm-client.tar.gz computed checksum did NOT match, exiting." + exit 1 +fi + +source /etc/bashrc +$ssh_cmd tar xzvf /srv/magnum/helm-client.tar.gz linux-amd64/helm -O > /srv/magnum/bin/helm +$ssh_cmd chmod +x /srv/magnum/bin/helm + +echo "END: ${step}" \ No newline at end of file diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 7719a4747f..dc23077e97 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1455,6 +1455,7 @@ resources: - get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh - get_file: ../../common/templates/kubernetes/fragments/enable-cinder-csi.sh # Helm Based Installation Configuration Scripts + - get_file: ../../common/templates/kubernetes/fragments/install-helm.sh - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh - str_replace: template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh} From 7e274ce1f60a18b4a7099dc762d1e4e5dd9e038c Mon Sep 17 00:00:00 2001 From: Dale Smith Date: Thu, 29 Feb 2024 11:01:36 +1300 Subject: [PATCH 17/17] Support Calico 3.26.x * Renames calico script to match the version it was intended for: 3.21. * Match calico_tag 3.26.* with the new manifest. All other versions will continue to use manifest intended for 3.21.* * Calico manifest for 3.26[1] is tested for K8s versions v1.24 through v1.28[2]. [1] https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/calico.yaml [2] https://docs.tigera.io/calico/3.26/getting-started/kubernetes/requirements Change-Id: I548e831c13be9b732303c945b2a7ba97a245b0df (cherry picked from commit 0fdec721283018c06d0f8548c66ffef83ca9c5d8) --- ...o-service.sh => calico-service-v3-21-x.sh} | 2 +- .../fragments/calico-service-v3-26-x.sh | 5013 +++++++++++++++++ .../templates/kubecluster.yaml | 12 +- .../unit/drivers/test_template_definition.py | 1 - 4 files changed, 5025 insertions(+), 3 deletions(-) rename magnum/drivers/common/templates/kubernetes/fragments/{calico-service.sh => calico-service-v3-21-x.sh} (99%) create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-26-x.sh diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-21-x.sh similarity index 99% rename from magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh rename to magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-21-x.sh index da60ef529f..c00c16578b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-21-x.sh @@ -1,4 +1,4 @@ -step="calico-service" +step="calico-service-v3-21-x" printf "Starting to run ${step}\n" set -e diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-26-x.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-26-x.sh new file mode 100644 index 0000000000..e2aeab548a --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service-v3-26-x.sh @@ -0,0 +1,5013 @@ +step="calico-service-v3-26-x" +printf "Starting to run ${step}\n" + +set -e +set +x +. /etc/sysconfig/heat-params +set -x + +if [ "$NETWORK_DRIVER" = "calico" ]; then + _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} + + CALICO_DEPLOY=/srv/magnum/kubernetes/manifests/calico-deploy.yaml + [ -f ${CALICO_DEPLOY} ] || { + echo "Writing File: $CALICO_DEPLOY" + mkdir -p $(dirname ${CALICO_DEPLOY}) + set +x + cat << EOF > ${CALICO_DEPLOY} +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format aa:nn or aa:nn:mm. + For standard community use aa:nn format, where aa and + nn are 16 bit number. For large community use aa:nn:mm + format, where aa, nn and mm are 32 bit number. Where, + aa is an AS Number, nn and mm are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + ignoredInterfaces: + description: IgnoredInterfaces indicates the network interfaces that + needs to be excluded when reading device routes. + items: + type: string + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in Specs.Communities or community value + of format aa:nn or aa:nn:mm. For standard community use + aa:nn format, where aa and nn are 16 bit number. For + large community use aa:nn:mm format, where aa, nn and + mm are 32 bit number. Where,aa is an AS Number, nn and + mm are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: bgpfilters.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPFilter + listKind: BGPFilterList + plural: bgpfilters + singular: bgpfilter + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of + the BGP Filter. + properties: + exportV4: + description: The ordered set of IPv4 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + matchOperator: + type: string + required: + - action + - cidr + - matchOperator + type: object + type: array + exportV6: + description: The ordered set of IPv6 BGPFilter rules acting on exporting + routes to a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + matchOperator: + type: string + required: + - action + - cidr + - matchOperator + type: object + type: array + importV4: + description: The ordered set of IPv4 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + matchOperator: + type: string + required: + - action + - cidr + - matchOperator + type: object + type: array + importV6: + description: The ordered set of IPv6 BGPFilter rules acting on importing + routes from a peer. + items: + description: BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + matchOperator: + type: string + required: + - action + - cidr + - matchOperator + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + filters: + description: The ordered set of BGPFilters applied on this BGP peer. + items: + type: string + type: array + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, + this is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be []:port + or : for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + reachableBy: + description: Add an exact, i.e. /32, static route toward peer IP in + order to prevent route flapping. ReachableBy contains the address + of the gateway which peer can be reached by. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + ttlSecurity: + description: TTLSecurity enables the generalized TTL security mechanism + (GTSM) which protects against spoofed packets by ignoring received + packets with a smaller than expected TTL value. The provided value + is the number of hops (edges) between the peers. + type: integer + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: Classes declares the types of information to monitor + for this calico/node, and allows for selective status reporting + about certain subsets of information. + items: + type: string + type: array + node: + description: The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: UpdatePeriodSeconds is the period at which CalicoNodeStatus + should be updated. Set to 0 to disable CalicoNodeStatus refresh. + Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: LastUpdated is a timestamp representing the server time + when CalicoNodeStatus object last updated. It is represented in + RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enable" or "Disable". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDSROptoutCIDRs: + description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded + from DSR. That is, clients in those CIDRs will accesses nodeports + as if BPFExternalServiceMode was set to Tunnel. + items: + type: string + type: array + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all host interfaces + with BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled, Strict or Loose. [Default: + Loose]' + type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfHostConntrackBypass: + description: 'BPFHostConntrackBypass Controls whether to bypass Linux + conntrack in BPF mode for workloads and services. [Default: true + - bypass Linux conntrack]' + type: boolean + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfL3IfacePattern: + description: BPFL3IfacePattern is a regular expression that allows + to list tunnel devices like wireguard or vxlan (i.e., L3 devices) + in addition to BPFDataIfacePattern. That is, tunnel interfaces not + created by Calico, that Calico workload traffic flows over as well + as any interfaces that handle incoming traffic to nodeports and + services from outside the cluster. + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + tc exec bpf debug. [Default: Off].' + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: "DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix's (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s] \n Deprecated: replaced + by the generic HealthTimeoutOverrides." + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override feature detection + based on auto-detected platform capabilities. Values are specified + in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" + or "false" will force the feature, empty or omitted values are auto-detected. + type: string + featureGates: + description: FeatureGates is used to enable or disable tech-preview + Calico features. Values are specified in a comma separated list + with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". + This is used to enable features that are not fully production ready. + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + non-OpenStack floating IP addresses. (OpenStack-derived floating + IPs are always programmed, regardless of this setting.) + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + healthTimeoutOverrides: + description: HealthTimeoutOverrides allows the internal watchdog timeouts + of individual subcomponents to be overridden. This is useful for + working around "false positive" liveness timeouts that can occur + in particularly stressful workloads or if CPU is constrained. For + a list of active subcomponents, see Felix's logs. + items: + properties: + name: + type: string + timeout: + type: string + required: + - name + - timeout + type: object + type: array + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is Auto. + type: string + iptablesFilterAllowAction: + type: string + iptablesFilterDenyAction: + description: IptablesFilterDenyAction controls what happens to traffic + that is denied by network policy. By default Calico blocks traffic + with an iptables "DROP" action. If you want to use "REJECT" action + instead you can configure it in here. + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. DeprecatesRouteTableRange, overrides RouteTableRange. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a exact or a prefix match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a exact or a prefix match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. + type: string + allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object + strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: AllowedUse controls what the IP pool will be used for. If + not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When natOutgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a exact or a prefix match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a exact or a prefix match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, global() + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-cni-plugin + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - bgpfilters + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: [ "crd.projectcalico.org" ] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-node-rbac.yaml +# CNI cluster role +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-cni-plugin +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - clusterinformations + - ippools + - ipreservations + - ipamconfigs + verbs: + - get + - list + - create + - update + - delete +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-cni-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-cni-plugin +subjects: +- kind: ServiceAccount + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: "${_prefix}cni:${CALICO_TAG}" + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: "${_prefix}cni:${CALICO_TAG}" + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: "${_prefix}node:${CALICO_TAG}" + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: "${_prefix}node:${CALICO_TAG}" + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Use fixed subnet CIDR to autodetect IP (supported since Calico v3.16.x) + - name: IP_AUTODETECTION_METHOD + value: "cidr=${CLUSTER_SUBNET_CIDR}" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "${CALICO_IPV4POOL_IPIP}" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within --cluster-cidr. + - name: CALICO_IPV4POOL_CIDR + value: "${CALICO_IPV4POOL}" + # Disable file logging so kubectl logs works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: "${_prefix}kube-controllers:${CALICO_TAG}" + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 +EOF + } + +set -x + + until [ "ok" = "$(kubectl get --raw='/healthz')" ] + do + echo "Waiting for Kubernetes API..." + sleep 5 + done + + /usr/bin/kubectl apply -f ${CALICO_DEPLOY} --namespace=kube-system +fi + +printf "Finished running ${step}\n" diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index dc23077e97..e9a40e04a2 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -36,6 +36,13 @@ conditions: - create_cluster_resources - is_worker + calico_v3_26: + yaql: + expression: $.data.calico_tag.startsWith("v3.26.") + data: + calico_tag: {get_param: calico_tag} + + parameters: # needs to become a list if we want to join master nodes? @@ -1435,7 +1442,10 @@ resources: - "source /etc/bashrc" - get_file: ../../common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh - get_file: ../../common/templates/kubernetes/fragments/core-dns-service.sh - - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh + - if: + - calico_v3_26 + - get_file: ../../common/templates/kubernetes/fragments/calico-service-v3-26-x.sh + - get_file: ../../common/templates/kubernetes/fragments/calico-service-v3-21-x.sh - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh - str_replace: template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh} diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 74dbe02d4a..1129bbbc0f 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -1170,7 +1170,6 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key, extra_subnet = mock_cluster.labels.get('extra_subnet') extra_security_group = mock_cluster.labels.get('extra_security_group') - octavia_provider = mock_cluster.labels.get('octavia_provider') octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm') octavia_lb_healthcheck = mock_cluster.labels.get(