diff --git a/.charts.yml b/.charts.yml index f6bfe9425..830d178f2 100644 --- a/.charts.yml +++ b/.charts.yml @@ -6,7 +6,7 @@ openstack_helm_dependencies: &openstack_helm_dependencies - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 charts: - name: barbican diff --git a/build/lint-jobs.py b/build/lint-jobs.py index 3edfe1283..af7ea34d0 100644 --- a/build/lint-jobs.py +++ b/build/lint-jobs.py @@ -7,6 +7,9 @@ def main(): passed = True for file in glob.glob("zuul.d/container-images/*.yaml"): + if file == "zuul.d/container-images/base.yaml": + continue + with open(file, "r") as file: configs = yaml.safe_load(file) diff --git a/build/manage-dockerfile.py b/build/manage-dockerfile.py index 068d0441b..3c031000a 100644 --- a/build/manage-dockerfile.py +++ b/build/manage-dockerfile.py @@ -2,10 +2,17 @@ import argparse import pathlib +import re from datetime import datetime +import requests -def update_dockerfile(dockerfile, timestamp): +OPENDEV_GIT_PATTERN = re.compile( + r"ADD (?:--keep-git-dir=true )?https://opendev.org/(?P[^/]+)/(?P[^/]+)\.git#\$\{(?P[^}]+)\} /src/(?P[^ ]+)?" # noqa +) + + +def update_rebuild_time(dockerfile, timestamp): rebuild_line = f"# Atmosphere-Rebuild-Time: {timestamp}" lines = dockerfile.read_text().splitlines() @@ -25,19 +32,66 @@ def update_dockerfile(dockerfile, timestamp): dockerfile.write_text("\n".join(new_lines) + "\n") +def bump_image(dockerfile, branch) -> bool: + content = dockerfile.read_text() + + opendev_match = OPENDEV_GIT_PATTERN.search(content) + if opendev_match: + namespace = opendev_match.group("namespace") + repo = opendev_match.group("repo") + ref = opendev_match.group("ref") + print(f"Bumping {repo} ({ref}) in {dockerfile}") + + url = f"https://opendev.org/api/v1/repos/{namespace}/{repo}/branches/{branch}" + response = requests.get(url) + response.raise_for_status() + data = response.json() + + # Update the Dockerfile with the latest commit + lines = content.splitlines() + new_lines = [] + for line in lines: + if line.startswith(f"ARG {ref}="): + new_lines.append(f"ARG {ref}={data['commit']['id']}") + else: + new_lines.append(line) + + dockerfile.write_text("\n".join(new_lines) + "\n") + return True + + # TODO: add support for other types of images + + print(f"Skipping {dockerfile}: No way to bump image") + return False + + def main(): parser = argparse.ArgumentParser() - parser.add_argument("--rebuild", action="store_true") - parser.add_argument( + subparsers = parser.add_subparsers(dest="command") + + force_rebuild_parser = subparsers.add_parser("force-rebuild") + force_rebuild_parser.add_argument( + "dockerfiles", type=pathlib.Path, nargs="+", help="List of Dockerfiles" + ) + + bump_parser = subparsers.add_parser("bump") + bump_parser.add_argument( + "--branch", + help="Branch to bump the image to", + ) + bump_parser.add_argument( "dockerfiles", type=pathlib.Path, nargs="+", help="List of Dockerfiles" ) - args = parser.parse_args() - if args.rebuild: - now_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + args = parser.parse_args() + now_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - for dockerfile in args.dockerfiles: - update_dockerfile(dockerfile, now_utc) + for dockerfile in args.dockerfiles: + if args.command == "bump": + if bump_image(dockerfile, args.branch): + update_rebuild_time(dockerfile, now_utc) + elif args.command == "force-rebuild": + update_rebuild_time(dockerfile, now_utc) if __name__ == "__main__": diff --git a/charts/barbican/charts/helm-toolkit/Chart.yaml b/charts/barbican/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/barbican/charts/helm-toolkit/Chart.yaml +++ b/charts/barbican/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/barbican/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/barbican/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/barbican/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/barbican/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/barbican/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/barbican/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/barbican/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/barbican/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/barbican/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/barbican/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/barbican/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/barbican/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/barbican/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/barbican/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/barbican/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/barbican/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/barbican/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/barbican/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/barbican/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/barbican/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/barbican/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/barbican/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/barbican/requirements.lock b/charts/barbican/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/barbican/requirements.lock +++ b/charts/barbican/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/barbican/requirements.yaml b/charts/barbican/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/barbican/requirements.yaml +++ b/charts/barbican/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/barbican/templates/bin/_simple_crypto_kek_rewrap.py.tpl b/charts/barbican/templates/bin/_simple_crypto_kek_rewrap.py.tpl index 7a5217527..f553548aa 100644 --- a/charts/barbican/templates/bin/_simple_crypto_kek_rewrap.py.tpl +++ b/charts/barbican/templates/bin/_simple_crypto_kek_rewrap.py.tpl @@ -33,11 +33,10 @@ class KekRewrap(object): def __init__(self, conf, old_kek): self.dry_run = False - self.db_engine = session.create_engine(conf.sql_connection) + self.db_engine = session.create_engine(conf.database.connection) self._session_creator = scoping.scoped_session( orm.sessionmaker( - bind=self.db_engine, - autocommit=True + bind=self.db_engine ) ) self.crypto_plugin = simple_crypto.SimpleCryptoPlugin(conf) diff --git a/charts/barbican/templates/configmap-etc.yaml b/charts/barbican/templates/configmap-etc.yaml index d2bff2c01..fba29565b 100644 --- a/charts/barbican/templates/configmap-etc.yaml +++ b/charts/barbican/templates/configmap-etc.yaml @@ -49,12 +49,12 @@ limitations under the License. {{- $_ := set .Values.conf.barbican.keystone_authtoken "memcache_secret_key" ( default ( randAlphaNum 64 ) .Values.endpoints.oslo_cache.auth.memcache_secret_key ) -}} {{- end -}} -{{- if empty .Values.conf.barbican.DEFAULT.sql_connection -}} +{{- if empty .Values.conf.barbican.database.connection -}} {{- $connection := tuple "oslo_db" "internal" "barbican" "mysql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" -}} {{- if .Values.manifests.certificates -}} -{{- $_ := (printf "%s?charset=utf8&ssl_ca=/etc/mysql/certs/ca.crt&ssl_key=/etc/mysql/certs/tls.key&ssl_cert=/etc/mysql/certs/tls.crt&ssl_verify_cert" $connection ) | set .Values.conf.barbican.DEFAULT "sql_connection" -}} +{{- $_ := (printf "%s?charset=utf8&ssl_ca=/etc/mysql/certs/ca.crt&ssl_key=/etc/mysql/certs/tls.key&ssl_cert=/etc/mysql/certs/tls.crt&ssl_verify_cert" $connection ) | set .Values.conf.barbican.database "connection" -}} {{- else -}} -{{- $_ := set .Values.conf.barbican.DEFAULT "sql_connection" $connection -}} +{{- $_ := set .Values.conf.barbican.database "connection" $connection -}} {{- end -}} {{- end -}} diff --git a/charts/barbican/templates/job-db-drop.yaml b/charts/barbican/templates/job-db-drop.yaml index b6f0a6fa3..f95e9faea 100644 --- a/charts/barbican/templates/job-db-drop.yaml +++ b/charts/barbican/templates/job-db-drop.yaml @@ -14,7 +14,7 @@ limitations under the License. {{- if .Values.manifests.job_db_drop }} {{- $serviceName := "barbican" -}} -{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "DEFAULT" "configDbKey" "sql_connection" -}} +{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" -}} {{- $dbDropJob := dict "envAll" . "serviceName" $serviceName "dbToDrop" $dbToDrop -}} {{- if .Values.manifests.certificates -}} {{- $_ := set $dbToDrop "dbAdminTlsSecret" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal -}} diff --git a/charts/barbican/templates/job-db-init.yaml b/charts/barbican/templates/job-db-init.yaml index afe16dcc6..5fdc24b5b 100644 --- a/charts/barbican/templates/job-db-init.yaml +++ b/charts/barbican/templates/job-db-init.yaml @@ -21,7 +21,7 @@ helm.sh/hook-weight: "-5" {{- if .Values.manifests.job_db_init }} {{- $serviceName := "barbican" -}} -{{- $dbToInit := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "DEFAULT" "configDbKey" "sql_connection" -}} +{{- $dbToInit := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" -}} {{- $dbInitJob := dict "envAll" . "serviceName" $serviceName "dbToInit" $dbToInit "jobAnnotations" (include "metadata.annotations.job.db_init" . | fromYaml) -}} {{- if .Values.manifests.certificates -}} {{- $_ := set $dbInitJob "dbAdminTlsSecret" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal -}} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml b/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml +++ b/charts/ceph-provisioners/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/ceph-provisioners/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/ceph-provisioners/requirements.lock b/charts/ceph-provisioners/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/ceph-provisioners/requirements.lock +++ b/charts/ceph-provisioners/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/ceph-provisioners/requirements.yaml b/charts/ceph-provisioners/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/ceph-provisioners/requirements.yaml +++ b/charts/ceph-provisioners/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/cinder/charts/helm-toolkit/Chart.yaml b/charts/cinder/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/cinder/charts/helm-toolkit/Chart.yaml +++ b/charts/cinder/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/cinder/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/cinder/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/cinder/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/cinder/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/cinder/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/cinder/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/cinder/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/cinder/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/cinder/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/cinder/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/cinder/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/cinder/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/cinder/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/cinder/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/cinder/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/cinder/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/cinder/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/cinder/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/cinder/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/cinder/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/cinder/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/cinder/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/cinder/requirements.lock b/charts/cinder/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/cinder/requirements.lock +++ b/charts/cinder/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/cinder/requirements.yaml b/charts/cinder/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/cinder/requirements.yaml +++ b/charts/cinder/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/designate/charts/helm-toolkit/Chart.yaml b/charts/designate/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/designate/charts/helm-toolkit/Chart.yaml +++ b/charts/designate/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/designate/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/designate/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/designate/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/designate/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/designate/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/designate/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/designate/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/designate/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/designate/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/designate/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/designate/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/designate/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/designate/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/designate/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/designate/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/designate/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/designate/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/designate/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/designate/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/designate/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/designate/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/designate/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/designate/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/designate/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/designate/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/designate/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/designate/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/designate/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/designate/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/designate/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/designate/requirements.lock b/charts/designate/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/designate/requirements.lock +++ b/charts/designate/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/designate/requirements.yaml b/charts/designate/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/designate/requirements.yaml +++ b/charts/designate/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/glance/charts/helm-toolkit/Chart.yaml b/charts/glance/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/glance/charts/helm-toolkit/Chart.yaml +++ b/charts/glance/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/glance/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/glance/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/glance/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/glance/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/glance/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/glance/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/glance/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/glance/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/glance/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/glance/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/glance/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/glance/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/glance/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/glance/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/glance/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/glance/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/glance/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/glance/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/glance/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/glance/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/glance/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/glance/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/glance/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/glance/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/glance/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/glance/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/glance/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/glance/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/glance/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/glance/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/glance/requirements.lock b/charts/glance/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/glance/requirements.lock +++ b/charts/glance/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/glance/requirements.yaml b/charts/glance/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/glance/requirements.yaml +++ b/charts/glance/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/heat/charts/helm-toolkit/Chart.yaml b/charts/heat/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/heat/charts/helm-toolkit/Chart.yaml +++ b/charts/heat/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/heat/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/heat/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/heat/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/heat/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/heat/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/heat/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/heat/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/heat/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/heat/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/heat/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/heat/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/heat/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/heat/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/heat/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/heat/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/heat/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/heat/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/heat/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/heat/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/heat/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/heat/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/heat/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/heat/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/heat/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/heat/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/heat/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/heat/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/heat/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/heat/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/heat/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/heat/requirements.lock b/charts/heat/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/heat/requirements.lock +++ b/charts/heat/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/heat/requirements.yaml b/charts/heat/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/heat/requirements.yaml +++ b/charts/heat/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/horizon/charts/helm-toolkit/Chart.yaml b/charts/horizon/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/horizon/charts/helm-toolkit/Chart.yaml +++ b/charts/horizon/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/horizon/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/horizon/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/horizon/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/horizon/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/horizon/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/horizon/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/horizon/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/horizon/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/horizon/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/horizon/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/horizon/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/horizon/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/horizon/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/horizon/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/horizon/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/horizon/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/horizon/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/horizon/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/horizon/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/horizon/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/horizon/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/horizon/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/horizon/requirements.lock b/charts/horizon/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/horizon/requirements.lock +++ b/charts/horizon/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/horizon/requirements.yaml b/charts/horizon/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/horizon/requirements.yaml +++ b/charts/horizon/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/ironic/charts/helm-toolkit/Chart.yaml b/charts/ironic/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/ironic/charts/helm-toolkit/Chart.yaml +++ b/charts/ironic/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/ironic/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/ironic/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/ironic/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/ironic/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/ironic/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/ironic/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/ironic/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/ironic/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/ironic/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/ironic/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/ironic/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/ironic/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/ironic/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/ironic/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/ironic/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/ironic/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/ironic/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/ironic/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/ironic/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/ironic/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/ironic/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/ironic/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/ironic/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/ironic/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/ironic/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/ironic/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/ironic/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/ironic/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/ironic/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/ironic/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/ironic/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/ironic/requirements.lock b/charts/ironic/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/ironic/requirements.lock +++ b/charts/ironic/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/ironic/requirements.yaml b/charts/ironic/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/ironic/requirements.yaml +++ b/charts/ironic/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/keystone/charts/helm-toolkit/Chart.yaml b/charts/keystone/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/keystone/charts/helm-toolkit/Chart.yaml +++ b/charts/keystone/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/keystone/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/keystone/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/keystone/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/keystone/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/keystone/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/keystone/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/keystone/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/keystone/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/keystone/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/keystone/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/keystone/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/keystone/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/keystone/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/keystone/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/keystone/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/keystone/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/keystone/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/keystone/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/keystone/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/keystone/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/keystone/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/keystone/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/keystone/requirements.lock b/charts/keystone/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/keystone/requirements.lock +++ b/charts/keystone/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/keystone/requirements.yaml b/charts/keystone/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/keystone/requirements.yaml +++ b/charts/keystone/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/libvirt/charts/helm-toolkit/Chart.yaml b/charts/libvirt/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/libvirt/charts/helm-toolkit/Chart.yaml +++ b/charts/libvirt/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/libvirt/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/libvirt/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/libvirt/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/libvirt/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/libvirt/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/libvirt/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/libvirt/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/libvirt/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/libvirt/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/libvirt/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/libvirt/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/libvirt/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/libvirt/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/libvirt/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/libvirt/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/libvirt/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/libvirt/requirements.lock b/charts/libvirt/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/libvirt/requirements.lock +++ b/charts/libvirt/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/libvirt/requirements.yaml b/charts/libvirt/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/libvirt/requirements.yaml +++ b/charts/libvirt/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/magnum/charts/helm-toolkit/Chart.yaml b/charts/magnum/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/magnum/charts/helm-toolkit/Chart.yaml +++ b/charts/magnum/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/magnum/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/magnum/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/magnum/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/magnum/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/magnum/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/magnum/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/magnum/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/magnum/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/magnum/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/magnum/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/magnum/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/magnum/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/magnum/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/magnum/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/magnum/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/magnum/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/magnum/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/magnum/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/magnum/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/magnum/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/magnum/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/magnum/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/magnum/requirements.lock b/charts/magnum/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/magnum/requirements.lock +++ b/charts/magnum/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/magnum/requirements.yaml b/charts/magnum/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/magnum/requirements.yaml +++ b/charts/magnum/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/manila/charts/helm-toolkit/Chart.yaml b/charts/manila/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/manila/charts/helm-toolkit/Chart.yaml +++ b/charts/manila/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/manila/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/manila/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/manila/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/manila/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/manila/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/manila/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/manila/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/manila/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/manila/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/manila/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/manila/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/manila/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/manila/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/manila/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/manila/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/manila/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/manila/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/manila/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/manila/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/manila/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/manila/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/manila/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/manila/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/manila/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/manila/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/manila/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/manila/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/manila/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/manila/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/manila/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/manila/requirements.lock b/charts/manila/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/manila/requirements.lock +++ b/charts/manila/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/manila/requirements.yaml b/charts/manila/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/manila/requirements.yaml +++ b/charts/manila/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/manila/templates/job-db-drop.yaml b/charts/manila/templates/job-db-drop.yaml index 9d63fa95b..61fa5f506 100644 --- a/charts/manila/templates/job-db-drop.yaml +++ b/charts/manila/templates/job-db-drop.yaml @@ -14,7 +14,7 @@ limitations under the License. {{- if .Values.manifests.job_db_drop }} {{- $serviceName := "manila" -}} -{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "DEFAULT" "configDbKey" "sql_connection" -}} +{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" -}} {{- $dbDropJob := dict "envAll" . "serviceName" $serviceName "dbToDrop" $dbToDrop -}} {{- if .Values.manifests.certificates -}} {{- $_ := set $dbToDrop "dbAdminTlsSecret" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal -}} diff --git a/charts/memcached/charts/helm-toolkit/Chart.yaml b/charts/memcached/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/memcached/charts/helm-toolkit/Chart.yaml +++ b/charts/memcached/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/memcached/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/memcached/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/memcached/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/memcached/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/memcached/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/memcached/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/memcached/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/memcached/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/memcached/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/memcached/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/memcached/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/memcached/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/memcached/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/memcached/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/memcached/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/memcached/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/memcached/requirements.lock b/charts/memcached/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/memcached/requirements.lock +++ b/charts/memcached/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/memcached/requirements.yaml b/charts/memcached/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/memcached/requirements.yaml +++ b/charts/memcached/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/neutron/charts/helm-toolkit/Chart.yaml b/charts/neutron/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/neutron/charts/helm-toolkit/Chart.yaml +++ b/charts/neutron/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/neutron/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/neutron/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/neutron/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/neutron/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/neutron/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/neutron/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/neutron/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/neutron/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/neutron/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/neutron/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/neutron/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/neutron/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/neutron/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/neutron/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/neutron/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/neutron/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/neutron/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/neutron/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/neutron/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/neutron/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/neutron/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/neutron/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/neutron/requirements.lock b/charts/neutron/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/neutron/requirements.lock +++ b/charts/neutron/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/neutron/requirements.yaml b/charts/neutron/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/neutron/requirements.yaml +++ b/charts/neutron/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/nova/charts/helm-toolkit/Chart.yaml b/charts/nova/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/nova/charts/helm-toolkit/Chart.yaml +++ b/charts/nova/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/nova/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/nova/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/nova/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/nova/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/nova/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/nova/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/nova/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/nova/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/nova/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/nova/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/nova/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/nova/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/nova/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/nova/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/nova/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/nova/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/nova/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/nova/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/nova/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/nova/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/nova/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/nova/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/nova/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/nova/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/nova/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/nova/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/nova/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/nova/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/nova/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/nova/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/nova/requirements.lock b/charts/nova/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/nova/requirements.lock +++ b/charts/nova/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/nova/requirements.yaml b/charts/nova/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/nova/requirements.yaml +++ b/charts/nova/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/octavia/charts/helm-toolkit/Chart.yaml b/charts/octavia/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/octavia/charts/helm-toolkit/Chart.yaml +++ b/charts/octavia/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/octavia/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/octavia/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/octavia/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/octavia/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/octavia/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/octavia/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/octavia/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/octavia/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/octavia/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/octavia/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/octavia/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/octavia/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/octavia/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/octavia/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/octavia/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/octavia/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/octavia/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/octavia/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/octavia/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/octavia/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/octavia/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/octavia/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/octavia/requirements.lock b/charts/octavia/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/octavia/requirements.lock +++ b/charts/octavia/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/octavia/requirements.yaml b/charts/octavia/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/octavia/requirements.yaml +++ b/charts/octavia/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/openvswitch/charts/helm-toolkit/Chart.yaml b/charts/openvswitch/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/openvswitch/charts/helm-toolkit/Chart.yaml +++ b/charts/openvswitch/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/openvswitch/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/openvswitch/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/openvswitch/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/openvswitch/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/openvswitch/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/openvswitch/requirements.lock b/charts/openvswitch/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/openvswitch/requirements.lock +++ b/charts/openvswitch/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/openvswitch/requirements.yaml b/charts/openvswitch/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/openvswitch/requirements.yaml +++ b/charts/openvswitch/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/ovn/charts/helm-toolkit/Chart.yaml b/charts/ovn/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/ovn/charts/helm-toolkit/Chart.yaml +++ b/charts/ovn/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/ovn/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/ovn/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/ovn/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/ovn/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/ovn/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/ovn/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/ovn/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/ovn/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/ovn/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/ovn/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/ovn/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/ovn/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/ovn/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/ovn/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/ovn/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/ovn/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/ovn/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/ovn/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/ovn/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/ovn/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/ovn/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/ovn/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/ovn/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/ovn/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/ovn/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/ovn/requirements.lock b/charts/ovn/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/ovn/requirements.lock +++ b/charts/ovn/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/ovn/requirements.yaml b/charts/ovn/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/ovn/requirements.yaml +++ b/charts/ovn/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/placement/charts/helm-toolkit/Chart.yaml b/charts/placement/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/placement/charts/helm-toolkit/Chart.yaml +++ b/charts/placement/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/placement/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/placement/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/placement/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/placement/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/placement/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/placement/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/placement/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/placement/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/placement/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/placement/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/placement/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/placement/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/placement/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/placement/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/placement/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/placement/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/placement/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/placement/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/placement/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/placement/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/placement/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/placement/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/placement/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/placement/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/placement/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/placement/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/placement/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/placement/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/placement/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/placement/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/placement/requirements.lock b/charts/placement/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/placement/requirements.lock +++ b/charts/placement/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/placement/requirements.yaml b/charts/placement/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/placement/requirements.yaml +++ b/charts/placement/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/charts/staffeln/charts/helm-toolkit/Chart.yaml b/charts/staffeln/charts/helm-toolkit/Chart.yaml index e6aec81ed..9d3aa1eb0 100644 --- a/charts/staffeln/charts/helm-toolkit/Chart.yaml +++ b/charts/staffeln/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.54 +version: 0.2.78 diff --git a/charts/staffeln/charts/helm-toolkit/requirements.lock b/charts/staffeln/charts/helm-toolkit/requirements.lock new file mode 100644 index 000000000..e28bc5d93 --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/requirements.lock @@ -0,0 +1,3 @@ +dependencies: [] +digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726 +generated: "0001-01-01T00:00:00Z" diff --git a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl index 12b84dec1..d7390d8be 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl @@ -50,7 +50,7 @@ return: | {{- $endpointScheme := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} {{- $userMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "auth" $userclass }} {{- $endpointUser := index $userMap "username" }} -{{- $endpointPass := index $userMap "password" }} +{{- $endpointPass := index $userMap "password" | urlquery }} {{- $endpointHost := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} {{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} {{- $endpointPath := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl index b7cf28738..b9ac9d9ab 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl @@ -100,7 +100,7 @@ examples: {{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false}} {{- $hostFqdnOverride := index $context.Values.endpoints ( $type | replace "-" "_" ) "host_fqdn_override" }} {{- $endpointUser := index $userMap "username" }} -{{- $endpointPass := index $userMap "password" }} +{{- $endpointPass := index $userMap "password" | urlquery }} {{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} {{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} {{- $local := dict "endpointCredsAndHosts" list -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_ingress.tpl index 4c476b2ce..792571cb7 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -59,7 +59,7 @@ examples: default: 9311 public: 80 usage: | - {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}} + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}} return: | --- apiVersion: networking.k8s.io/v1 @@ -67,16 +67,16 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" rules: - host: barbican http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -86,7 +86,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -96,7 +96,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -108,10 +108,10 @@ examples: metadata: name: barbican-namespace-fqdn annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public hosts: @@ -121,7 +121,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -133,10 +133,10 @@ examples: metadata: name: barbican-cluster-fqdn annotations: - kubernetes.io/ingress.class: "nginx-cluster" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx-cluster" tls: - secretName: barbican-tls-public hosts: @@ -146,7 +146,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -194,7 +194,7 @@ examples: default: 9311 public: 80 usage: | - {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" ) -}} + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}} return: | --- apiVersion: networking.k8s.io/v1 @@ -202,10 +202,10 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public hosts: @@ -217,7 +217,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -227,7 +227,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -237,7 +237,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -294,7 +294,7 @@ examples: name: ca-issuer kind: Issuer usage: | - {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}} + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}} return: | --- apiVersion: networking.k8s.io/v1 @@ -302,12 +302,12 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" cert-manager.io/issuer: ca-issuer certmanager.k8s.io/issuer: ca-issuer nginx.ingress.kubernetes.io/backend-protocol: https nginx.ingress.kubernetes.io/secure-backends: "true" spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public-certmanager hosts: @@ -319,7 +319,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -329,7 +329,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -339,7 +339,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -396,7 +396,7 @@ examples: name: ca-issuer kind: ClusterIssuer usage: | - {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer") -}} + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}} return: | --- apiVersion: networking.k8s.io/v1 @@ -404,12 +404,12 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" cert-manager.io/cluster-issuer: ca-issuer certmanager.k8s.io/cluster-issuer: ca-issuer nginx.ingress.kubernetes.io/backend-protocol: https nginx.ingress.kubernetes.io/secure-backends: "true" spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public-certmanager hosts: @@ -421,7 +421,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -431,7 +431,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -441,7 +441,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: barbican-api @@ -479,7 +479,7 @@ examples: grafana: public: grafana-tls-public usage: | - {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}} + {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}} {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} return: | --- @@ -488,16 +488,16 @@ examples: metadata: name: grafana annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" rules: - host: grafana http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -507,7 +507,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -517,7 +517,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -529,10 +529,10 @@ examples: metadata: name: grafana-namespace-fqdn annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: grafana-tls-public hosts: @@ -543,7 +543,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -553,7 +553,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -565,10 +565,10 @@ examples: metadata: name: grafana-cluster-fqdn annotations: - kubernetes.io/ingress.class: "nginx-cluster" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx-cluster" tls: - secretName: grafana-tls-public hosts: @@ -579,7 +579,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -589,7 +589,7 @@ examples: http: paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix backend: service: name: grafana-dashboard @@ -602,11 +602,12 @@ examples: {{- $vHost := index . "vHost" -}} {{- $backendName := index . "backendName" -}} {{- $backendPort := index . "backendPort" -}} +{{- $pathType := index . "pathType" -}} - host: {{ $vHost }} http: paths: - path: / - pathType: ImplementationSpecific + pathType: {{ $pathType }} backend: service: name: {{ $backendName }} @@ -624,6 +625,7 @@ examples: {{- $backendServiceType := index . "backendServiceType" -}} {{- $backendPort := index . "backendPort" -}} {{- $endpoint := index . "endpoint" | default "public" -}} +{{- $pathType := index . "pathType" | default "Prefix" -}} {{- $certIssuer := index . "certIssuer" | default "" -}} {{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} {{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} @@ -639,7 +641,6 @@ kind: Ingress metadata: name: {{ $ingressName }} annotations: - kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }} {{- if $certIssuer }} cert-manager.io/{{ $certIssuerType }}: {{ $certIssuer }} certmanager.k8s.io/{{ $certIssuerType }}: {{ $certIssuer }} @@ -650,6 +651,7 @@ metadata: {{- end }} {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }} {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }} {{- if $certIssuer }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} @@ -681,7 +683,7 @@ spec: {{- end }} rules: {{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }} -{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }} +{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }} {{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }} {{- end }} {{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }} @@ -695,9 +697,9 @@ kind: Ingress metadata: name: {{ printf "%s-%s-%s" $ingressName $ingressController "fqdn" }} annotations: - kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }} {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }} {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }} {{- if hasKey $host $endpoint }} {{- $endpointHost := index $host $endpoint }} @@ -706,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: @@ -719,7 +725,7 @@ spec: {{- end }} rules: {{- range $vHost := $vHosts }} -{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort }} +{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }} {{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }} {{- end }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl index 5d98c8b7f..6b77004f0 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-bootstrap.tpl @@ -51,6 +51,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl index 62ed11916..2b7ff2cdc 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl @@ -54,6 +54,7 @@ metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-delete-policy": hook-succeeded +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl index 745e8dab8..b8a1dce3b 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl @@ -52,6 +52,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl index 24d2496d1..4696c88fd 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-db-sync.tpl @@ -49,6 +49,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl index 3a7df7ff9..d69c9e6ec 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl @@ -52,6 +52,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl index a109e3cc0..9604c6372 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-service.tpl @@ -52,6 +52,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl index 905eb71a6..58dcdc5c6 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl @@ -74,6 +74,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl index 698206426..2cfadafe3 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl @@ -42,6 +42,7 @@ metadata: {{ toYaml $jobLabels | indent 4 }} {{- end }} annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl index 29cb99378..b5fdc09c3 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl @@ -49,6 +49,7 @@ metadata: {{- end }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl index 50d9af599..77d1a71e9 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl @@ -47,6 +47,7 @@ metadata: annotations: "helm.sh/hook-delete-policy": before-hook-creation {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} {{- if $jobAnnotations }} {{ toYaml $jobAnnotations | indent 4 }} {{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl index 4854bb1ec..7ad505b55 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl @@ -17,6 +17,11 @@ abstract: | Creates a manifest for a authenticating a registry with a secret examples: - values: | + annotations: + secret: + oci_image_registry: + {{ $serviceName }}: + custom.tld/key: "value" secrets: oci_image_registry: {{ $serviceName }}: {{ $keyName }} @@ -36,30 +41,8 @@ examples: kind: Secret metadata: name: {{ $secretName }} - type: kubernetes.io/dockerconfigjson - data: - dockerconfigjson: {{ $dockerAuth }} - - - values: | - secrets: - oci_image_registry: - {{ $serviceName }}: {{ $keyName }} - endpoints: - oci_image_registry: - name: oci-image-registry - auth: - enabled: true - {{ $serviceName }}: - name: {{ $userName }} - password: {{ $password }} - usage: | - {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}} - return: | - --- - apiVersion: v1 - kind: Secret - metadata: - name: {{ $secretName }} + annotations: + custom.tld/key: "value" type: kubernetes.io/dockerconfigjson data: dockerconfigjson: {{ $dockerAuth }} @@ -87,6 +70,8 @@ apiVersion: v1 kind: Secret metadata: name: {{ $secretName }} + annotations: +{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }} type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: {{ $dockerAuth }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl index 24a70450c..c80034030 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl @@ -17,6 +17,11 @@ abstract: | Creates a manifest for a services public tls secret examples: - values: | + annotations: + secret: + tls: + key_manager_api_public: + custom.tld/key: "value" secrets: tls: key_manager: @@ -41,6 +46,8 @@ examples: kind: Secret metadata: name: barbican-tls-public + annotations: + custom.tld/key: "value" type: kubernetes.io/tls data: tls.key: Rk9PLUtFWQo= @@ -88,11 +95,15 @@ examples: {{- if kindIs "map" $endpointHost }} {{- if hasKey $endpointHost "tls" }} {{- if and $endpointHost.tls.key $endpointHost.tls.crt }} + +{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} --- apiVersion: v1 kind: Secret metadata: name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} + annotations: +{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }} type: kubernetes.io/tls data: tls.key: {{ $endpointHost.tls.key | b64enc }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 03884fa18..c6a7521d2 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -124,7 +125,12 @@ except: # Delete DB try: - root_engine.execute("DROP DATABASE IF EXISTS {0}".format(database)) + with root_engine.connect() as connection: + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) + try: + connection.commit() + except AttributeError: + pass logger.info("Deleted database {0}".format(database)) except: logger.critical("Could not drop database {0}".format(database)) @@ -132,7 +138,12 @@ except: # Delete DB User try: - root_engine.execute("DROP USER IF EXISTS {0}".format(user)) + with root_engine.connect() as connection: + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) + try: + connection.commit() + except AttributeError: + pass logger.info("Deleted user {0}".format(user)) except: logger.critical("Could not delete user {0}".format(user)) diff --git a/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 6027b9515..1917f78b4 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -124,7 +125,12 @@ except: # Create DB try: - root_engine.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + with root_engine.connect() as connection: + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) + try: + connection.commit() + except AttributeError: + pass logger.info("Created database {0}".format(database)) except: logger.critical("Could not create database {0}".format(database)) @@ -132,11 +138,16 @@ except: # Create DB User try: - root_engine.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) - root_engine.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + with root_engine.connect() as connection: + connection.execute( + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) + connection.execute( + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) + try: + connection.commit() + except AttributeError: + pass logger.info("Created user {0} for {1}".format(user, database)) except: logger.critical("Could not create user {0} for {1}".format(user, database)) diff --git a/charts/staffeln/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/staffeln/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/staffeln/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/charts/staffeln/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl index 3963bd405..695cb2e47 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl @@ -49,6 +49,13 @@ # A random number between min and max delay is generated # to set the delay. # +# RGW backup throttle limits variables: +# export THROTTLE_BACKUPS_ENABLED Boolean variableto control backup functionality +# export THROTTLE_LIMIT Number of simultaneous RGW upload sessions +# export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned +# export THROTTLE_RETRY_AFTER Time in seconds to wait before retry +# export THROTTLE_CONTAINER_NAME Name of RGW container to place flag falies into +# # The database-specific functions that need to be implemented are: # dump_databases_to_directory [scope] # where: @@ -84,8 +91,10 @@ # specified by the "LOCAL_DAYS_TO_KEEP" variable. # 4) Removing remote backup tarballs (from the remote gateway) which are older # than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable. +# 5) Controlling remote storage gateway load from client side and throttling it +# by using a dedicated RGW container to store flag files defining upload session +# in progress # - # Note: not using set -e in this script because more elaborate error handling # is needed. @@ -95,7 +104,7 @@ log_backup_error_exit() { log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}" rm -f $ERR_LOG_FILE rm -rf $TMP_DIR - exit $ERRCODE + exit 0 } log_verify_backup_exit() { @@ -104,7 +113,7 @@ log_verify_backup_exit() { log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}" rm -f $ERR_LOG_FILE # rm -rf $TMP_DIR - exit $ERRCODE + exit 0 } @@ -218,6 +227,113 @@ send_to_remote_server() { echo "Sleeping for ${DELAY} seconds to spread the load in time..." sleep ${DELAY} + #--------------------------------------------------------------------------- + # Remote backup throttling + export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g') + if $THROTTLE_BACKUPS_ENABLED; then + # Remove Quotes from the constants which were added due to reading + # from secret. + export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g') + export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g') + export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g') + export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g') + + # load balance delay + RESULT=$(openstack container list 2>&1) + + if [[ $? -eq 0 ]]; then + echo $RESULT | grep $THROTTLE_CONTAINER_NAME + if [[ $? -ne 0 ]]; then + # Find the swift URL from the keystone endpoint list + SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}') + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog." + return 2 + fi + + # Get a token from keystone + TOKEN=$(openstack token issue -f value -c id) + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get keystone token." + return 2 + fi + + # Create the container + RES_FILE=$(mktemp -p /tmp) + curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \ + -H "X-Auth-Token: ${TOKEN}" \ + -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE + + if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then + log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}" + cat $RES_FILE + rm -f $RES_FILE + return 2 + fi + rm -f $RES_FILE + + swift stat $THROTTLE_CONTAINER_NAME + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation." + return 2 + fi + fi + else + echo $RESULT | grep -E "HTTP 401|HTTP 403" + if [[ $? -eq 0 ]]; then + log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}" + return 1 + else + echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50" + if [[ $? -eq 0 ]]; then + log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}" + # In this case, keystone or the site/node may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}" + return 1 + fi + fi + fi + + NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l) + log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now." + while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]] + do + log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!" + log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...." + sleep ${THROTTLE_RETRY_AFTER} + NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l) + log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now." + done + + # Create a lock file in THROTTLE_CONTAINER + THROTTLE_FILEPATH=$(mktemp -d) + THROTTLE_FILE=${CONTAINER_NAME}.lock + date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE + + # Create an object to store the file + openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!" + return 2 + fi + + swift post $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}" + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!" + return 2 + fi + openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation." + return 2 + fi + fi + + #--------------------------------------------------------------------------- + # Create an object to store the file openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE if [[ $? -ne 0 ]]; then @@ -243,7 +359,25 @@ send_to_remote_server() { log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values" return 2 fi - rm -rf ${REMOTE_FILE} + rm -f ${REMOTE_FILE} + + #--------------------------------------------------------------------------- + # Remote backup throttling + export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g') + if $THROTTLE_BACKUPS_ENABLED; then + # Remove flag file + # Delete an object to remove the flag file + openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}" + return 0 + else + log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed." + fi + rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE} + fi + + #--------------------------------------------------------------------------- log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully." return 0 diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl new file mode 100644 index 000000000..fc426142f --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_job_annotations.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the job spec of a component. +examples: + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + usage: | + {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + usage: | + {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + keystone_bootstrap: + usage: | + {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_job_annotations" -}} +{{- $envAll := index . 1 -}} +{{- $component := index . 0 | replace "-" "_" -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "job") -}} +{{- $annotationsMap := $envAll.Values.annotations.job -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $annotationsMap "default" ) -}} +{{- $defaultAnnotations = $annotationsMap.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl new file mode 100644 index 000000000..ecff6e96a --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the pod spec of a component. +examples: + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + usage: | + {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + usage: | + {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + nova_api: + usage: | + {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_pod_annotations" -}} +{{- $component := index . 0 -}} +{{- $envAll := index . 1 -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "pod") -}} +{{- $annotationsMap := $envAll.Values.annotations.pod -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $annotationsMap "default" ) -}} +{{- $defaultAnnotations = $annotationsMap.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl new file mode 100644 index 000000000..19c438088 --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl @@ -0,0 +1,81 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the secret spec of a component. +examples: + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + usage: | + {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + usage: | + {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + oslo_db: + admin: + usage: | + {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_secret_annotations" -}} +{{- $secretType := index . 0 -}} +{{- $userClass := index . 1 | replace "-" "_" -}} +{{- $envAll := index . 2 -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "secret") -}} +{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}} +{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_image.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_image.tpl index 029c93de5..678b8447f 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/snippets/_image.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_image.tpl @@ -19,7 +19,7 @@ values: | images: tags: test_image: docker.io/port/test:version-foo - image_foo: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + image_foo: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal pull_policy: IfNotPresent local_registry: active: true diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl index bed712e59..ad628daca 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl @@ -19,7 +19,7 @@ abstract: | values: | images: tags: - dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal pull_policy: IfNotPresent local_registry: active: true @@ -76,7 +76,7 @@ usage: | {{ tuple . "calico_node" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" }} return: | - name: init - image: "quay.io/airshipit/kubernetes-entrypoint:v1.0.0" + image: "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal" imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: false diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl new file mode 100644 index 000000000..08521e0fe --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }} +{{- range $s3Bucket := .Values.storage.s3.buckets }} +- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_ACCESS_KEY_ID +- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_SECRET_ACCESS_KEY +{{- end }} +{{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/staffeln/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/staffeln/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/staffeln/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/staffeln/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/staffeln/requirements.lock b/charts/staffeln/requirements.lock new file mode 100644 index 000000000..e28bc5d93 --- /dev/null +++ b/charts/staffeln/requirements.lock @@ -0,0 +1,3 @@ +dependencies: [] +digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726 +generated: "0001-01-01T00:00:00Z" diff --git a/charts/staffeln/requirements.yaml b/charts/staffeln/requirements.yaml index 4124d0145..52e723e18 100644 --- a/charts/staffeln/requirements.yaml +++ b/charts/staffeln/requirements.yaml @@ -1,16 +1,4 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - dependencies: - - name: helm-toolkit - repository: file://../../openstack-helm-infra/helm-toolkit - version: ">= 0.1.0" +- name: helm-toolkit + repository: https://tarballs.openstack.org/openstack-helm-infra + version: 0.2.78 diff --git a/charts/staffeln/templates/job-db-drop.yaml b/charts/staffeln/templates/job-db-drop.yaml index dffa5aba8..906c88d4d 100644 --- a/charts/staffeln/templates/job-db-drop.yaml +++ b/charts/staffeln/templates/job-db-drop.yaml @@ -14,7 +14,7 @@ limitations under the License. {{- if .Values.manifests.job_db_drop }} {{- $serviceName := "staffeln" -}} -{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "DEFAULT" "configDbKey" "sql_connection" -}} +{{- $dbToDrop := dict "adminSecret" .Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" -}} {{- $dbDropJob := dict "envAll" . "serviceName" $serviceName "dbToDrop" $dbToDrop -}} {{- if .Values.manifests.certificates -}} {{- $_ := set $dbToDrop "dbAdminTlsSecret" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal -}} diff --git a/charts/tempest/charts/helm-toolkit/Chart.yaml b/charts/tempest/charts/helm-toolkit/Chart.yaml index d4c0ea20c..9d3aa1eb0 100644 --- a/charts/tempest/charts/helm-toolkit/Chart.yaml +++ b/charts/tempest/charts/helm-toolkit/Chart.yaml @@ -9,4 +9,4 @@ name: helm-toolkit sources: - https://opendev.org/openstack/openstack-helm-infra - https://opendev.org/openstack/openstack-helm -version: 0.2.69 +version: 0.2.78 diff --git a/charts/tempest/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/charts/tempest/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl index 6877b7bfb..728b99435 100644 --- a/charts/tempest/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -14,7 +14,8 @@ limitations under the License. {{/* abstract: | - Resolves 'hostname:port' for an endpoint + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', examples: - values: | endpoints: @@ -46,6 +47,23 @@ examples: {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} return: | 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 */}} {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} @@ -53,7 +71,19 @@ examples: {{- $endpoint := index . 1 -}} {{- $port := index . 2 -}} {{- $context := index . 3 -}} -{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} -{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} -{{- printf "%s:%s" $endpointHostname $endpointPort -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} {{- end -}} diff --git a/charts/tempest/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/charts/tempest/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl index b2ec6486c..24eb56942 100644 --- a/charts/tempest/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -40,9 +40,9 @@ return: | {{- $context := index . 3 -}} {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} {{- if kindIs "string" $endpointMap.path }} -{{- printf "%s" $endpointMap.path | default "/" -}} +{{- printf "%s" $endpointMap.path | default "" -}} {{- else -}} -{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} {{- printf "%s" $endpointPath -}} {{- end -}} {{- end -}} diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 000000000..332ca9943 --- /dev/null +++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl b/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl index cacb4b813..792571cb7 100644 --- a/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/manifests/_ingress.tpl @@ -708,6 +708,10 @@ spec: {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} {{- $vHosts = append $vHosts $v }} {{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} tls: diff --git a/charts/tempest/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl b/charts/tempest/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl index 1e28da9ca..c6a7521d2 100644 --- a/charts/tempest/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Drop') @@ -125,7 +126,7 @@ except: # Delete DB try: with root_engine.connect() as connection: - connection.execute("DROP DATABASE IF EXISTS {0}".format(database)) + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -138,7 +139,7 @@ except: # Delete DB User try: with root_engine.connect() as connection: - connection.execute("DROP USER IF EXISTS {0}".format(user)) + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) try: connection.commit() except AttributeError: diff --git a/charts/tempest/charts/helm-toolkit/templates/scripts/_db-init.py.tpl b/charts/tempest/charts/helm-toolkit/templates/scripts/_db-init.py.tpl index 110cd98eb..1917f78b4 100644 --- a/charts/tempest/charts/helm-toolkit/templates/scripts/_db-init.py.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -33,6 +33,7 @@ except ImportError: PARSER_OPTS = {"strict": False} import logging from sqlalchemy import create_engine +from sqlalchemy import text # Create logger, console handler and formatter logger = logging.getLogger('OpenStack-Helm DB Init') @@ -125,7 +126,7 @@ except: # Create DB try: with root_engine.connect() as connection: - connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) try: connection.commit() except AttributeError: @@ -139,10 +140,10 @@ except: try: with root_engine.connect() as connection: connection.execute( - "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( - user, password, mysql_x509)) + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) connection.execute( - "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)) + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))) try: connection.commit() except AttributeError: diff --git a/charts/tempest/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/charts/tempest/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl index 3739f9554..e6e9c6e8e 100644 --- a/charts/tempest/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -29,7 +29,9 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User creadential RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ @@ -37,7 +39,9 @@ RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[//:]' '{print $4}') RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ awk -F'[@]' '{print $1}' | \ - awk -F'[//:]' '{print $5}') + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") # Extract User vHost RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl index 48b53fa10..5c2dedb06 100644 --- a/charts/tempest/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl +++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -40,6 +40,9 @@ return: | release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} application: {{ $application }} component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} {{- if ($envAll.Values.pod).labels }} {{- if hasKey $envAll.Values.pod.labels $component }} {{ index $envAll.Values.pod "labels" $component | toYaml }} diff --git a/charts/tempest/charts/helm-toolkit/templates/snippets/_service_params.tpl b/charts/tempest/charts/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 000000000..6233a9355 --- /dev/null +++ b/charts/tempest/charts/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/charts/tempest/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/charts/tempest/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 000000000..bdb28c331 --- /dev/null +++ b/charts/tempest/charts/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/charts/tempest/requirements.lock b/charts/tempest/requirements.lock index 4f63a87d4..b94deaddb 100644 --- a/charts/tempest/requirements.lock +++ b/charts/tempest/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 -digest: sha256:f971f98746c97193da5ff7a44d2401ae7d91201a49ed9f23d52359a1b6e9d0ef + version: 0.2.78 +digest: sha256:760c5997580ac8d30a64dfe92d6a54052b783ddf90cacbc5a4c95c1330988742 generated: "0001-01-01T00:00:00Z" diff --git a/charts/tempest/requirements.yaml b/charts/tempest/requirements.yaml index b3d5dcfc9..52e723e18 100644 --- a/charts/tempest/requirements.yaml +++ b/charts/tempest/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: helm-toolkit repository: https://tarballs.openstack.org/openstack-helm-infra - version: 0.2.69 + version: 0.2.78 diff --git a/images/barbican/Dockerfile b/images/barbican/Dockerfile index 6d6e07bdc..8372ab3c7 100644 --- a/images/barbican/Dockerfile +++ b/images/barbican/Dockerfile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -# Atmosphere-Rebuild-Time: 2024-06-25T22:49:25Z +# Atmosphere-Rebuild-Time: 2024-10-29T22:19:48Z ARG RELEASE FROM harbor.atmosphere.dev/library/openstack-venv-builder:${RELEASE} AS build -ARG BARBICAN_GIT_REF=ca57ef5436e20e90cf6cd6853efe3c89a9afd986 +ARG BARBICAN_GIT_REF=e1f7b30e9381932fe26f31d4a1f675be7d9242d9 ADD --keep-git-dir=true https://opendev.org/openstack/barbican.git#${BARBICAN_GIT_REF} /src/barbican RUN git -C /src/barbican fetch --unshallow RUN --mount=type=cache,mode=0755,target=/root/.cache/pip,sharing=private < -Date: Thu, 04 Jul 2024 14:09:23 -0400 -Subject: [PATCH] Fix port group network metadata generation - -When switching to using OpenStack SDK, there was a change missed -that didn't account for the SDK returning generators instead of -a list, so the loop on ports and port groups made it so that it -started returning an empty list afterwards. - -Since there is no a masse of ports for a baremetal system usually, -we take the generator into a list right away to prevent this. - -Closes-Bug: #2071972 -Change-Id: I90766f8c225d834bb2eec606754107ea6a212f6d ---- - -diff --git a/nova/tests/unit/virt/ironic/test_driver.py b/nova/tests/unit/virt/ironic/test_driver.py -index 736eac2..bf32918 100644 ---- a/nova/tests/unit/virt/ironic/test_driver.py -+++ b/nova/tests/unit/virt/ironic/test_driver.py -@@ -2785,8 +2785,8 @@ - node_id=self.node.id, - address='00:00:00:00:00:02', - port_group_id=portgroup.id) -- self.mock_conn.ports.return_value = [port1, port2] -- self.mock_conn.port_groups.return_value = [portgroup] -+ self.mock_conn.ports.return_value = iter([port1, port2]) -+ self.mock_conn.port_groups.return_value = iter([portgroup]) - - metadata = self.driver._get_network_metadata(self.node, - self.network_info) -diff --git a/nova/virt/ironic/driver.py b/nova/virt/ironic/driver.py -index 3af85d3..c990218 100644 ---- a/nova/virt/ironic/driver.py -+++ b/nova/virt/ironic/driver.py -@@ -1050,10 +1050,10 @@ - :param network_info: Instance network information. - """ - base_metadata = netutils.get_network_metadata(network_info) -- ports = self.ironic_connection.ports(node=node.id, details=True) -- port_groups = self.ironic_connection.port_groups( -+ ports = list(self.ironic_connection.ports(node=node.id, details=True)) -+ port_groups = list(self.ironic_connection.port_groups( - node=node.id, details=True, -- ) -+ )) - vif_id_to_objects = {'ports': {}, 'portgroups': {}} - for collection, name in ((ports, 'ports'), - (port_groups, 'portgroups')): diff --git a/images/octavia/Dockerfile b/images/octavia/Dockerfile index e7aa491b7..cbe6c2a3e 100644 --- a/images/octavia/Dockerfile +++ b/images/octavia/Dockerfile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -# Atmosphere-Rebuild-Time: 2024-06-25T22:49:25Z +# Atmosphere-Rebuild-Time: 2024-10-29T22:19:48Z ARG RELEASE FROM harbor.atmosphere.dev/library/openstack-venv-builder:${RELEASE} AS build -ARG OCTAVIA_GIT_REF=824b51a1dad80292b7a8ad5d61bf3ce706b1fb29 +ARG OCTAVIA_GIT_REF=fd071d3391e5695e50748f74af4529186d63e472 ADD --keep-git-dir=true https://opendev.org/openstack/octavia.git#${OCTAVIA_GIT_REF} /src/octavia RUN git -C /src/octavia fetch --unshallow ADD --keep-git-dir=true https://opendev.org/openstack/ovn-octavia-provider.git#master /src/ovn-octavia-provider diff --git a/images/openstack-runtime/Dockerfile b/images/openstack-runtime/Dockerfile index d968a317b..c950d143c 100644 --- a/images/openstack-runtime/Dockerfile +++ b/images/openstack-runtime/Dockerfile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Atmosphere-Rebuild-Time: 2024-06-25T22:49:25Z +# Atmosphere-Rebuild-Time: 2024-10-14T18:43:39Z ARG RELEASE diff --git a/images/openstack-venv-builder/Dockerfile b/images/openstack-venv-builder/Dockerfile index 5aaf023c5..80131dfde 100644 --- a/images/openstack-venv-builder/Dockerfile +++ b/images/openstack-venv-builder/Dockerfile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -# Atmosphere-Rebuild-Time: 2024-06-25T22:49:25Z +# Atmosphere-Rebuild-Time: 2024-10-29T22:19:48Z ARG RELEASE FROM harbor.atmosphere.dev/library/ubuntu-cloud-archive:${RELEASE} AS requirements -ARG REQUIREMENTS_GIT_REF=18098b9abacbd8d7257bebc1b302294f634441ab +ARG REQUIREMENTS_GIT_REF=f2d82273f2bad2dd670b7d1f6047353f61a098f4 ADD --keep-git-dir=true https://opendev.org/openstack/requirements.git#${REQUIREMENTS_GIT_REF} /src/requirements RUN cp /src/requirements/upper-constraints.txt /upper-constraints.txt RUN <