[tmp] Noble images

.charts.yml

@@ -6,7 +6,7 @@
   openstack_helm_dependencies: &openstack_helm_dependencies
     - name: helm-toolkit
       repository: https://tarballs.openstack.org/openstack-helm-infra
-      version: 0.2.69
+      version: 0.2.78
 
 charts:
   - name: barbican

build/lint-jobs.py

@@ -7,6 +7,9 @@ def main():
     passed = True
 
     for file in glob.glob("zuul.d/container-images/*.yaml"):
+        if file == "zuul.d/container-images/base.yaml":
+            continue
+
         with open(file, "r") as file:
             configs = yaml.safe_load(file)
 

build/manage-dockerfile.py

@@ -2,10 +2,17 @@
 
 import argparse
 import pathlib
+import re
 from datetime import datetime
 
+import requests
 
-def update_dockerfile(dockerfile, timestamp):
+OPENDEV_GIT_PATTERN = re.compile(
+    r"ADD (?:--keep-git-dir=true )?https://opendev.org/(?P<namespace>[^/]+)/(?P<repo>[^/]+)\.git#\$\{(?P<ref>[^}]+)\} /src/(?P<dir>[^ ]+)?"  # noqa
+)
+
+
+def update_rebuild_time(dockerfile, timestamp):
     rebuild_line = f"# Atmosphere-Rebuild-Time: {timestamp}"
 
     lines = dockerfile.read_text().splitlines()
@@ -25,19 +32,66 @@ def update_dockerfile(dockerfile, timestamp):
     dockerfile.write_text("\n".join(new_lines) + "\n")
 
 
+def bump_image(dockerfile, branch) -> bool:
+    content = dockerfile.read_text()
+
+    opendev_match = OPENDEV_GIT_PATTERN.search(content)
+    if opendev_match:
+        namespace = opendev_match.group("namespace")
+        repo = opendev_match.group("repo")
+        ref = opendev_match.group("ref")
+        print(f"Bumping {repo} ({ref}) in {dockerfile}")
+
+        url = f"https://opendev.org/api/v1/repos/{namespace}/{repo}/branches/{branch}"
+        response = requests.get(url)
+        response.raise_for_status()
+        data = response.json()
+
+        # Update the Dockerfile with the latest commit
+        lines = content.splitlines()
+        new_lines = []
+        for line in lines:
+            if line.startswith(f"ARG {ref}="):
+                new_lines.append(f"ARG {ref}={data['commit']['id']}")
+            else:
+                new_lines.append(line)
+
+        dockerfile.write_text("\n".join(new_lines) + "\n")
+        return True
+
+    # TODO: add support for other types of images
+
+    print(f"Skipping {dockerfile}: No way to bump image")
+    return False
+
+
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--rebuild", action="store_true")
-    parser.add_argument(
+    subparsers = parser.add_subparsers(dest="command")
+
+    force_rebuild_parser = subparsers.add_parser("force-rebuild")
+    force_rebuild_parser.add_argument(
+        "dockerfiles", type=pathlib.Path, nargs="+", help="List of Dockerfiles"
+    )
+
+    bump_parser = subparsers.add_parser("bump")
+    bump_parser.add_argument(
+        "--branch",
+        help="Branch to bump the image to",
+    )
+    bump_parser.add_argument(
         "dockerfiles", type=pathlib.Path, nargs="+", help="List of Dockerfiles"
     )
-    args = parser.parse_args()
 
-    if args.rebuild:
-        now_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    args = parser.parse_args()
+    now_utc = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
 
-        for dockerfile in args.dockerfiles:
-            update_dockerfile(dockerfile, now_utc)
+    for dockerfile in args.dockerfiles:
+        if args.command == "bump":
+            if bump_image(dockerfile, args.branch):
+                update_rebuild_time(dockerfile, now_utc)
+        elif args.command == "force-rebuild":
+            update_rebuild_time(dockerfile, now_utc)
 
 
 if __name__ == "__main__":

charts/barbican/charts/helm-toolkit/Chart.yaml

@@ -9,4 +9,4 @@ name: helm-toolkit
 sources:
 - https://opendev.org/openstack/openstack-helm-infra
 - https://opendev.org/openstack/openstack-helm
-version: 0.2.69
+version: 0.2.78

charts/barbican/charts/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl

@@ -14,7 +14,8 @@ limitations under the License.
 
 {{/*
 abstract: |
-  Resolves 'hostname:port' for an endpoint
+  Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g
+  'hostname1:port1,hostname2:port2,hostname3:port3',
 examples:
   - values: |
       endpoints:
@@ -46,14 +47,43 @@ examples:
       {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
     return: |
       127.0.0.1:3306
+  - values: |
+      endpoints:
+        oslo_cache:
+          hosts:
+            default: memcached
+          host_fqdn_override:
+            default: null
+          statefulset:
+            name: openstack-memcached-memcached
+            replicas: 3
+          port:
+            memcache:
+              default: 11211
+    usage: |
+      {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+    return: |
+      openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211
 */}}
 
 {{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
 {{- $type := index . 0 -}}
 {{- $endpoint := index . 1 -}}
 {{- $port := index . 2 -}}
 {{- $context := index . 3 -}}
-{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }}
-{{- printf "%s:%s" $endpointHostname $endpointPort -}}
+{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}}
+{{- $local := dict "endpointHosts" list -}}
+{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}}
+{{- if $ssMap -}}
+{{-   $endpointHostPrefix := $ssMap.name -}}
+{{-   $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }}
+{{-   range $podInt := until ( atoi (print $ssMap.replicas ) ) -}}
+{{-     $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}}
+{{-     $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}}
+{{-   end -}}
+{{- else -}}
+{{-   $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}}
+{{-   $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}}
+{{- end -}}
+{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }}
 {{- end -}}

charts/barbican/charts/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl

@@ -40,9 +40,9 @@ return: |
 {{- $context := index . 3 -}}
 {{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }}
 {{- if kindIs "string" $endpointMap.path }}
-{{- printf "%s" $endpointMap.path | default "/" -}}
+{{- printf "%s" $endpointMap.path | default "" -}}
 {{- else -}}
-{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "/" }}
+{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }}
 {{- printf "%s" $endpointPath -}}
 {{- end -}}
 {{- end -}}

charts/barbican/charts/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl

@@ -0,0 +1,51 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{/*
+abstract: |
+  Renders out the configmap <service>-oslo-policy.
+values: |
+  conf:
+    policy.d:
+      file1:
+        foo: bar
+      file2:
+        foo: baz
+usage: |
+{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }}
+return: |
+  ---
+  apiVersion: v1
+  kind: Secret
+  metadata:
+    name: keystone-oslo-policy
+  data:
+    file1: base64of(foo: bar)
+    file2: base64of(foo: baz)
+*/}}
+{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}}
+{{- $envAll := index . "envAll" -}}
+{{- $serviceName := index . "serviceName" -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $serviceName }}-oslo-policy
+type: Opaque
+data:
+  {{- range $key, $value := index $envAll.Values.conf "policy.d" }}
+  {{- if $value }}
+  {{ $key }}: {{ toYaml $value | b64enc }}
+  {{- else }}
+  {{ $key }}: {{ "\n" | b64enc }}
+  {{- end }}
+  {{- end }}
+{{- end -}}

charts/barbican/charts/helm-toolkit/templates/manifests/_ingress.tpl

@@ -708,6 +708,10 @@ spec:
 {{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }}
 {{- $vHosts = append $vHosts $v }}
 {{- end }}
+{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }}
+{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }}
+{{- $vHosts = append $vHosts $alias_host }}
+{{- end }}
 {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
 {{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }}
   tls:

charts/barbican/charts/helm-toolkit/templates/scripts/_db-drop.py.tpl

@@ -33,6 +33,7 @@ except ImportError:
     PARSER_OPTS = {"strict": False}
 import logging
 from sqlalchemy import create_engine
+from sqlalchemy import text
 
 # Create logger, console handler and formatter
 logger = logging.getLogger('OpenStack-Helm DB Drop')
@@ -125,7 +126,7 @@ except:
 # Delete DB
 try:
     with root_engine.connect() as connection:
-        connection.execute("DROP DATABASE IF EXISTS {0}".format(database))
+        connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database)))
         try:
             connection.commit()
         except AttributeError:
@@ -138,7 +139,7 @@ except:
 # Delete DB User
 try:
     with root_engine.connect() as connection:
-        connection.execute("DROP USER IF EXISTS {0}".format(user))
+        connection.execute(text("DROP USER IF EXISTS {0}".format(user)))
         try:
             connection.commit()
         except AttributeError:

charts/barbican/charts/helm-toolkit/templates/scripts/_db-init.py.tpl

@@ -33,6 +33,7 @@ except ImportError:
     PARSER_OPTS = {"strict": False}
 import logging
 from sqlalchemy import create_engine
+from sqlalchemy import text
 
 # Create logger, console handler and formatter
 logger = logging.getLogger('OpenStack-Helm DB Init')
@@ -125,7 +126,7 @@ except:
 # Create DB
 try:
     with root_engine.connect() as connection:
-        connection.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database))
+        connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database)))
         try:
             connection.commit()
         except AttributeError:
@@ -139,10 +140,10 @@ except:
 try:
     with root_engine.connect() as connection:
         connection.execute(
-            "CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format(
-                user, password, mysql_x509))
+            text("CREATE USER IF NOT EXISTS \'{0}\'@\'%%\' IDENTIFIED BY \'{1}\' {2}".format(
+                user, password, mysql_x509)))
         connection.execute(
-            "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user))
+            text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\'".format(database, user)))
         try:
             connection.commit()
         except AttributeError:

charts/barbican/charts/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl

@@ -29,15 +29,19 @@ RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \
   awk -F'[//:]' '{print $4}')
 RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \
   awk -F'[@]' '{print $1}' | \
-  awk -F'[//:]' '{print $5}')
+  awk -F'[//:]' '{print $5}' | \
+  sed 's/%/\\x/g' | \
+  xargs -0 printf "%b")
 
 # Extract User creadential
 RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \
   awk -F'[@]' '{print $1}' | \
   awk -F'[//:]' '{print $4}')
 RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \
   awk -F'[@]' '{print $1}' | \
-  awk -F'[//:]' '{print $5}')
+  awk -F'[//:]' '{print $5}' | \
+  sed 's/%/\\x/g' | \
+  xargs -0 printf "%b")
 
 # Extract User vHost
 RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \

charts/barbican/charts/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl

@@ -40,6 +40,9 @@ return: |
 release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }}
 application: {{ $application }}
 component: {{ $component }}
+app.kubernetes.io/name: {{ $application }}
+app.kubernetes.io/component: {{ $component }}
+app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }}
 {{- if ($envAll.Values.pod).labels }}
 {{- if hasKey $envAll.Values.pod.labels $component }}
 {{ index $envAll.Values.pod "labels" $component | toYaml }}