Changes to support Pydantic v2 (#3)

* Changes to support Pydantic v2

* Remove use of deprecated parse_obj

* Use most recent kube-custom-resource patch

* Deal better with long versions (i.e. branch names)

* Pick up kube-custom-resource fixes

* Bump kube-custom-resource to pick up fixes

* Changes to libraries merged and tagged + tests added

* We don't need to run a full test suite

.github/workflows/build-push-artifacts.yaml

@@ -1,6 +1,20 @@
 name: Publish artifacts
-# Run the tasks on every push
-on: push
+
+on:
+  # Publish artifacts on every push to main and every tag
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+  # Also allow publication to be done via a workflow call
+  # In this case, the chart version is returned as an output
+  workflow_call:
+    outputs:
+      chart-version:
+        description: The chart version that was published
+        value: ${{ jobs.build_push_chart.outputs.chart-version }}
+
 jobs:
   build_push_images:
     name: Build and push images
@@ -42,6 +56,8 @@ jobs:
     runs-on: ubuntu-latest
     # Only build and push the chart if the images built successfully
     needs: [build_push_images]
+    outputs:
+      chart-version: ${{ steps.semver.outputs.version }}
     steps:
       - name: Check out the repository
         uses: actions/checkout@v3

.github/workflows/test-pr.yaml

@@ -0,0 +1,61 @@
+name: Test Azimuth deployment
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - ready_for_review
+      - reopened
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  # This job exists so that PRs from outside the main repo are rejected
+  fail_on_remote:
+    runs-on: ubuntu-latest
+    steps:
+      - name: PR must be from a branch in the stackhpc/azimuth-identity-operator repo
+        run: exit ${{ github.repository == 'stackhpc/azimuth-identity-operator' && '0' || '1' }}
+
+  publish_artifacts:
+    needs: [fail_on_remote]
+    uses: ./.github/workflows/build-push-artifacts.yaml
+
+  run_azimuth_tests:
+    needs: [publish_artifacts]
+    runs-on: ubuntu-latest
+    steps:
+      # Check out the configuration repository
+      - name: Set up Azimuth environment
+        uses: stackhpc/azimuth-config/.github/actions/setup@main
+        with:
+          os-clouds: ${{ secrets.OS_CLOUDS }}
+          environment-prefix: identity-ci
+          # Use the version of the chart that we just built
+          # We also don't need all the tests
+          # The workstation is sufficient to test that the OIDC discovery is working
+          extra-vars: |
+            azimuth_identity_operator_chart_version: ${{ needs.publish_artifacts.outputs.chart-version }}
+            generate_tests_caas_test_case_slurm_enabled: false
+            generate_tests_caas_test_case_repo2docker_enabled: false
+            generate_tests_caas_test_case_rstudio_enabled: false
+            generate_tests_kubernetes_suite_enabled: false
+            generate_tests_kubernetes_apps_suite_enabled: false
+
+      # Provision Azimuth using the azimuth-ops version under test
+      - name: Provision Azimuth
+        uses: stackhpc/azimuth-config/.github/actions/provision@main
+
+      # # Run the tests
+      - name: Run Azimuth tests
+        uses: stackhpc/azimuth-config/.github/actions/test@main
+
+      # Tear down the environment
+      - name: Destroy Azimuth
+        uses: stackhpc/azimuth-config/.github/actions/destroy@main
+        if: ${{ always() }}

azimuth_identity/config.py

@@ -1,10 +1,18 @@
 import typing as t
 
-from pydantic import Field, AnyHttpUrl, FilePath, conint, constr, root_validator, validator
+from pydantic import TypeAdapter, Field, AnyHttpUrl as PyAnyHttpUrl, conint, constr
+from pydantic.functional_validators import AfterValidator
 
 from configomatic import Configuration as BaseConfiguration, Section, LoggingConfiguration
 
 
+#: Type for a string that validates as a URL
+AnyHttpUrl = t.Annotated[
+    str,
+    AfterValidator(lambda v: str(TypeAdapter(PyAnyHttpUrl).validate_python(v)))
+]
+
+
 class SecretRef(Section):
     """
     A reference to a secret.
@@ -56,12 +64,19 @@ class DexConfig(Section):
     keycloak_client_secret_bytes: conint(gt = 0) = 64
 
 
+def strip_trailing_slash(v: str) -> str:
+    """
+    Strips trailing slashes from the given string.
+    """
+    return v.rstrip("/")
+
+
 class KeycloakConfig(Section):
     """
     Configuration for the target Keycloak instance.
     """
     #: The base URL of the Keycloak instance
-    base_url: AnyHttpUrl
+    base_url: t.Annotated[AnyHttpUrl, AfterValidator(strip_trailing_slash)]
 
     #: The client ID to use when authenticating with Keycloak
     client_id: constr(min_length = 1)
@@ -102,13 +117,6 @@ class KeycloakConfig(Section):
         default_factory = lambda: { "realm-management": ["realm-admin"] }
     )
 
-    @validator("base_url")
-    def validate_base_url(cls, v):
-        """
-        Strips trailing slashes from the base URL if present.
-        """
-        return v.rstrip("/")
-
 
 class HelmClientConfiguration(Section):
     """
@@ -129,15 +137,15 @@ class HelmClientConfiguration(Section):
     unpack_directory: t.Optional[str] = None
 
 
-class Configuration(BaseConfiguration):
+class Configuration(
+    BaseConfiguration,
+    default_path = "/etc/azimuth/identity-operator.yaml",
+    path_env_var = "AZIMUTH_IDENTITY_CONFIG",
+    env_prefix = "AZIMUTH_IDENTITY"
+):
     """
     Top-level configuration model.
     """
-    class Config:
-        default_path = "/etc/azimuth/identity-operator.yaml"
-        path_env_var = "AZIMUTH_IDENTITY_CONFIG"
-        env_prefix = "AZIMUTH_IDENTITY"
-
     #: The logging configuration
     logging: LoggingConfiguration = Field(default_factory = LoggingConfiguration)
 

azimuth_identity/dex.py

@@ -40,7 +40,7 @@ async def ensure_tls_secret(ekclient, realm: api.Realm):
             },
         },
     }
-    kopf.adopt(secret_data, realm.dict())
+    kopf.adopt(secret_data, realm.model_dump())
     eksecrets = await ekclient.api("v1").resource("secrets")
     _ = await eksecrets.create_or_patch(
         secret_name,
@@ -134,7 +134,7 @@ async def ensure_config_secret(
                 "config.yaml": yaml.safe_dump(next_config),
             },
         }
-        kopf.adopt(secret_data, realm.dict())
+        kopf.adopt(secret_data, realm.model_dump())
         _ = await eksecrets.create_or_patch(
             secret_name,
             secret_data,
@@ -209,7 +209,7 @@ async def ensure_ingresses(
             ],
         },
     }
-    kopf.adopt(ingress_data, realm.dict())
+    kopf.adopt(ingress_data, realm.model_dump())
     _ = await ekclient.apply_object(ingress_data, force = True)
     auth_annotations = {
         "nginx.ingress.kubernetes.io/auth-url": settings.dex.ingress_auth_url,
@@ -282,7 +282,7 @@ async def ensure_ingresses(
             ],
         },
     }
-    kopf.adopt(ingress_data, realm.dict())
+    kopf.adopt(ingress_data, realm.model_dump())
     _ = await ekclient.apply_object(ingress_data, force = True)
 
 

azimuth_identity/models/v1alpha1/platform.py

@@ -1,6 +1,6 @@
 import typing as t
 
-from pydantic import Extra, Field, constr
+from pydantic import Field
 
 from kube_custom_resource import CustomResource, schema
 
@@ -9,11 +9,11 @@ class ZenithServiceSpec(schema.BaseModel):
     """
     The spec for a Zenith service.
     """
-    subdomain: constr(regex = r"[a-z0-9]+") = Field(
+    subdomain: schema.constr(pattern = r"[a-z0-9]+") = Field(
         ...,
         description = "The subdomain of the Zenith service."
     )
-    fqdn: constr(regex = r"[a-z0-9\.-]+") = Field(
+    fqdn: schema.constr(pattern = r"[a-z0-9\.-]+") = Field(
         ...,
         description = "The FQDN of the Zenith service."
     )
@@ -23,7 +23,7 @@ class PlatformSpec(schema.BaseModel):
     """
     The spec for an Azimuth identity platform.
     """
-    realm_name: t.Optional[constr(regex = r"[a-z0-9-]+")] = Field(
+    realm_name: schema.constr(pattern = r"[a-z0-9-]+") = Field(
         ...,
         description = "The name of the realm that the platform belongs to."
     )
@@ -47,13 +47,10 @@ class PlatformPhase(str, schema.Enum):
     FAILED   = "Failed"
 
 
-class PlatformStatus(schema.BaseModel):
+class PlatformStatus(schema.BaseModel, extra = "allow"):
     """
     The status of an Azimuth identity platform.
     """
-    class Config:
-        extra = Extra.allow
-
     phase: PlatformPhase = Field(
         PlatformPhase.UNKNOWN.value,
         description = "The phase of the platform."

azimuth_identity/models/v1alpha1/realm.py

@@ -1,6 +1,4 @@
-import typing as t
-
-from pydantic import Extra, Field, AnyHttpUrl, constr
+from pydantic import Field
 
 from kube_custom_resource import CustomResource, schema
 
@@ -9,7 +7,7 @@ class RealmSpec(schema.BaseModel):
     """
     The spec for an Azimuth identity realm.
     """
-    tenancy_id: constr(min_length = 1) = Field(
+    tenancy_id: schema.constr(min_length = 1) = Field(
         ...,
         description = "The ID of the Azimuth tenancy that the realm is for."
     )
@@ -26,22 +24,19 @@ class RealmPhase(str, schema.Enum):
     FAILED   = "Failed"
 
 
-class RealmStatus(schema.BaseModel):
+class RealmStatus(schema.BaseModel, extra = "allow"):
     """
     The status of an Azimuth identity realm.
     """
-    class Config:
-        extra = Extra.allow
-
     phase: RealmPhase = Field(
         RealmPhase.UNKNOWN.value,
         description = "The phase of the realm."
     )
-    oidc_issuer_url: t.Optional[AnyHttpUrl] = Field(
+    oidc_issuer_url: schema.Optional[schema.AnyHttpUrl] = Field(
         None,
         description = "The OIDC issuer URL for the realm."
     )
-    admin_url: t.Optional[AnyHttpUrl] = Field(
+    admin_url: schema.Optional[schema.AnyHttpUrl] = Field(
         None,
         description = "The admin URL for the realm."
     )

azimuth_identity/operator.py

@@ -80,7 +80,7 @@ async def save_instance_status(instance):
         {
             # Include the resource version for optimistic concurrency
             "metadata": { "resourceVersion": instance.metadata.resource_version },
-            "status": instance.status.dict(exclude_defaults = True),
+            "status": instance.status.model_dump(exclude_defaults = True),
         },
         namespace = instance.metadata.namespace
     )
@@ -97,7 +97,7 @@ def decorator(func):
         @functools.wraps(func)
         async def handler(**handler_kwargs):
             if "instance" not in handler_kwargs:
-                handler_kwargs["instance"] = model.parse_obj(handler_kwargs["body"])
+                handler_kwargs["instance"] = model.model_validate(handler_kwargs["body"])
             try:
                 return await func(**handler_kwargs)
             except ApiError as exc:
@@ -185,7 +185,7 @@ async def reconcile_platform(instance: api.Platform, param, **kwargs):
             )
         else:
             raise
-    realm: api.Realm = api.Realm.parse_obj(realm)
+    realm: api.Realm = api.Realm.model_validate(realm)
     if realm.status.phase != api.RealmPhase.READY:
         raise kopf.TemporaryError(
             f"Realm '{instance.spec.realm_name}' is not yet ready",
@@ -302,7 +302,7 @@ async def delete_platform(instance: api.Platform, **kwargs):
             return
         else:
             raise
-    realm: api.Realm = api.Realm.parse_obj(realm)
+    realm: api.Realm = api.Realm.model_validate(realm)
     realm_name = keycloak.realm_name(realm)
     # Remove the clients for all the services
     await keycloak.prune_platform_service_clients(realm_name, instance, all = True)

chart/templates/_helpers.tpl

@@ -30,7 +30,15 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 Labels for a chart-level resource.
 */}}
 {{- define "azimuth-identity-operator.labels" -}}
-helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | lower | trunc 63 | trimSuffix "-" }}
+helm.sh/chart: {{
+  printf "%s-%s" .Chart.Name .Chart.Version |
+    replace "+" "_" |
+    lower |
+    trunc 63 |
+    trimSuffix "-" |
+    trimSuffix "." |
+    trimSuffix "_"
+}}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}

requirements.txt

@@ -1,29 +1,29 @@
-aiohttp==3.8.5
+aiohttp==3.8.6
 aiosignal==1.3.1
-annotated-types==0.5.0
-anyio==3.7.1
-async-timeout==4.0.2
+annotated-types==0.6.0
+anyio==4.0.0
+async-timeout==4.0.3
 attrs==23.1.0
 certifi==2023.7.22
-charset-normalizer==3.2.0
-click==8.1.6
-configomatic @ git+https://github.com/stackhpc/configomatic.git@a53458c00bae1d94ba2fcb6cf14c530de44aa297
-easykube @ git+https://github.com/stackhpc/easykube.git@594e65190e6f13d66f069feaece534f7595c1656
-exceptiongroup==1.1.2
+charset-normalizer==3.3.2
+click==8.1.7
+configomatic==0.2.0
+easykube==0.1.1
+exceptiongroup==1.1.3
 frozenlist==1.4.0
 h11==0.14.0
-httpcore==0.17.3
-httpx==0.24.1
+httpcore==1.0.1
+httpx==0.25.1
 idna==3.4
-iso8601==2.0.0
+iso8601==2.1.0
 kopf==1.36.2
-kube-custom-resource @ git+https://github.com/stackhpc/kube-custom-resource.git@106a72837395ba871c6fcb13992a38478c50ae7a
+kube-custom-resource==0.2.0
 multidict==6.0.4
-pydantic==1.10.12
-pydantic_core==2.4.0
-pyhelm3 @ git+https://github.com/stackhpc/pyhelm3.git@cacf99d706851b67a57249726e94adedf03c6451
+pydantic==2.4.2
+pydantic_core==2.10.1
+pyhelm3==0.2.0
 python-json-logger==2.0.7
 PyYAML==6.0.1
 sniffio==1.3.0
-typing_extensions==4.7.1
+typing_extensions==4.8.0
 yarl==1.9.2