Support teams (#363)

* feat: add support for teams

* add skip_non_organization_members field

* change: refactor otterdog config class to support exclude filters for teams

* fix: twofa seed key for pass provider

* fix: timeout for web client

* fix: update README with pass key change

* fix: twofa seed key for plain provider

* fix: bitwarden provider

* chore: add more debugging output, reduce timeout to 10s

* chore(deps): update dependencies

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 
+- Added support for teams.
 - Use asyncer to speed up retrieval of live settings. ([#209](https://github.com/eclipse-csi/otterdog/issues/209))
 
 ### Changed

DEPENDENCIES

@@ -16,7 +16,7 @@ pypi/pypi/-/attrs/24.3.0
 pypi/pypi/-/blinker/1.9.0
 pypi/pypi/-/certifi/2024.12.14
 pypi/pypi/-/cffi/1.17.1
-pypi/pypi/-/charset-normalizer/3.4.0
+pypi/pypi/-/charset-normalizer/3.4.1
 pypi/pypi/-/chevron/0.14.0
 pypi/pypi/-/click/8.1.8
 pypi/pypi/-/colorama/0.4.6
@@ -78,7 +78,7 @@ pypi/pypi/-/semver/3.0.2
 pypi/pypi/-/six/1.17.0
 pypi/pypi/-/smmap/5.0.1
 pypi/pypi/-/sniffio/1.3.1
-pypi/pypi/-/starlette/0.42.0
+pypi/pypi/-/starlette/0.45.1
 pypi/pypi/-/text-unidecode/1.3
 pypi/pypi/-/typing-extensions/4.12.2
 pypi/pypi/-/url-normalize/1.4.3

README.md

@@ -102,7 +102,7 @@ Create a `otterdog.json` file with the following content (replace bracketed valu
         "api_token": "<GitHub PAT>",
         "username": "<Username>",
         "password": "<Password>",
-        "2fa_seed": "<2FA TOTP seed>"
+        "twofa_seed": "<2FA TOTP seed>"
       }
     }
   ]
@@ -199,7 +199,7 @@ required credential data:
         "api_token": "<path/to/api_token>",
         "username": "<path/to/username>",
         "password": "<path/to/password>",
-        "2fa_seed": "<path/to/2fa_seed>"
+        "twofa_seed": "<path/to/2fa_seed>"
       }
     }
   ]

docs/reference/organization/team.md

@@ -0,0 +1,42 @@
+Definition of an organization `Team`, the following properties are supported:
+
+| Key                             | Value        | Description                                                                          | Note                  |
+|---------------------------------|--------------|--------------------------------------------------------------------------------------|-----------------------|
+| _name_                          | string       | The name of the team                                                                 |                       |
+| _description_                   | string       | The description of the team                                                          |                       |
+| _privacy_                       | string       | The level of privacy this team should have                                           | `visible` or `secret` |
+| _notifications_                 | boolean      | Whether the team members receive notifications when the team is @mentioned           |                       |
+| _members_                       | list[string] | List of users that should be a member of the team                                    |                       |
+| _skip_members_                  | boolean      | If `true`, team members will be ignored                                              |                       |
+| _skip_non_organization_members_ | boolean      | If `true`, users which are not yet organization members can not be added to the team |                       |
+
+
+## Jsonnet Function
+
+``` jsonnet
+orgs.newTeam('<name>') {
+  <key>: <value>
+}
+```
+
+## Validation rules
+
+- setting `privacy` must be one of `visible` or `secret`, any other value triggers an error
+- specifying a non-empty list of `members` while `skip_members` is enabled, triggers an error
+- specifying a user in `members` that is not yet an organization member while `skip_non_organization_members` is enabled, triggers an error
+
+## Example usage
+
+=== "jsonnet"
+    ``` jsonnet
+    orgs.newOrg('OtterdogTest') {
+      ...
+      teams+: [
+        orgs.newTeam('committers') {
+          description: "The project committers",
+          privacy: "visible",
+        },
+      ],
+      ...
+    }
+    ```

mkdocs.yml

@@ -55,6 +55,7 @@ nav:
         - Organization Variable: reference/organization/variable.md
         - Organization Ruleset: reference/organization/ruleset.md
         - Custom Property: reference/organization/custom-property.md
+        - Team: reference/organization/team.md
         - Repository:
           - reference/organization/repository/index.md
           - Workflow Settings: reference/organization/repository/workflow-settings.md

otterdog/config.py

@@ -8,10 +8,12 @@
 
 from __future__ import annotations
 
+import dataclasses
 import json
 import os
 import re
 from abc import abstractmethod
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Protocol
 
 from otterdog.credentials import CredentialProvider
@@ -21,6 +23,8 @@
 from .utils import deep_merge_dict, query_json
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from otterdog.credentials import Credentials
 
 _logger = get_logger(__name__)
@@ -133,70 +137,112 @@ def is_supported_secret_provider(self, provider_type: str) -> bool: ...
     def get_secret(self, data: str) -> str: ...
 
 
-class OtterdogConfig(SecretResolver):
-    def __init__(self, config_file: str, local_mode: bool, working_dir: str | None = None):
-        if not os.path.exists(config_file):
-            raise RuntimeError(f"configuration file '{config_file}' not found")
-
-        self._config_file = os.path.realpath(config_file)
-        self._config_dir = os.path.dirname(self._config_file)
+class CredentialResolver(SecretResolver):
+    def __init__(self, config: OtterdogConfig) -> None:
+        self._config = config
         self._credential_providers: dict[str, CredentialProvider] = {}
 
-        self._local_mode = local_mode
+    def _get_credential_provider(self, provider_type: str) -> CredentialProvider | None:
+        provider = self._credential_providers.get(provider_type)
+        if provider is None:
+            provider = CredentialProvider.create(
+                provider_type, query_json(f"defaults.{provider_type}", self._config.configuration) or {}
+            )
+            if provider is not None:
+                self._credential_providers[provider_type] = provider
 
-        with open(config_file) as f:
-            self._configuration = json.load(f)
+        return provider
 
-        if working_dir is None:
-            override_defaults_file = os.path.join(self._config_dir, ".otterdog-defaults.json")
-            if os.path.exists(override_defaults_file):
-                with open(override_defaults_file) as defaults_file:
-                    defaults = json.load(defaults_file)
-                    _logger.trace("loading default overrides from '%s'", override_defaults_file)
-                    self._configuration["defaults"] = deep_merge_dict(
-                        defaults, self._configuration.setdefault("defaults")
-                    )
+    def get_credentials(self, org_config: OrganizationConfig, only_token: bool = False) -> Credentials:
+        provider_type = org_config.credential_data.get("provider")
 
-        self._jsonnet_config = query_json("defaults.jsonnet", self._configuration) or {}
-        self._github_config = query_json("defaults.github", self._configuration) or {}
-        self._default_credential_provider = query_json("defaults.credentials.provider", self._configuration) or ""
+        if provider_type is None:
+            provider_type = self._config.default_credential_provider
 
-        if working_dir is None:
-            self._jsonnet_base_dir = os.path.join(self._config_dir, self._jsonnet_config.get("config_dir", "orgs"))
+        if not provider_type:
+            raise RuntimeError(f"no credential provider configured for organization '{org_config.name}'")
+
+        provider = self._get_credential_provider(provider_type)
+        if provider is not None:
+            return provider.get_credentials(org_config.name, org_config.credential_data, only_token)
+        else:
+            raise RuntimeError(f"unsupported credential provider '{provider_type}'")
+
+    def is_supported_secret_provider(self, provider_type: str) -> bool:
+        # TODO: make this cleaner
+        return provider_type in ["pass", "bitwarden"]
+
+    def get_secret(self, secret_data: str) -> str:
+        if secret_data and ":" in secret_data:
+            provider_type, data = re.split(":", secret_data)
+            provider = self._get_credential_provider(provider_type)
+            if provider is not None:
+                return provider.get_secret(data)
+            else:
+                return secret_data
         else:
-            self._jsonnet_base_dir = os.path.join(working_dir, self._jsonnet_config.get("config_dir", "orgs"))
-            if not os.path.exists(self._jsonnet_base_dir):
-                os.makedirs(self._jsonnet_base_dir)
+            return secret_data
+
+
+@dataclasses.dataclass(frozen=True)
+class OtterdogConfig:
+    configuration: Mapping[str, Any]
+    local_mode: bool
+    working_dir: str
+
+    _jsonnet_config: Mapping[str, Any] = dataclasses.field(init=False)
+    _github_config: Mapping[str, Any] = dataclasses.field(init=False)
+    _default_credential_provider: str = dataclasses.field(init=False)
+    _jsonnet_base_dir: str = dataclasses.field(init=False)
 
-        organizations = self._configuration.get("organizations", [])
+    _organizations_map: dict[str, OrganizationConfig] = dataclasses.field(init=False, default_factory=dict)
+    _organizations: list[OrganizationConfig] = dataclasses.field(init=False, default_factory=list)
 
-        self._organizations_map = {}
-        self._organizations = []
+    def __post_init__(self):
+        object.__setattr__(self, "_jsonnet_config", query_json("defaults.jsonnet", self.configuration) or {})
+        object.__setattr__(self, "_github_config", query_json("defaults.github", self.configuration) or {})
+        object.__setattr__(
+            self, "_default_credential_provider", query_json("defaults.credentials.provider", self.configuration) or ""
+        )
+
+        object.__setattr__(
+            self,
+            "_jsonnet_base_dir",
+            os.path.join(self.working_dir, self._jsonnet_config.get("config_dir", "orgs")),
+        )
+        if not os.path.exists(self._jsonnet_base_dir):
+            os.makedirs(self._jsonnet_base_dir)
+
+        organizations = self.configuration.get("organizations", [])
         for org in organizations:
             org_config = OrganizationConfig.from_dict(org, self)
             self._organizations.append(org_config)
             self._organizations_map[org_config.name.lower()] = org_config
             self._organizations_map[org_config.github_id.lower()] = org_config
 
     @property
-    def config_file(self) -> str:
-        return self._config_file
+    def jsonnet_base_dir(self) -> str:
+        return self._jsonnet_base_dir
 
     @property
-    def config_dir(self) -> str:
-        return self._config_dir
+    def default_config_repo(self) -> str:
+        return self._github_config.get("config_repo", ".otterdog")
 
     @property
-    def jsonnet_base_dir(self) -> str:
-        return self._jsonnet_base_dir
+    def default_team_exclusions(self) -> list[str]:
+        return self._github_config.get("exclude_teams", [])
 
-    @property
-    def local_mode(self) -> bool:
-        return self._local_mode
+    @cached_property
+    def exclude_teams_pattern(self) -> re.Pattern | None:
+        team_exclusions = self.default_team_exclusions
+        if len(team_exclusions) == 0:
+            return None
+        else:
+            return re.compile("|".join(team_exclusions))
 
     @property
-    def default_config_repo(self) -> str:
-        return self._github_config.get("config_repo", ".otterdog")
+    def default_credential_provider(self) -> str:
+        return self._default_credential_provider
 
     @property
     def default_base_template(self) -> str:
@@ -231,50 +277,27 @@ def get_organization_config(self, project_or_organization_name: str) -> Organiza
             raise RuntimeError(f"unknown organization with name / github_id '{project_or_organization_name}'")
         return org_config
 
-    def _get_credential_provider(self, provider_type: str) -> CredentialProvider | None:
-        provider = self._credential_providers.get(provider_type)
-        if provider is None:
-            provider = CredentialProvider.create(
-                provider_type, query_json(f"defaults.{provider_type}", self._configuration) or {}
-            )
-            if provider is not None:
-                self._credential_providers[provider_type] = provider
-
-        return provider
-
-    def get_credentials(self, org_config: OrganizationConfig, only_token: bool = False) -> Credentials:
-        provider_type = org_config.credential_data.get("provider")
-
-        if provider_type is None:
-            provider_type = self._default_credential_provider
-
-        if not provider_type:
-            raise RuntimeError(f"no credential provider configured for organization '{org_config.name}'")
+    @classmethod
+    def from_file(cls, config_file: str, local_mode: bool, working_dir: str | None = None) -> OtterdogConfig:
+        if not os.path.exists(config_file):
+            raise RuntimeError(f"configuration file '{config_file}' not found")
 
-        provider = self._get_credential_provider(provider_type)
-        if provider is not None:
-            return provider.get_credentials(org_config.name, org_config.credential_data, only_token)
-        else:
-            raise RuntimeError(f"unsupported credential provider '{provider_type}'")
+        config_file_file = os.path.realpath(config_file)
+        config_file_dir = os.path.dirname(config_file)
 
-    def is_supported_secret_provider(self, provider_type: str) -> bool:
-        # TODO: make this cleaner
-        return provider_type in ["pass", "bitwarden"]
+        with open(config_file_file) as f:
+            configuration = json.load(f)
 
-    def get_secret(self, secret_data: str) -> str:
-        if secret_data and ":" in secret_data:
-            provider_type, data = re.split(":", secret_data)
-            provider = self._get_credential_provider(provider_type)
-            if provider is not None:
-                return provider.get_secret(data)
-            else:
-                return secret_data
-        else:
-            return secret_data
+        if working_dir is None:
+            override_defaults_file = os.path.join(config_file_dir, ".otterdog-defaults.json")
+            if os.path.exists(override_defaults_file):
+                with open(override_defaults_file) as defaults_file:
+                    defaults = json.load(defaults_file)
+                    _logger.trace("loading default overrides from '%s'", override_defaults_file)
+                    configuration["defaults"] = deep_merge_dict(defaults, configuration.setdefault("defaults", {}))
 
-    def __repr__(self):
-        return f"OtterdogConfig('{self.config_file}')"
+        return cls(configuration, local_mode, working_dir if working_dir is not None else config_file_dir)
 
     @classmethod
-    def from_file(cls, config_file: str, local_mode: bool):
-        return cls(config_file, local_mode)
+    def from_dict(cls, configuration: Mapping[str, Any], local_mode: bool, working_dir: str) -> OtterdogConfig:
+        return cls(configuration, local_mode, working_dir)

otterdog/credentials/bitwarden_provider.py

@@ -26,8 +26,10 @@ class BitwardenVault(CredentialProvider):
     A class to provide convenient access to a bitwarden vault.
     """
 
-    def __init__(self, api_token_key: str | None):
-        self._api_token_key = api_token_key or "api_token_admin"
+    _KEY_API_TOKEN = "api_token_admin"
+
+    def __init__(self, api_token_key: str = _KEY_API_TOKEN):
+        self._api_token_key = api_token_key
 
         _logger.debug("unlocking bitwarden vault")
         self._status, output = subprocess.getstatusoutput("bw unlock --check")  # noqa: S605, S607

otterdog/credentials/pass_provider.py

@@ -28,7 +28,7 @@ class PassVault(CredentialProvider):
     KEY_API_TOKEN = "api_token"
     KEY_USERNAME = "username"
     KEY_PASSWORD = "password"
-    KEY_TWOFA_SEED = "2fa_seed"
+    KEY_TWOFA_SEED = "twofa_seed"
 
     def __init__(
         self,

otterdog/credentials/plain_provider.py

@@ -29,7 +29,7 @@ class PlainVault(CredentialProvider):
     KEY_API_TOKEN = "api_token"
     KEY_USERNAME = "username"
     KEY_PASSWORD = "password"
-    KEY_TWOFA_SEED = "2fa_seed"
+    KEY_TWOFA_SEED = "twofa_seed"
 
     def get_credentials(self, org_name: str, data: dict[str, Any], only_token: bool = False) -> Credentials:
         github_token = self._retrieve_key(self.KEY_API_TOKEN, data)

otterdog/jsonnet.py

@@ -28,6 +28,7 @@ class JsonnetConfig:
 
     create_org = "newOrg"
     create_org_role = "newOrgRole"
+    create_org_team = "newTeam"
     create_org_custom_property = "newCustomProperty"
     create_org_webhook = "newOrgWebhook"
     create_org_secret = "newOrgSecret"
@@ -126,6 +127,16 @@ def default_org_role_config(self):
             _logger.debug("no default org role config found, roles will be skipped")
             return None
 
+    @cached_property
+    def default_team_config(self):
+        try:
+            # load the default team config
+            team_snippet = f"(import '{self.template_file}').{self.create_org_team}('default')"
+            return jsonnet_evaluate_snippet(team_snippet)
+        except RuntimeError:
+            _logger.debug("no default team config found, teams will be skipped")
+            return None
+
     @cached_property
     def default_org_custom_property_config(self):
         try: