feat: add preliminiary github app version (#192)

* Use quart as framework

* Add tasks

* Use asyncio in provider

* Use more async where needed

* Get rid of eclipse_project property

* Adjust docker config

docker/Dockerfile

@@ -0,0 +1,54 @@
+# build jsonnet-bundler using a go environment
+FROM docker.io/library/golang:1.18 AS builder-go
+RUN go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
+
+# build otterdog using a python environment
+FROM docker.io/library/python:3.10.10-slim as builder-python3
+
+RUN apt-get update \
+    && apt-get install -y \
+        golang
+
+WORKDIR /app
+
+ENV PIP_DEFAULT_TIMEOUT=100 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=1 \
+    POETRY_VERSION=1.7.1
+
+COPY ../otterdog ./otterdog
+COPY ../pyproject.toml ../poetry.lock ../README.md ./hypercorn-cfg.toml ./
+COPY ./entrypoint.sh ./docker/entrypoint.sh
+COPY ./start-webapp ./docker/start-webapp
+
+RUN pip install "poetry==$POETRY_VERSION"
+
+RUN poetry config virtualenvs.in-project true && \
+    poetry install --only=main,app --no-root && \
+    poetry build && \
+    poetry install --only-root
+
+# create the final image having python3.10 as base
+FROM python:3.10.10-slim
+
+RUN apt-get update \
+    && apt-get install -y \
+        git
+
+COPY --from=builder-go /go/bin/jb /usr/bin/jb
+COPY --from=builder-python3 /app/.venv /app/.venv
+COPY --from=builder-python3 /app/otterdog /app/otterdog
+COPY --from=builder-python3 /app/docker/entrypoint.sh /app/entrypoint.sh
+COPY --from=builder-python3 /app/docker/start-webapp /app/start-webapp
+COPY --from=builder-python3 /app/hypercorn-cfg.toml /app/hypercorn-cfg.toml
+
+RUN chmod +x /app/entrypoint.sh
+RUN chmod +x /app/start-webapp
+
+WORKDIR /app
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+ENTRYPOINT ["/app/entrypoint.sh"]

docker/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# if any of the commands in your code fails for any reason, the entire script fails
+set -o errexit
+# fail exit if one of your pipe command fails
+set -o pipefail
+# exits if any of your variables is not set
+set -o nounset
+
+exec "$@"

docker/hypercorn-cfg.toml

@@ -0,0 +1,5 @@
+bind = "0.0.0.0:5000"
+workers = 1
+accesslog = '-'
+loglevel = 'info'
+h11_max_incomplete_size = 4

docker/start-webapp

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#
+# *******************************************************************************
+# Copyright (c) 2024 Eclipse Foundation and others.
+# This program and the accompanying materials are made available
+# under the terms of the MIT License
+# which is available at https://spdx.org/licenses/MIT.html
+# SPDX-License-Identifier: MIT
+# *******************************************************************************
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+.venv/bin/hypercorn --config hypercorn-cfg.toml otterdog.app

otterdog/app.py

@@ -6,9 +6,43 @@
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
 
+import os
+from sys import exit
+
+from decouple import config  # type: ignore
+
+from otterdog.webapp import create_app
+from otterdog.webapp.config import config_dict
+
+# WARNING: Don't run with debug turned on in production!
+DEBUG: bool = config("DEBUG", default=True, cast=bool)
+
+# Determine which configuration to use
+config_mode = "Debug" if DEBUG else "Production"
+
+try:
+    app_config = config_dict[config_mode]
+except KeyError:
+    exit("Error: Invalid <config_mode>. Expected values [Debug, Production] ")
+
+app = create_app(app_config)  # type: ignore
+
+if os.path.exists(app_config.APP_ROOT):
+    os.chdir(app_config.APP_ROOT)
+else:
+    app.logger.error(f"APP_ROOT '{app_config.APP_ROOT}' does not exist, exiting.")
+    exit(1)
+
+if DEBUG:
+    app.logger.info("DEBUG           = " + str(DEBUG))
+    app.logger.info("Environment     = " + config_mode)
+    app.logger.info("QUART_APP       = " + app_config.QUART_APP)
+    app.logger.info("APP_ROOT        = " + app_config.APP_ROOT)
+    app.logger.info("OTTERDOG_CONFIG = " + app_config.OTTERDOG_CONFIG)
+
 
 def run():
-    print("Hello World!")
+    app.run(debug=True)
 
 
 if __name__ == "__main__":

otterdog/cli.py

@@ -6,6 +6,7 @@
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
 
+import asyncio
 import importlib.metadata
 import sys
 import traceback
@@ -521,7 +522,7 @@ def _execute_operation(organizations: list[str], operation: Operation):
 
         for organization in organizations:
             org_config = config.get_organization_config(organization)
-            exit_code = max(exit_code, operation.execute(org_config))
+            exit_code = max(exit_code, asyncio.run(operation.execute(org_config)))
 
         operation.post_execute()
         sys.exit(exit_code)

otterdog/config.py

@@ -16,7 +16,10 @@
 import jq  # type: ignore
 
 from . import credentials
-from .credentials import CredentialProvider, bitwarden_provider, pass_provider
+from .credentials import CredentialProvider
+from .credentials.bitwarden_provider import BitwardenVault
+from .credentials.inmemory_provider import InmemoryVault
+from .credentials.pass_provider import PassVault
 from .jsonnet import JsonnetConfig
 
 
@@ -25,14 +28,12 @@ def __init__(
         self,
         name: str,
         github_id: str,
-        eclipse_project: Optional[str],
         config_repo: str,
         jsonnet_config: JsonnetConfig,
         credential_data: dict[str, Any],
     ):
         self._name = name
         self._github_id = github_id
-        self._eclipse_project = eclipse_project
         self._config_repo = config_repo
         self._jsonnet_config = jsonnet_config
         self._credential_data = credential_data
@@ -45,10 +46,6 @@ def name(self):
     def github_id(self) -> str:
         return self._github_id
 
-    @property
-    def eclipse_project(self) -> Optional[str]:
-        return self._eclipse_project
-
     @property
     def config_repo(self) -> str:
         return self._config_repo
@@ -61,9 +58,13 @@ def jsonnet_config(self) -> JsonnetConfig:
     def credential_data(self) -> dict[str, Any]:
         return self._credential_data
 
+    @credential_data.setter
+    def credential_data(self, data: dict[str, Any]) -> None:
+        self._credential_data = data
+
     def __repr__(self) -> str:
         return (
-            f"OrganizationConfig('{self.name}', '{self.github_id}', '{self.eclipse_project}', "
+            f"OrganizationConfig('{self.name}', '{self.github_id}', "
             f"'{self.config_repo}', {json.dumps(self.credential_data)})"
         )
 
@@ -77,8 +78,6 @@ def from_dict(cls, data: dict[str, Any], otterdog_config: OtterdogConfig) -> Org
         if github_id is None:
             raise RuntimeError(f"missing required github_id for organization config with name '{name}'")
 
-        eclipse_project = data.get("eclipse_project")
-
         config_repo = data.get("config_repo", otterdog_config.default_config_repo)
         base_template = data.get("base_template", otterdog_config.default_base_template)
 
@@ -93,11 +92,27 @@ def from_dict(cls, data: dict[str, Any], otterdog_config: OtterdogConfig) -> Org
         if data is None:
             raise RuntimeError(f"missing required credentials for organization config with name '{name}'")
 
-        return cls(name, github_id, eclipse_project, config_repo, jsonnet_config, data)
+        return cls(name, github_id, config_repo, jsonnet_config, data)
+
+    @classmethod
+    def of(
+        cls, github_id: str, credential_data: dict[str, Any], work_dir: str, otterdog_config: OtterdogConfig
+    ) -> OrganizationConfig:
+        config_repo = otterdog_config.default_config_repo
+        base_dir = os.path.join(otterdog_config.jsonnet_base_dir, work_dir)
+
+        jsonnet_config = JsonnetConfig(
+            github_id,
+            base_dir,
+            otterdog_config.default_base_template,
+            otterdog_config.local_mode,
+        )
+
+        return cls(github_id, github_id, config_repo, jsonnet_config, credential_data)
 
 
 class OtterdogConfig:
-    def __init__(self, config_file: str, local_mode: bool):
+    def __init__(self, config_file: str, local_mode: bool, working_dir: Optional[str] = None):
         if not os.path.exists(config_file):
             raise RuntimeError(f"configuration file '{config_file}' not found")
 
@@ -112,15 +127,24 @@ def __init__(self, config_file: str, local_mode: bool):
 
         self._jsonnet_config = jq.compile(".defaults.jsonnet // {}").input(self._configuration).first()
         self._github_config = jq.compile(".defaults.github // {}").input(self._configuration).first()
+        self._default_credential_provider = (
+            jq.compile('.defaults.credentials.provider // ""').input(self._configuration).first()
+        )
 
-        self._jsonnet_base_dir = os.path.join(self._config_dir, self._jsonnet_config.get("config_dir", "orgs"))
+        if working_dir is None:
+            self._jsonnet_base_dir = os.path.join(self._config_dir, self._jsonnet_config.get("config_dir", "orgs"))
+        else:
+            self._jsonnet_base_dir = os.path.join(working_dir, self._jsonnet_config.get("config_dir", "orgs"))
+            if not os.path.exists(self._jsonnet_base_dir):
+                os.makedirs(self._jsonnet_base_dir)
 
         organizations = self._configuration.get("organizations", [])
 
         self._organizations = {}
         for org in organizations:
             org_config = OrganizationConfig.from_dict(org, self)
             self._organizations[org_config.name] = org_config
+            self._organizations[org_config.github_id] = org_config
 
     @property
     def config_file(self) -> str:
@@ -149,7 +173,7 @@ def organization_configs(self) -> dict[str, OrganizationConfig]:
     def get_organization_config(self, organization_name: str) -> OrganizationConfig:
         org_config = self._organizations.get(organization_name)
         if org_config is None:
-            raise RuntimeError(f"unknown organization with name '{organization_name}'")
+            raise RuntimeError(f"unknown organization with name / github_id '{organization_name}'")
         return org_config
 
     def _get_credential_provider(self, provider_type: str) -> credentials.CredentialProvider:
@@ -163,15 +187,37 @@ def _get_credential_provider(self, provider_type: str) -> credentials.Credential
                         .first()
                     )
 
-                    provider = bitwarden_provider.BitwardenVault(api_token_key)
+                    provider = BitwardenVault(api_token_key)
                     self._credential_providers[provider_type] = provider
 
                 case "pass":
                     password_store_dir = (
                         jq.compile('.defaults.pass.password_store_dir // ""').input(self._configuration).first()
                     )
 
-                    provider = pass_provider.PassVault(password_store_dir)
+                    username_pattern = (
+                        jq.compile('.defaults.pass.username_pattern // ""').input(self._configuration).first()
+                    )
+
+                    password_pattern = (
+                        jq.compile('.defaults.pass.password_pattern // ""').input(self._configuration).first()
+                    )
+
+                    twofa_seed_pattern = (
+                        jq.compile('.defaults.pass.twofa_seed_pattern // ""').input(self._configuration).first()
+                    )
+
+                    api_token_pattern = (
+                        jq.compile('.defaults.pass.username_pattern // ""').input(self._configuration).first()
+                    )
+
+                    provider = PassVault(
+                        password_store_dir, username_pattern, password_pattern, twofa_seed_pattern, api_token_pattern
+                    )
+                    self._credential_providers[provider_type] = provider
+
+                case "inmemory":
+                    provider = InmemoryVault()
                     self._credential_providers[provider_type] = provider
 
                 case _:
@@ -181,11 +227,15 @@ def _get_credential_provider(self, provider_type: str) -> credentials.Credential
 
     def get_credentials(self, org_config: OrganizationConfig, only_token: bool = False) -> credentials.Credentials:
         provider_type = org_config.credential_data.get("provider")
+
         if provider_type is None:
+            provider_type = self._default_credential_provider
+
+        if not provider_type:
             raise RuntimeError(f"no credential provider configured for organization '{org_config.name}'")
 
         provider = self._get_credential_provider(provider_type)
-        return provider.get_credentials(org_config.eclipse_project, org_config.credential_data, only_token)
+        return provider.get_credentials(org_config.name, org_config.credential_data, only_token)
 
     def get_secret(self, secret_data: str) -> str:
         if secret_data and ":" in secret_data:

otterdog/credentials/__init__.py

@@ -9,7 +9,7 @@
 import dataclasses
 import time
 from abc import abstractmethod
-from typing import Optional, Protocol
+from typing import Any, Optional, Protocol
 
 import mintotp  # type: ignore
 
@@ -72,9 +72,7 @@ def __str__(self) -> str:
 
 class CredentialProvider(Protocol):
     @abstractmethod
-    def get_credentials(
-        self, eclipse_project: Optional[str], data: dict[str, str], only_token: bool = False
-    ) -> Credentials:
+    def get_credentials(self, org_name: str, data: dict[str, Any], only_token: bool = False) -> Credentials:
         ...
 
     @abstractmethod

otterdog/credentials/bitwarden_provider.py

@@ -9,7 +9,7 @@
 import json
 import re
 import subprocess
-from typing import Optional
+from typing import Any
 
 from otterdog import utils
 from otterdog.credentials import CredentialProvider, Credentials
@@ -34,9 +34,7 @@ def __init__(self, api_token_key: str):
     def is_unlocked(self) -> bool:
         return self._status == 0
 
-    def get_credentials(
-        self, eclipse_project: Optional[str], data: dict[str, str], only_token: bool = False
-    ) -> Credentials:
+    def get_credentials(self, org_name: str, data: dict[str, Any], only_token: bool = False) -> Credentials:
         assert self.is_unlocked()
 
         item_id = data.get("item_id")