WIP Adding a toggleable additional Gitea external mirror

data_safe_haven/config/config_sections.py

@@ -54,6 +54,7 @@ class ConfigSectionSRE(BaseModel, validate_assignment=True):
     # Mutable objects can be used as default arguments in Pydantic:
     # https://docs.pydantic.dev/latest/concepts/models/#fields-with-non-hashable-default-values
     admin_email_address: EmailAddress
+    external_git_mirror: bool = False
     admin_ip_addresses: list[IpAddress] = []
     databases: UniqueList[DatabaseSystem] = []
     data_provider_ip_addresses: list[IpAddress] = []

data_safe_haven/config/sre_config.py

@@ -94,6 +94,7 @@ def template(cls: type[Self], tier: int | None = None) -> SREConfig:
                 data_provider_ip_addresses=[
                     "List of IP addresses belonging to data providers"
                 ],
+                external_git_mirror="True/False: whether to deploy an external mirror git server (True), or only an internal server",  # type: ignore
                 remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct(
                     allow_copy=remote_desktop_allow_copy,
                     allow_paste=remote_desktop_allow_paste,

data_safe_haven/infrastructure/common/ip_ranges.py

@@ -16,6 +16,7 @@ class SREIpRanges:
     data_configuration = vnet.next_subnet(8)
     data_private = vnet.next_subnet(8)
     desired_state = vnet.next_subnet(8)
+    external_git_mirror = vnet.next_subnet(8)
     firewall = vnet.next_subnet(64)  # 64 address minimum
     firewall_management = vnet.next_subnet(64)  # 64 address minimum
     guacamole_containers = vnet.next_subnet(8)

data_safe_haven/infrastructure/programs/declarative_sre.py

@@ -318,6 +318,7 @@ def __call__(self) -> None:
                 dns_server_ip=dns.ip_address,
                 dockerhub_credentials=dockerhub_credentials,
                 gitea_database_password=data.password_gitea_database_admin,
+                external_git_mirror=self.config.sre.external_git_mirror,
                 hedgedoc_database_password=data.password_hedgedoc_database_admin,
                 ldap_server_hostname=identity.hostname,
                 ldap_server_port=identity.server_port,
@@ -334,6 +335,7 @@ def __call__(self) -> None:
                 subnet_containers=networking.subnet_user_services_containers,
                 subnet_containers_support=networking.subnet_user_services_containers_support,
                 subnet_databases=networking.subnet_user_services_databases,
+                subnet_external_git_mirror=networking.subnet_external_git_mirror,
                 subnet_software_repositories=networking.subnet_user_services_software_repositories,
             ),
             tags=self.tags,
@@ -362,7 +364,10 @@ def __call__(self) -> None:
                 clamav_mirror_hostname=clamav_mirror.hostname,
                 database_service_admin_password=data.password_database_service_admin,
                 dns_private_zones=dns.private_zones,
-                gitea_hostname=user_services.gitea_server.hostname,
+                gitea_hostnames={
+                    availability: server.hostname
+                    for (availability, server) in user_services.gitea_server.items()
+                },
                 hedgedoc_hostname=user_services.hedgedoc_server.hostname,
                 ldap_group_filter=ldap_group_filter,
                 ldap_group_search_base=ldap_group_search_base,

data_safe_haven/infrastructure/programs/sre/desired_state.py

@@ -46,7 +46,7 @@ def __init__(
         clamav_mirror_hostname: Input[str],
         database_service_admin_password: Input[str],
         dns_private_zones: Input[dict[str, network.PrivateZone]],
-        gitea_hostname: Input[str],
+        gitea_hostnames: Input[dict[str, str]],
         hedgedoc_hostname: Input[str],
         ldap_group_filter: Input[str],
         ldap_group_search_base: Input[str],
@@ -64,7 +64,8 @@ def __init__(
         self.clamav_mirror_hostname = clamav_mirror_hostname
         self.database_service_admin_password = database_service_admin_password
         self.dns_private_zones = dns_private_zones
-        self.gitea_hostname = gitea_hostname
+        self.internal_gitea_hostname = gitea_hostnames.get("internal")
+        self.external_gitea_hostname = gitea_hostnames.get("external")
         self.hedgedoc_hostname = hedgedoc_hostname
         self.ldap_group_filter = ldap_group_filter
         self.ldap_group_search_base = ldap_group_search_base
@@ -161,7 +162,7 @@ def __init__(
             source=Output.all(
                 clamav_mirror_hostname=props.clamav_mirror_hostname,
                 database_service_admin_password=props.database_service_admin_password,
-                gitea_hostname=props.gitea_hostname,
+                gitea_hostname=props.internal_gitea_hostname,
                 hedgedoc_hostname=props.hedgedoc_hostname,
                 ldap_group_filter=props.ldap_group_filter,
                 ldap_group_search_base=props.ldap_group_search_base,

data_safe_haven/infrastructure/programs/sre/gitea_server.py

@@ -29,6 +29,7 @@ def __init__(
         database_subnet_id: Input[str],
         dns_server_ip: Input[str],
         dockerhub_credentials: DockerHubCredentials,
+        gitea_server: Input[str],
         ldap_server_hostname: Input[str],
         ldap_server_port: Input[int],
         ldap_username_attribute: Input[str],
@@ -49,6 +50,7 @@ def __init__(
         )
         self.dns_server_ip = dns_server_ip
         self.dockerhub_credentials = dockerhub_credentials
+        self.gitea_server = gitea_server
         self.ldap_server_hostname = ldap_server_hostname
         self.ldap_server_port = ldap_server_port
         self.ldap_username_attribute = ldap_username_attribute
@@ -82,7 +84,7 @@ def __init__(
             access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED,
             account_name=props.storage_account_name,
             resource_group_name=props.resource_group_name,
-            share_name="gitea-caddy",
+            share_name=f"{props.gitea_server}-gitea-caddy",
             share_quota=1,
             signed_identifiers=[],
             opts=child_opts,
@@ -92,7 +94,7 @@ def __init__(
             access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED,
             account_name=props.storage_account_name,
             resource_group_name=props.resource_group_name,
-            share_name="gitea-gitea",
+            share_name=f"{props.gitea_server}-gitea-gitea",
             share_quota=1,
             signed_identifiers=[],
             opts=child_opts,
@@ -117,9 +119,14 @@ def __init__(
         )
 
         # Upload Gitea configuration script
-        gitea_configure_sh_reader = FileReader(
-            resources_path / "gitea" / "gitea" / "configure.mustache.sh"
-        )
+        if props.gitea_server == "external":
+            gitea_configure_sh_reader = FileReader(
+                resources_path / "gitea_external" / "gitea" / "configure.mustache.sh"
+            )
+        else:
+            gitea_configure_sh_reader = FileReader(
+                resources_path / "gitea" / "gitea" / "configure.mustache.sh"
+            )
         gitea_configure_sh = Output.all(
             admin_email="[email protected]",
             admin_username="dshadmin",
@@ -167,12 +174,12 @@ def __init__(
         # Define a PostgreSQL server and default database
         db_gitea_repository_name = "gitea"
         db_server_gitea = PostgresqlDatabaseComponent(
-            f"{self._name}_db_gitea",
+            f"{self._name}_db_gitea_{props.gitea_server}",
             PostgresqlDatabaseProps(
                 database_names=[db_gitea_repository_name],
                 database_password=props.database_password,
                 database_resource_group_name=props.resource_group_name,
-                database_server_name=f"{stack_name}-db-server-gitea",
+                database_server_name=f"{stack_name}-db-server-gitea-{props.gitea_server}",
                 database_subnet_id=props.database_subnet_id,
                 database_username=props.database_username,
                 disable_secure_transport=False,
@@ -182,10 +189,10 @@ def __init__(
             tags=child_tags,
         )
 
-        # Define the container group with guacd, guacamole and caddy
+        # Define the container group with gitea and caddy
         container_group = containerinstance.ContainerGroup(
             f"{self._name}_container_group",
-            container_group_name=f"{stack_name}-container-group-gitea",
+            container_group_name=f"{stack_name}-container-group-gitea-{props.gitea_server}",
             containers=[
                 containerinstance.ContainerArgs(
                     image="caddy:2.8.4",
@@ -341,7 +348,7 @@ def __init__(
             LocalDnsRecordProps(
                 base_fqdn=props.sre_fqdn,
                 private_ip_address=get_ip_address_from_container_group(container_group),
-                record_name="gitea",
+                record_name=f"{props.gitea_server}-gitea",
                 resource_group_name=props.resource_group_name,
             ),
             opts=ResourceOptions.merge(

data_safe_haven/infrastructure/programs/sre/networking.py

@@ -85,6 +85,41 @@ def __init__(
         )
 
         # Define NSGs
+        nsg_external_git_mirror = network.NetworkSecurityGroup(
+            f"{self._name}_nsg_external_git_mirror",
+            location=props.location,
+            network_security_group_name=f"{stack_name}-nsg-external-git-mirror",
+            resource_group_name=props.resource_group_name,
+            security_rules=[
+                # Inbound
+                network.SecurityRuleArgs(
+                    access=network.SecurityRuleAccess.DENY,
+                    description="Deny all other inbound traffic.",
+                    destination_address_prefix="*",
+                    destination_port_range="*",
+                    direction=network.SecurityRuleDirection.INBOUND,
+                    name="DenyAllOtherInbound",
+                    priority=NetworkingPriorities.ALL_OTHER,
+                    protocol=network.SecurityRuleProtocol.ASTERISK,
+                    source_address_prefix="*",
+                    source_port_range="*",
+                ),
+                network.SecurityRuleArgs(
+                    access=network.SecurityRuleAccess.ALLOW,
+                    description="Allow inbound connections from user container services.",
+                    destination_address_prefix=SREIpRanges.external_git_mirror.prefix,
+                    destination_port_range="*",
+                    direction=network.SecurityRuleDirection.INBOUND,
+                    name="AllowGitServersInbound",
+                    priority=NetworkingPriorities.INTERNAL_SRE_EXTERNAL_GIT_MIRROR,
+                    protocol=network.SecurityRuleProtocol.ASTERISK,
+                    source_address_prefix=SREIpRanges.user_services_containers.prefix,
+                    source_port_range="*",
+                ),
+            ],
+            opts=child_opts,
+            tags=child_tags,
+        )
         nsg_application_gateway = network.NetworkSecurityGroup(
             f"{self._name}_nsg_application_gateway",
             location=props.location,
@@ -1564,6 +1599,7 @@ def __init__(
         subnet_data_configuration_name = "DataConfigurationSubnet"
         subnet_desired_state_name = "DataDesiredStateSubnet"
         subnet_data_private_name = "DataPrivateSubnet"
+        subnet_external_git_mirror_name = "ExternalGitMirrorSubnet"
         subnet_firewall_name = "AzureFirewallSubnet"
         subnet_firewall_management_name = "AzureFirewallManagementSubnet"
         subnet_guacamole_containers_name = "GuacamoleContainersSubnet"
@@ -1754,6 +1790,22 @@ def __init__(
                     ),
                     route_table=network.RouteTableArgs(id=route_table.id),
                 ),
+                # User services external git mirror
+                network.SubnetArgs(
+                    address_prefix=SREIpRanges.external_git_mirror.prefix,
+                    delegations=[
+                        network.DelegationArgs(
+                            name="SubnetDelegationContainerGroups",
+                            service_name="Microsoft.ContainerInstance/containerGroups",
+                            type="Microsoft.Network/virtualNetworks/subnets/delegations",
+                        ),
+                    ],
+                    name=subnet_external_git_mirror_name,
+                    network_security_group=network.NetworkSecurityGroupArgs(
+                        id=nsg_external_git_mirror.id
+                    ),
+                    route_table=network.RouteTableArgs(id=route_table.id),
+                ),
                 # User services containers support
                 network.SubnetArgs(
                     address_prefix=SREIpRanges.user_services_containers_support.prefix,
@@ -1999,6 +2051,11 @@ def __init__(
             resource_group_name=props.resource_group_name,
             virtual_network_name=sre_virtual_network.name,
         )
+        self.subnet_external_git_mirror = network.get_subnet_output(
+            subnet_name=subnet_external_git_mirror_name,
+            resource_group_name=props.resource_group_name,
+            virtual_network_name=sre_virtual_network.name,
+        )
         self.subnet_firewall = network.get_subnet_output(
             subnet_name=subnet_firewall_name,
             resource_group_name=props.resource_group_name,

data_safe_haven/infrastructure/programs/sre/user_services.py

@@ -28,6 +28,7 @@ def __init__(
         dns_server_ip: Input[str],
         dockerhub_credentials: DockerHubCredentials,
         gitea_database_password: Input[str],
+        external_git_mirror: Input[bool],
         hedgedoc_database_password: Input[str],
         ldap_server_hostname: Input[str],
         ldap_server_port: Input[int],
@@ -44,13 +45,15 @@ def __init__(
         subnet_containers: Input[network.GetSubnetResult],
         subnet_containers_support: Input[network.GetSubnetResult],
         subnet_databases: Input[network.GetSubnetResult],
+        subnet_external_git_mirror: Input[network.GetSubnetResult],
         subnet_software_repositories: Input[network.GetSubnetResult],
     ) -> None:
         self.database_service_admin_password = database_service_admin_password
         self.databases = databases
         self.dns_server_ip = dns_server_ip
         self.dockerhub_credentials = dockerhub_credentials
         self.gitea_database_password = gitea_database_password
+        self.external_git_mirror = external_git_mirror
         self.hedgedoc_database_password = hedgedoc_database_password
         self.ldap_server_hostname = ldap_server_hostname
         self.ldap_server_port = ldap_server_port
@@ -73,6 +76,9 @@ def __init__(
         self.subnet_databases_id = Output.from_input(subnet_databases).apply(
             get_id_from_subnet
         )
+        self.subnet_external_git_mirror_id = Output.from_input(
+            subnet_external_git_mirror
+        ).apply(get_id_from_subnet)
         self.subnet_software_repositories_id = Output.from_input(
             subnet_software_repositories
         ).apply(get_id_from_subnet)
@@ -93,30 +99,43 @@ def __init__(
         child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self))
         child_tags = {"component": "user services"} | (tags if tags else {})
 
-        # Deploy the Gitea server
-        self.gitea_server = SREGiteaServerComponent(
-            "sre_gitea_server",
-            stack_name,
-            SREGiteaServerProps(
-                containers_subnet_id=props.subnet_containers_id,
-                database_subnet_id=props.subnet_containers_support_id,
-                database_password=props.gitea_database_password,
-                dns_server_ip=props.dns_server_ip,
-                dockerhub_credentials=props.dockerhub_credentials,
-                ldap_server_hostname=props.ldap_server_hostname,
-                ldap_server_port=props.ldap_server_port,
-                ldap_username_attribute=props.ldap_username_attribute,
-                ldap_user_filter=props.ldap_user_filter,
-                ldap_user_search_base=props.ldap_user_search_base,
-                location=props.location,
-                resource_group_name=props.resource_group_name,
-                sre_fqdn=props.sre_fqdn,
-                storage_account_key=props.storage_account_key,
-                storage_account_name=props.storage_account_name,
-            ),
-            opts=child_opts,
-            tags=child_tags,
-        )
+        # Deploy the Gitea servers
+        self.gitea_server = {}  # type: dict[str, SREGiteaServerComponent]
+        if props.external_git_mirror:
+            gitea_servers = ["internal", "external"]
+            subnet = {
+                "internal": props.subnet_containers_id,
+                "external": props.subnet_external_git_mirror_id,
+            }
+        else:
+            gitea_servers = ["internal"]
+            subnet = {"internal": props.subnet_containers_id}
+
+        for gitea_server in gitea_servers:
+            self.gitea_server[gitea_server] = SREGiteaServerComponent(
+                f"sre_{gitea_server}_gitea_server",
+                stack_name,
+                SREGiteaServerProps(
+                    containers_subnet_id=subnet[gitea_server],
+                    database_subnet_id=props.subnet_containers_support_id,
+                    database_password=props.gitea_database_password,
+                    dns_server_ip=props.dns_server_ip,
+                    dockerhub_credentials=props.dockerhub_credentials,
+                    gitea_server=gitea_server,
+                    ldap_server_hostname=props.ldap_server_hostname,
+                    ldap_server_port=props.ldap_server_port,
+                    ldap_username_attribute=props.ldap_username_attribute,
+                    ldap_user_filter=props.ldap_user_filter,
+                    ldap_user_search_base=props.ldap_user_search_base,
+                    location=props.location,
+                    resource_group_name=props.resource_group_name,
+                    sre_fqdn=props.sre_fqdn,
+                    storage_account_key=props.storage_account_key,
+                    storage_account_name=props.storage_account_name,
+                ),
+                opts=child_opts,
+                tags=child_tags,
+            )
 
         # Deploy the HedgeDoc server
         self.hedgedoc_server = SREHedgeDocServerComponent(

data_safe_haven/resources/gitea_external/caddy/Caddyfile

@@ -0,0 +1,14 @@
+# Refer to the Caddy docs for more information:
+# https://caddyserver.com/docs/caddyfile
+{
+	log {
+		format console {
+			level_format upper
+		}
+		level DEBUG
+	}
+}
+
+:80 {
+	reverse_proxy http://localhost:3000
+}

data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+# Ensure that default admin user exists
+until su-exec "$USER" /usr/local/bin/gitea admin user list --admin | grep "{{admin_username}}" > /dev/null 2>&1; do
+    echo "$(date -Iseconds) Attempting to create default admin user '{{admin_username}}'..." | tee -a /var/log/configuration
+    su-exec "$USER" /usr/local/bin/gitea admin user create --admin --username "{{admin_username}}" --random-password --random-password-length 20 --email "{{admin_email}}" 2> /dev/null
+    sleep 1
+done

data_safe_haven/resources/gitea_external/gitea/entrypoint.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+# Add configuration as an s6 target
+mkdir -p /etc/s6/setup
+rm /etc/s6/setup/run 2> /dev/null
+ln -s /app/custom/configure.sh /etc/s6/setup/run
+
+# Run the usual entrypoint
+/usr/bin/entrypoint

data_safe_haven/types/enums.py

@@ -85,6 +85,7 @@ class NetworkingPriorities(int, Enum):
     INTERNAL_SRE_DATA_CONFIGURATION = 1900
     INTERNAL_SRE_DATA_DESIRED_STATE = 1910
     INTERNAL_SRE_DATA_PRIVATE = 1920
+    INTERNAL_SRE_EXTERNAL_GIT_MIRROR = 1930
     INTERNAL_SRE_GUACAMOLE_CONTAINERS = 2000
     INTERNAL_SRE_GUACAMOLE_CONTAINERS_SUPPORT = 2100
     INTERNAL_SRE_IDENTITY_CONTAINERS = 2200