jenkins-controller: Expose jenkins over HTTPS

Deploy caddy reverse proxy, redirecting public HTTPS requests to jenkins service on the jenkins-controller VM. No need to open port 80 as caddy can deploy let's encrypt certificates using tls-alpn-01 (over HTTPS). Also, adds a 'persistent' state disk on jenkins-controller VM to store let's encrypt data for caddy. Signed-off-by: Henri Rosten <[email protected]>
tiiuae · Feb 26, 2024 · d9e01ac · d9e01ac
1 parent 88d0be2
commit d9e01ac
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 12 deletions.
diff --git a/hosts/azure/jenkins-controller/configuration.nix b/hosts/azure/jenkins-controller/configuration.nix
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 {
   pkgs,
+  config,
   self,
   lib,
   ...
@@ -90,6 +91,18 @@ in {
     ];
   };
 
+  # Configure /var/lib/caddy in /etc/fstab.
+  # Due to an implicit RequiresMountsFor=$state-dir, systemd
+  # will block starting the service until this mounted.
+  fileSystems."/var/lib/caddy" = {
+    device = "/dev/disk/by-lun/11";
+    fsType = "ext4";
+    options = [
+      "x-systemd.makefs"
+      "x-systemd.growfs"
+    ];
+  };
+
   services.jenkins = {
     enable = true;
     listenAddress = "localhost";
@@ -294,7 +307,37 @@ in {
     post-build-hook = ${post-build-hook}
   '';
 
-  # TODO: deploy reverse proxy, sort out authentication (SSO?)
+  # TODO: use https://caddyserver.com/docs/caddyfile-tutorial#environment-variables for domain
+  services.caddy = {
+    enable = true;
+    configFile = pkgs.writeTextDir "Caddyfile" ''
+      # Disable the admin API, we don't want to reconfigure Caddy at runtime.
+      {
+        admin off
+      }
+
+      # Proxy all requests to jenkins.
+      https://{$SITE_ADDRESS} {
+        reverse_proxy localhost:8081
+      }
+    '';
+  };
+
+  # workaround for https://github.com/NixOS/nixpkgs/issues/272532
+  # FUTUREWORK: rebase once https://github.com/NixOS/nixpkgs/pull/272617 landed
+  services.caddy.enableReload = false;
+  systemd.services.caddy.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.caddy}/bin/caddy run --environ --config ${config.services.caddy.configFile}/Caddyfile"
+  ];
+  systemd.services.caddy.serviceConfig.EnvironmentFile = "/run/caddy.env";
+
+  # Wait for cloud-init mounting before we start caddy.
+  systemd.services.caddy.after = ["cloud-init.service"];
+  systemd.services.caddy.requires = ["cloud-init.service"];
+
+  # Expose the HTTPS port. No need for HTTP, as caddy can use TLS-ALPN-01.
+  networking.firewall.allowedTCPPorts = [443];
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
 

diff --git a/terraform/jenkins-controller.tf b/terraform/jenkins-controller.tf
@@ -62,6 +62,10 @@ module "jenkins_controller_vm" {
       {
         content = join("\n", toset(module.builder_vm[*].virtual_machine_private_ip_address))
         "path"  = "/var/lib/builder-keyscan/scanlist"
+      },
+      {
+        content = "SITE_ADDRESS=ghaf-jenkins-controller-${local.env}.northeurope.cloudapp.azure.com",
+        "path"  = "/run/caddy.env"
       }
     ]
   })])
@@ -70,14 +74,24 @@ module "jenkins_controller_vm" {
   subnet_id          = azurerm_subnet.jenkins.id
 
   # Attach disk to the VM
-  data_disks = [{
-    name            = azurerm_managed_disk.jenkins_controller_jenkins_state.name
-    managed_disk_id = azurerm_managed_disk.jenkins_controller_jenkins_state.id
-    lun             = "10"
-    # create_option = "Attach"
-    caching      = "None"
-    disk_size_gb = azurerm_managed_disk.jenkins_controller_jenkins_state.disk_size_gb
-  }]
+  data_disks = [
+    {
+      name            = azurerm_managed_disk.jenkins_controller_jenkins_state.name
+      managed_disk_id = azurerm_managed_disk.jenkins_controller_jenkins_state.id
+      lun             = "10"
+      # create_option = "Attach"
+      caching      = "None"
+      disk_size_gb = azurerm_managed_disk.jenkins_controller_jenkins_state.disk_size_gb
+    },
+    {
+      name            = data.azurerm_managed_disk.jenkins_controller_caddy_state.name
+      managed_disk_id = data.azurerm_managed_disk.jenkins_controller_caddy_state.id
+      lun             = "11"
+      create_option   = "Attach"
+      caching         = "None"
+      disk_size_gb    = data.azurerm_managed_disk.jenkins_controller_caddy_state.disk_size_gb
+    }
+  ]
 }
 
 resource "azurerm_network_interface_security_group_association" "jenkins_controller_vm" {
@@ -91,13 +105,13 @@ resource "azurerm_network_security_group" "jenkins_controller_vm" {
   location            = azurerm_resource_group.infra.location
 
   security_rule {
-    name                       = "AllowSSHInbound"
+    name                       = "AllowSSHHTTPSInbound"
     priority                   = 400
     direction                  = "Inbound"
     access                     = "Allow"
     protocol                   = "Tcp"
     source_port_range          = "*"
-    destination_port_ranges    = [22]
+    destination_port_ranges    = [22, 443]
     source_address_prefix      = "*"
     destination_address_prefix = "*"
   }

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -42,7 +42,7 @@ variable "location" {
 }
 
 # Use azure_region module to get the short name of the Azure region,
-# see: https://registry.terraform.io/modules/claranet/regions/azurerm/latest 
+# see: https://registry.terraform.io/modules/claranet/regions/azurerm/latest
 module "azure_region" {
   source       = "claranet/regions/azurerm"
   azure_region = var.location
@@ -209,5 +209,9 @@ data "azurerm_managed_disk" "binary_cache_caddy_state" {
   resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
 }
 
+data "azurerm_managed_disk" "jenkins_controller_caddy_state" {
+  name                = "jenkins-controller-vm-caddy-state-${local.ws}"
+  resource_group_name = "ghaf-infra-persistent-${local.shortloc}"
+}
 
 ################################################################################
diff --git a/terraform/persistent/workspace-specific/main.tf b/terraform/persistent/workspace-specific/main.tf
@@ -73,4 +73,13 @@ resource "azurerm_managed_disk" "binary_cache_caddy_state" {
   disk_size_gb         = 1
 }
 
+resource "azurerm_managed_disk" "jenkins_controller_caddy_state" {
+  name                 = "jenkins-controller-vm-caddy-state-${local.ws}"
+  resource_group_name  = data.azurerm_resource_group.persistent.name
+  location             = data.azurerm_resource_group.persistent.location
+  storage_account_type = "Standard_LRS"
+  create_option        = "Empty"
+  disk_size_gb         = 1
+}
+
 ################################################################################