Add monitoring server configuration

Signed-off-by: Joonas Rautiola <[email protected]>

.sops.yaml

@@ -10,6 +10,7 @@ keys:
   - &karim age122lvqyrdqz30fkfututykl0yle9u63u2em6e4aut7e5draws83ns3npt3a
   - &jrautiola age15jq5gjjd7ypsdlqfjtqy4red57v8ggqq9na6u3xffznu678nydpsuuwjg0
   - &binarycache age1s47a3y44j695gemcl0kqgjlxxvaa50de9s69jy2l6vc8xtmk5pcskhpknl
+  - &monitoring age17s9sc2cgt9t30cyl65zya8p4zmwnndrx2r896e7gzgl08sjn0qmq3t6shs
 creation_rules:
   - path_regex: terraform/secrets.yaml$
     key_groups:
@@ -34,3 +35,8 @@ creation_rules:
     - age:
       - *jrautiola
       - *binarycache
+  - path_regex: hosts/monitoring/secrets.yaml$
+    key_groups:
+    - age:
+      - *jrautiola
+      - *monitoring

hosts/binarycache/default.nix

@@ -23,6 +23,7 @@
       service-openssh
       service-binary-cache
       service-nginx
+      service-node-exporter
       user-jrautiola
       user-cazfi
       user-hydra
@@ -37,13 +38,6 @@
   };
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  services.openssh.enable = true;
-
-  boot.loader.grub = {
-    enable = true;
-    # qemu vms are using SeaBIOS which is not UEFI
-    efiSupport = false;
-  };
 
   networking = {
     hostName = "binarycache";

hosts/default.nix

@@ -17,6 +17,7 @@
     host-build01 = import ./build01;
     host-ghafhydra = import ./ghafhydra;
     host-binarycache = import ./binarycache;
+    host-monitoring = import ./monitoring;
   };
 
   flake.nixosConfigurations = let
@@ -36,5 +37,9 @@
       inherit specialArgs;
       modules = [self.nixosModules.host-binarycache];
     };
+    monitoring = lib.nixosSystem {
+      inherit specialArgs;
+      modules = [self.nixosModules.host-monitoring];
+    };
   };
 }

hosts/monitoring/default.nix

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  self,
+  inputs,
+  lib,
+  config,
+  ...
+}: let
+  # "public" but really only available with ficolo vpn
+  public-ip = "172.18.20.108";
+in {
+  imports = lib.flatten [
+    (with inputs; [
+      nix-serve-ng.nixosModules.default
+      disko.nixosModules.disko
+    ])
+    (with self.nixosModules; [
+      common
+      qemu-common
+      service-openssh
+      service-nginx
+      service-node-exporter
+      user-jrautiola
+    ])
+    ./disk-config.nix
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  networking = {
+    hostName = "monitoring";
+    nameservers = ["1.1.1.1" "8.8.8.8"];
+    firewall = {
+      allowedTCPPorts = [config.services.prometheus.port config.services.grafana.settings.server.http_port];
+      allowedUDPPorts = [config.services.prometheus.port config.services.grafana.settings.server.http_port];
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+
+    settings = {
+      server = {
+        http_port = 3000;
+        http_addr = "127.0.0.1";
+      };
+
+      # disable telemetry
+      analytics = {
+        reporting_enabled = false;
+        feedback_links_enabled = false;
+      };
+
+      # allow read-only access to dashboards without login
+      "auth.anonymous".enabled = true;
+    };
+
+    provision.datasources.settings.datasources = [
+      {
+        name = "prometheus";
+        type = "prometheus";
+        isDefault = true;
+        url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+      }
+    ];
+  };
+
+  services.prometheus = {
+    enable = true;
+
+    port = 9090;
+    listenAddress = "0.0.0.0";
+    webExternalUrl = "http://${public-ip}:${toString config.services.prometheus.port}";
+    checkConfig = true;
+
+    scrapeConfigs = [
+      {
+        job_name = "ficolo-node-exporter";
+        static_configs = [
+          {
+            targets = [
+              "172.18.20.109:9002" # binarycache
+              "172.18.20.105:9999" # build4
+            ];
+          }
+        ];
+      }
+    ];
+  };
+
+  services.nginx = {
+    virtualHosts = {
+      "_" = {
+        default = true;
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+          proxyWebsockets = true;
+        };
+      };
+    };
+  };
+}

hosts/monitoring/disk-config.nix

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+# BIOS compatible gpt partition
+{
+  disko.devices = {
+    disk = {
+      vda = {
+        device = "/dev/vda";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size = "1M";
+              type = "EF02";
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/monitoring/secrets.yaml

@@ -0,0 +1,30 @@
+ssh_host_ed25519_key: ENC[AES256_GCM,data:3syc79zImKOJ85pvTNTFN1HWC4cX2aguP8OtlbxiElZWVC6nanJ6boAhjaO/sdIh9tppmrUdBID6QJsLnFABwtEwUj2O0oGYUddtZxerXnyEeP4EncSubJQDtmJxdVWMT7lnu0nseZT54CozbhQ+7EdX6kEOGwLR2Hm0nbaGNnHQLZKaxN37gKycG2csLs1Z2pI03dutFk4E7z1bD0QMnp2V6fzl1Gig4F4eip47nuWC76YDcVDLn8lrOsn5Hsbdbf63OfGqfw4HzEZrXrtf2TWkUgsek6zGJGZeAOMvEZHOFrzj9ojYpKuBtTcG6Dw2XgKIdld96ujwhiHzm8sWjBSp0XQs2lU0W7LndNypOA5PwZwVrQguh1fVTJlcCLJPqSi0DlIbdjpUaZAdAWwxsHkB2Mj1z9sh3K9CA/Ia045VbVixXZUobxERIkvqkhCmoF5WubaMwZFLBE2pF2VaRwm9pHXg9bbDh9PPoACsTgyFYBtmOJKgPk5uik7uj3+KyxeNhp09Lc34XzrATx6z,iv:M9WhCLuUDyo4w44R1AQovxNRSYaAQWtlYQK2EHkRubY=,tag:GyceOfzblQnH6WPCXlsgQg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age15jq5gjjd7ypsdlqfjtqy4red57v8ggqq9na6u3xffznu678nydpsuuwjg0
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByUzBsYVg0SlgxK1ZjWEZz
+            Q092VHdvYnExN09TK3p0U3k2NmVKK0FHdVJrCk1jYlBKYjh6NU1vVHRzbmFmSllq
+            bzBDdzhZRWV6MDFBRGRXTHNpMk9HQ28KLS0tIEtNZUVMWndKOFVnZkhXWGVwb1dS
+            dzJEN1Y5d01vanRtV2J1aFhRbnM1bjAK5N+aLBhNtHRsz2l+rjTU+Szj/bHDgq1m
+            xc6uZKNIqwcXP+t308mKDDJdFxZRp/AkJqgV1IYFRs8U9CLZi11ccw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age17s9sc2cgt9t30cyl65zya8p4zmwnndrx2r896e7gzgl08sjn0qmq3t6shs
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1ejRkQVFxYzBiYVEzMXlz
+            T2VqTk5YSVRtcmJ6QnFLanZlS1FqeFhZeDNNCnNEK25tTzNNT0tQMDBFSTcxVjN3
+            cHpBTXpINEEvVUJWMExLVm5vTUVWUkkKLS0tIHorZUVYZ2RTMmVJdWRXa3h5MmdJ
+            Sk1sQU9iUVBJRG1jRFcyTWg2ODA4UUUKCO2FYq3r6RZhtEMrLzs+hl+LJNTaH/M4
+            hyzYEGTzQahj0JzbRLeQPFmgV9x49N6nUpgY4fdkI81RM4Q/M827JQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-11-23T10:29:16Z"
+    mac: ENC[AES256_GCM,data:ZDofz3Sj4oGeuntdpUSqTIG2AMUmzNi3E/hYzE79wbmQNz4Fjb7nU26Ko7wR1V7p1ilYFCykL5vFiJzHTwXP2nFR6DsycADquKXGcM6sKTSJrxPcKdym/pA3OW5UoSanBBSQrDLkAldyZdjqSQpCrovwyfUkt1ikR2WT0YTcW94=,iv:+mZl+Z35Zu75BOACAlXWCcp7fHAcHOqu5QQomNIHWko=,tag:en9ihk127dmSm/jfnMIpXQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3

hosts/monitoring/secrets.yaml.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+
+SPDX-License-Identifier: Apache-2.0

hosts/qemu-common.nix

@@ -1,9 +1,24 @@
 # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
 #
 # SPDX-License-Identifier: Apache-2.0
-_: {
+{
   services.qemuGuest.enable = true;
-  boot.kernelParams = ["console=ttyS0" "earlyprintk=ttyS0" "rootdelay=300" "panic=1" "boot.panic_on_fail"];
-  boot.initrd.availableKernelModules = ["ahci" "xhci_pci" "virtio_pci" "sr_mod" "virtio_blk" "uhci_hcd" "ehci_pci" "virtio_scsi"];
-  boot.initrd.kernelModules = ["kvm-intel" "dm-snapshot"];
+
+  boot = {
+    kernelParams = ["console=ttyS0" "earlyprintk=ttyS0" "rootdelay=300" "panic=1" "boot.panic_on_fail"];
+    initrd = {
+      availableKernelModules = ["ahci" "xhci_pci" "virtio_pci" "sr_mod" "virtio_blk" "uhci_hcd" "ehci_pci" "virtio_scsi"];
+      kernelModules = ["kvm-intel" "dm-snapshot"];
+    };
+
+    loader.grub = {
+      enable = true;
+      # qemu vms are using SeaBIOS which is not UEFI
+      efiSupport = false;
+    };
+  };
+
+  networking.extraHosts = ''
+    172.18.20.109 cache.vedenemo.dev
+  '';
 }

services/default.nix

@@ -7,5 +7,6 @@
     service-hydra = import ./hydra;
     service-nginx = import ./nginx;
     service-openssh = import ./openssh;
+    service-node-exporter = import ./node-exporter;
   };
 }

services/node-exporter/default.nix

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{config, ...}: {
+  networking.firewall = {
+    allowedTCPPorts = [config.services.prometheus.exporters.node.port];
+    allowedUDPPorts = [config.services.prometheus.exporters.node.port];
+  };
+
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = ["systemd"];
+      port = 9002;
+    };
+  };
+}

tasks.py

@@ -79,6 +79,9 @@ class TargetHost:
         "binarycache-ficolo": TargetHost(
             hostname="172.18.20.109", nixosconfig="binarycache"
         ),
+        "monitoring-ficolo": TargetHost(
+            hostname="172.18.20.108", nixosconfig="monitoring"
+        ),
     }
 )