Skip to content

Commit

Permalink
Add multiple addresses to admin nixos module
Browse files Browse the repository at this point in the history
Signed-off-by: Manuel Bluhm <[email protected]>
  • Loading branch information
mbssrc committed Nov 20, 2024
1 parent 4f1ffae commit 9aea237
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 125 deletions.
132 changes: 48 additions & 84 deletions nixos/modules/admin.nix
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ let
mkIf
types
trivial
strings
concatStringsSep
attrsets
unique
;
inherit (import ./definitions.nix { inherit config lib; })
transportSubmodule
tlsSubmodule
;
in
{
Expand All @@ -31,22 +37,9 @@ in
default = "localhost";
};

addr = mkOption {
description = "IPv4 address.";
type = types.str;
default = "127.0.0.1";
};

port = mkOption {
description = "Port of the admin service. Defaults to '9001'.";
type = types.str;
default = "9001";
};

protocol = mkOption {
description = "Transport protocol, defaults to 'tcp'.";
type = types.str;
default = "tcp";
addresses = mkOption {
description = "List of addresses for the admin service to listen on. Requires a list of 'transportSubmodule'. The host name should be ignored.";
type = types.listOf transportSubmodule;
};

services = mkOption {
Expand All @@ -64,38 +57,7 @@ in
TLS options for gRPC connections. It is enabled by default to discourage unprotected connections,
and requires paths to certificates and key being set. To disable it use 'tls.enable = false;'.
'';
type =
with types;
submodule {
options = {
enable = mkOption {
description = "Enable TLS. Defaults to 'true'.";
type = bool;
default = true;
};
caCertPath = mkOption {
description = "Path to the CA certificate file.";
type = str;
default = "";
};
certPath = mkOption {
description = "Path to the service certificate file.";
type = str;
default = "";
};
keyPath = mkOption {
description = "Path to the service key file.";
type = str;
default = "";
};
};
};
default = {
enable = true;
caCertPath = "";
certPath = "";
keyPath = "";
};
type = tlsSubmodule;
};
};

Expand All @@ -108,43 +70,45 @@ in
}
];

systemd.services.givc-admin = {
description = "GIVC admin module.";
enable = true;
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "exec";
ExecStart = "${givc-admin}/bin/givc-admin";
Restart = "always";
RestartSec = 1;
};
environment =
{
"NAME" = "${cfg.name}";
"ADDR" = "${cfg.addr}";
"PORT" = "${cfg.port}";
"PROTO" = "${cfg.protocol}";
"TYPE" = "4";
"SUBTYPE" = "5";
"TLS" = "${trivial.boolToString cfg.tls.enable}";
"SERVICES" = "${concatStringsSep " " cfg.services}";
}
// attrsets.optionalAttrs cfg.tls.enable {
"CA_CERT" = "${cfg.tls.caCertPath}";
"HOST_CERT" = "${cfg.tls.certPath}";
"HOST_KEY" = "${cfg.tls.keyPath}";
}
// attrsets.optionalAttrs cfg.debug {
"RUST_BACKTRACE" = "1";
"GIVC_LOG" = "debug";
};
};
networking.firewall.allowedTCPPorts =
systemd.services.givc-admin =
let
port = lib.strings.toInt cfg.port;
tcpAddresses = lib.filter (addr: addr.protocol == "tcp") cfg.addresses;
unixAddresses = lib.filter (addr: addr.protocol == "unix") cfg.addresses;
args = concatStringsSep " " (
(map (addr: "--listen-tcp ${addr.addr}:${addr.port}") tcpAddresses)
++ (map (addr: "--listen-unix ${addr.addr}") unixAddresses)
);
in
[ port ];
{
description = "GIVC admin module.";
enable = true;
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "exec";
ExecStart = "${givc-admin}/bin/givc-admin ${args}";
Restart = "always";
RestartSec = 1;
};
environment =
{
"NAME" = "${cfg.name}";
"TYPE" = "4";
"SUBTYPE" = "5";
"TLS" = "${trivial.boolToString cfg.tls.enable}";
"SERVICES" = "${concatStringsSep " " cfg.services}";
}
// attrsets.optionalAttrs cfg.tls.enable {
"CA_CERT" = "${cfg.tls.caCertPath}";
"HOST_CERT" = "${cfg.tls.certPath}";
"HOST_KEY" = "${cfg.tls.keyPath}";
}
// attrsets.optionalAttrs cfg.debug {
"RUST_BACKTRACE" = "1";
"GIVC_LOG" = "debug";
};
};
networking.firewall.allowedTCPPorts = unique (map (addr: strings.toInt addr.port) cfg.addresses);
};
}
46 changes: 25 additions & 21 deletions nixos/tests/admin.nix
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ let
};
admin = {
name = "admin-vm";
addr = addrs.adminvm;
port = "9001";
protocol = "tcp"; # go version expect word "tcp" here, but it unused
addresses = [
{
name = "admin-vm";
addr = addrs.adminvm;
port = "9001";
protocol = "tcp";
}
];
};
mkTls = name: {
enable = tls;
Expand Down Expand Up @@ -48,9 +53,7 @@ in
enable = true;
debug = true;
inherit (admin) name;
inherit (admin) addr;
inherit (admin) port;
inherit (admin) protocol;
inherit (admin) addresses;
tls = mkTls "admin-vm";
};
};
Expand All @@ -70,7 +73,7 @@ in
port = "9000";
protocol = "tcp";
};
inherit admin;
admin = lib.head admin.addresses;
services = [
"[email protected]"
"[email protected]"
Expand Down Expand Up @@ -151,7 +154,7 @@ in
'';
givc.sysvm = {
enable = true;
inherit admin;
admin = lib.head admin.addresses;
agent = {
addr = addrs.guivm;
name = "gui-vm";
Expand Down Expand Up @@ -208,7 +211,7 @@ in
name = "chromium-vm";
addr = addrs.appvm;
};
inherit admin;
admin = lib.head admin.addresses;
tls = mkTls "chromium-vm";
applications = [
{
Expand All @@ -228,6 +231,7 @@ in
let
cli = "${self'.packages.givc-admin-rs.cli}/bin/givc-cli";
expected = "givc-ghaf-host.service"; # Name which we _expect_ to see registered in admin server's registry
adminAddr = lib.head admin.addresses;
in
# FIXME: why it so bizzare? (derived from name in cert)
''
Expand Down Expand Up @@ -297,7 +301,7 @@ in
time.sleep(1)
# Ensure, that hostvm's agent registered in admin service. It take ~10 seconds to spin up and register itself
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} test ensure --retry 60 ${expected}"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} test ensure --retry 60 ${expected}"))
with subtest("setup gui vm"):
# Ensure that sway in guiVM finished startup
Expand All @@ -313,13 +317,13 @@ in
time.sleep(5) # Give ssh some time to setup remote socket
with subtest("set locale and timezone"):
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} set-locale en_US.UTF-8"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} set-locale en_US.UTF-8"))
adminvm.wait_for_file("/etc/locale-givc.conf")
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} set-timezone UTC"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} set-timezone UTC"))
adminvm.wait_for_file("/etc/timezone.conf")
with subtest("Clean run"):
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm foot"))
time.sleep(10) # Give few seconds to application to spin up
wait_for_window("ghaf@appvm")
Expand All @@ -328,34 +332,34 @@ in
appvm.succeed("pkill foot")
time.sleep(10)
# .. then ask to restart
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm foot"))
wait_for_window("ghaf@appvm")
with subtest("pause/resume/stop application"):
appvm.succeed("pgrep foot")
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} pause [email protected]"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} pause [email protected]"))
time.sleep(20)
js = hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} query-list --as-json 2>/dev/null")
js = hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} query-list --as-json 2>/dev/null")
foot = by_name("[email protected]", json.loads(js))
assert foot["status"] == "Paused"
res = appvm.succeed("cat /sys/fs/cgroup/user.slice/user-1000.slice/[email protected]/app.slice/app-foot.slice/[email protected]/cgroup.events")
assert "frozen 1" in res
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} resume [email protected]"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} resume [email protected]"))
time.sleep(20)
res = appvm.succeed("cat /sys/fs/cgroup/user.slice/user-1000.slice/[email protected]/app.slice/app-foot.slice/[email protected]/cgroup.events")
assert "frozen 0" in res
js = hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} query-list --as-json 2>/dev/null")
js = hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} query-list --as-json 2>/dev/null")
foot = by_name("[email protected]", json.loads(js))
assert foot["status"] == "Running"
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} stop [email protected]"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} stop [email protected]"))
appvm.fail("pgrep foot")
with subtest("clear exit and restart"):
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm clearexit"))
time.sleep(20) # Give few seconds to application to spin up, exit, then start it again
print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit"))
print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm clearexit"))
'';
};
};
Expand Down
26 changes: 14 additions & 12 deletions nixos/tests/dbus.nix
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@ let
adminvm = "192.168.101.10";
appvm = "192.168.101.100";
};
admin = {
adminConfig = {
name = "admin-vm";
addr = addrs.adminvm;
port = "9001";
protocol = "tcp";
addresses = [
{
name = "admin-vm";
addr = addrs.adminvm;
port = "9001";
protocol = "tcp";
}
];
};
mkTls = name: {
enable = tls;
Expand Down Expand Up @@ -48,10 +53,7 @@ in
environment.systemPackages = [ pkgs.grpcurl ];
givc.admin = {
enable = true;
inherit (admin) name;
inherit (admin) addr;
inherit (admin) port;
inherit (admin) protocol;
inherit (adminConfig) addresses;
tls = mkTls "admin-vm";
debug = false;
};
Expand Down Expand Up @@ -125,7 +127,7 @@ in

givc.sysvm = {
enable = true;
inherit admin;
admin = lib.head adminConfig.addresses;
agent = {
addr = addrs.guivm;
name = "gui-vm";
Expand Down Expand Up @@ -211,7 +213,7 @@ in

givc.sysvm = {
enable = true;
inherit admin;
admin = lib.head adminConfig.addresses;
agent = {
addr = addrs.netvm;
name = "net-vm";
Expand Down Expand Up @@ -289,7 +291,7 @@ in

givc.sysvm = {
enable = true;
inherit admin;
admin = lib.head adminConfig.addresses;
agent = {
addr = addrs.audiovm;
name = "audio-vm";
Expand Down Expand Up @@ -377,7 +379,7 @@ in

givc.appvm = {
enable = true;
inherit admin;
admin = lib.head adminConfig.addresses;
agent = {
addr = addrs.appvm;
name = "chromium-vm";
Expand Down
Loading

0 comments on commit 9aea237

Please sign in to comment.