From cccb71138e355332c33c1f5333db8e393a112445 Mon Sep 17 00:00:00 2001
From: shivaraj-bh <sbh69840@gmail.com>
Date: Sun, 27 Oct 2024 19:48:47 +0530
Subject: [PATCH 1/3] revert: "refactor: consolidate all settings in
 `defaultProcessSettings` (#348)"

This reverts commit 1e682a85c4dbc98cad8e2a940e1759a66941df58.
---
 nix/lib.nix                         | 12 ------------
 nix/services/apache-kafka.nix       |  5 +++++
 nix/services/cassandra.nix          | 10 ++++++++++
 nix/services/clickhouse/default.nix | 10 ++++++++++
 nix/services/elasticsearch.nix      |  8 ++++++++
 nix/services/grafana.nix            |  8 ++++++++
 nix/services/memcached.nix          |  5 +++++
 nix/services/mysql/default.nix      | 10 ++++++++++
 nix/services/nginx/default.nix      | 10 ++++++++++
 nix/services/ollama.nix             |  5 +++++
 nix/services/open-webui.nix         |  5 +++++
 nix/services/pgadmin.nix            | 10 ++++++++++
 nix/services/postgres/default.nix   | 10 ++++++++++
 nix/services/prometheus.nix         | 10 ++++++++++
 nix/services/redis-cluster.nix      | 10 ++++++++++
 nix/services/redis.nix              | 10 ++++++++++
 nix/services/searxng.nix            |  5 +++++
 nix/services/tempo.nix              |  3 +++
 nix/services/tika.nix               |  5 +++++
 nix/services/weaviate.nix           | 10 ++++++++++
 nix/services/zookeeper.nix          |  5 +++++
 21 files changed, 154 insertions(+), 12 deletions(-)

diff --git a/nix/lib.nix b/nix/lib.nix
index 908a060d..a9078f48 100644
--- a/nix/lib.nix
+++ b/nix/lib.nix
@@ -38,18 +38,6 @@
               '';
               default = {
                 namespace = lib.mkDefault config.namespace;
-                # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
-                availability = {
-                  restart = lib.mkDefault "on_failure";
-                  max_restarts = lib.mkDefault 5;
-                };
-                readiness_probe = {
-                  initial_delay_seconds = lib.mkDefault 2;
-                  period_seconds = lib.mkDefault 10;
-                  timeout_seconds = lib.mkDefault 4;
-                  success_threshold = lib.mkDefault 1;
-                  failure_threshold = lib.mkDefault 5;
-                };
               };
             };
             settings = lib.mkOption {
diff --git a/nix/services/apache-kafka.nix b/nix/services/apache-kafka.nix
index 51d45bd6..642cbf80 100644
--- a/nix/services/apache-kafka.nix
+++ b/nix/services/apache-kafka.nix
@@ -170,6 +170,11 @@ with lib;
               readiness_probe = {
                 # TODO: need to find a better way to check if kafka is ready. Maybe use one of the scripts in bin?
                 exec.command = "${pkgs.netcat.nc}/bin/nc -z localhost ${builtins.toString config.port}";
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
 
               availability = {
diff --git a/nix/services/cassandra.nix b/nix/services/cassandra.nix
index 184c6f01..8c23b3ca 100644
--- a/nix/services/cassandra.nix
+++ b/nix/services/cassandra.nix
@@ -146,8 +146,18 @@ in
                 exec.command = ''
                   echo 'show version;' | CQLSH_HOST=${config.listenAddress} CQLSH_PORT=${toString config.nativeTransportPort} ${config.package}/bin/cqlsh
                 '';
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
 
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability = {
+                restart = "on_failure";
+                max_restarts = 5;
+              };
             };
         };
       };
diff --git a/nix/services/clickhouse/default.nix b/nix/services/clickhouse/default.nix
index 14c37f4b..f31bbf0b 100644
--- a/nix/services/clickhouse/default.nix
+++ b/nix/services/clickhouse/default.nix
@@ -154,8 +154,18 @@ in
                     host = "localhost";
                     port = if (lib.hasAttr "http_port" config.extraConfig) then config.extraConfig.http_port else 8123;
                   };
+                  initial_delay_seconds = 2;
+                  period_seconds = 10;
+                  timeout_seconds = 4;
+                  success_threshold = 1;
+                  failure_threshold = 5;
                 };
                 depends_on."${name}-init".condition = "process_completed_successfully";
+                # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+                availability = {
+                  restart = "on_failure";
+                  max_restarts = 5;
+                };
               };
           };
       };
diff --git a/nix/services/elasticsearch.nix b/nix/services/elasticsearch.nix
index 4186bdc1..07da86b0 100644
--- a/nix/services/elasticsearch.nix
+++ b/nix/services/elasticsearch.nix
@@ -185,9 +185,17 @@ in
             readiness_probe = {
               exec.command = "${pkgs.curl}/bin/curl -f -k http://${config.listenAddress}:${toString config.port}";
               initial_delay_seconds = 15;
+              period_seconds = 10;
               timeout_seconds = 2;
+              success_threshold = 1;
+              failure_threshold = 5;
             };
 
+            # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+            availability = {
+              restart = "on_failure";
+              max_restarts = 5;
+            };
           };
       };
     };
diff --git a/nix/services/grafana.nix b/nix/services/grafana.nix
index c89e52e9..8170e88b 100644
--- a/nix/services/grafana.nix
+++ b/nix/services/grafana.nix
@@ -167,9 +167,17 @@ in
                 path = "/api/health";
               };
               initial_delay_seconds = 15;
+              period_seconds = 10;
               timeout_seconds = 2;
+              success_threshold = 1;
+              failure_threshold = 5;
             };
 
+            # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+            availability = {
+              restart = "on_failure";
+              max_restarts = 5;
+            };
           };
       };
     };
diff --git a/nix/services/memcached.nix b/nix/services/memcached.nix
index bcd08cc7..be6b7103 100644
--- a/nix/services/memcached.nix
+++ b/nix/services/memcached.nix
@@ -45,6 +45,11 @@ in
         exec.command = ''
           echo -e "stats\nquit" | ${pkgs.netcat}/bin/nc ${config.bind} ${toString config.port} > /dev/null 2>&1
         '';
+        initial_delay_seconds = 2;
+        period_seconds = 10;
+        timeout_seconds = 4;
+        success_threshold = 1;
+        failure_threshold = 5;
       };
 
       # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
diff --git a/nix/services/mysql/default.nix b/nix/services/mysql/default.nix
index e4d1ce2a..d3c6fcbc 100644
--- a/nix/services/mysql/default.nix
+++ b/nix/services/mysql/default.nix
@@ -284,8 +284,18 @@ in
                   # Turns out using `--defaults-file` alone doesn't make the readiness_probe work unless `MYSQL_UNIX_PORT` is set.
                   # Hence the use of `--socket`.
                   exec.command = "${config.package}/bin/mysqladmin --socket=${config.socketDir}/mysql.sock ping -h localhost";
+                  initial_delay_seconds = 2;
+                  period_seconds = 10;
+                  timeout_seconds = 4;
+                  success_threshold = 1;
+                  failure_threshold = 5;
                 };
 
+                # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+                availability = {
+                  restart = "on_failure";
+                  max_restarts = 5;
+                };
               };
             "${name}-configure" = {
               command = configureScript;
diff --git a/nix/services/nginx/default.nix b/nix/services/nginx/default.nix
index f1738f1f..47453d2c 100644
--- a/nix/services/nginx/default.nix
+++ b/nix/services/nginx/default.nix
@@ -102,6 +102,16 @@ in
             readiness_probe = {
               # FIXME need a better health check
               exec.command = "[ -e ${config.dataDir}/nginx/nginx.pid ]";
+              initial_delay_seconds = 2;
+              period_seconds = 10;
+              timeout_seconds = 4;
+              success_threshold = 1;
+              failure_threshold = 5;
+            };
+            # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+            availability = {
+              restart = "on_failure";
+              max_restarts = 5;
             };
           };
         };
diff --git a/nix/services/ollama.nix b/nix/services/ollama.nix
index b4330579..639a0804 100644
--- a/nix/services/ollama.nix
+++ b/nix/services/ollama.nix
@@ -104,6 +104,11 @@ in
                   host = config.host;
                   port = config.port;
                 };
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
               availability = {
                 restart = "on_failure";
diff --git a/nix/services/open-webui.nix b/nix/services/open-webui.nix
index 44b0b929..d63511bd 100644
--- a/nix/services/open-webui.nix
+++ b/nix/services/open-webui.nix
@@ -83,6 +83,11 @@ in
                   host = config.host;
                   port = config.port;
                 };
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
               availability = {
                 restart = "on_failure";
diff --git a/nix/services/pgadmin.nix b/nix/services/pgadmin.nix
index 09a37bdc..bff99243 100644
--- a/nix/services/pgadmin.nix
+++ b/nix/services/pgadmin.nix
@@ -151,8 +151,18 @@ in
                     port = config.port;
                     path = "/misc/ping";
                   };
+                  initial_delay_seconds = 2;
+                  period_seconds = 10;
+                  timeout_seconds = 4;
+                  success_threshold = 1;
+                  failure_threshold = 5;
                 };
                 depends_on."${name}-init".condition = "process_completed_successfully";
+                # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+                availability = {
+                  restart = "on_failure";
+                  max_restarts = 5;
+                };
               };
           };
       };
diff --git a/nix/services/postgres/default.nix b/nix/services/postgres/default.nix
index 72c12222..f170efe7 100644
--- a/nix/services/postgres/default.nix
+++ b/nix/services/postgres/default.nix
@@ -326,8 +326,18 @@ in
                 shutdown.signal = 2;
                 readiness_probe = {
                   exec.command = "${config.package}/bin/pg_isready ${lib.concatStringsSep " " pg_isreadyArgs}";
+                  initial_delay_seconds = 2;
+                  period_seconds = 10;
+                  timeout_seconds = 4;
+                  success_threshold = 1;
+                  failure_threshold = 5;
                 };
                 depends_on."${name}-init".condition = "process_completed_successfully";
+                # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+                availability = {
+                  restart = "on_failure";
+                  max_restarts = 5;
+                };
               };
           };
         };
diff --git a/nix/services/prometheus.nix b/nix/services/prometheus.nix
index 4cfb1e9c..cf04b21a 100644
--- a/nix/services/prometheus.nix
+++ b/nix/services/prometheus.nix
@@ -82,8 +82,18 @@ in
                   port = config.port;
                   path = "/-/ready";
                 };
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
 
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability = {
+                restart = "on_failure";
+                max_restarts = 5;
+              };
             };
         };
       };
diff --git a/nix/services/redis-cluster.nix b/nix/services/redis-cluster.nix
index 9d8f6711..9afba325 100644
--- a/nix/services/redis-cluster.nix
+++ b/nix/services/redis-cluster.nix
@@ -103,8 +103,18 @@ in
 
               readiness_probe = {
                 exec.command = "${config.package}/bin/redis-cli -p ${port} ping";
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
 
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability = {
+                restart = "on_failure";
+                max_restarts = 5;
+              };
             };
           hosts = lib.mapAttrsToList (_: cfg: "${config.bind}:${builtins.toString cfg.port}") config.nodes;
           clusterCreateScript = pkgs.writeShellApplication {
diff --git a/nix/services/redis.nix b/nix/services/redis.nix
index f4444177..6b5c3039 100644
--- a/nix/services/redis.nix
+++ b/nix/services/redis.nix
@@ -104,7 +104,17 @@ in
                       "${config.package}/bin/redis-cli -s ${transformedSocketPath} ${toString config.port} ping"
                     else
                       "${config.package}/bin/redis-cli -p ${toString config.port} ping";
+                  initial_delay_seconds = 2;
+                  period_seconds = 10;
+                  timeout_seconds = 4;
+                  success_threshold = 1;
+                  failure_threshold = 5;
                 };
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability = {
+                restart = "on_failure";
+                max_restarts = 5;
+              };
             };
         };
       };
diff --git a/nix/services/searxng.nix b/nix/services/searxng.nix
index 573a1edd..1b30b0ac 100644
--- a/nix/services/searxng.nix
+++ b/nix/services/searxng.nix
@@ -72,6 +72,11 @@ in
                 host = config.host;
                 port = config.port;
               };
+              initial_delay_seconds = 2;
+              period_seconds = 10;
+              timeout_seconds = 4;
+              success_threshold = 1;
+              failure_threshold = 5;
             };
           };
         };
diff --git a/nix/services/tempo.nix b/nix/services/tempo.nix
index 1d12bbdb..22e6e966 100644
--- a/nix/services/tempo.nix
+++ b/nix/services/tempo.nix
@@ -112,7 +112,10 @@ in
                 path = "/ready";
               };
               initial_delay_seconds = 15;
+              period_seconds = 10;
               timeout_seconds = 2;
+              success_threshold = 1;
+              failure_threshold = 5;
             };
             availability = {
               restart = "on_failure";
diff --git a/nix/services/tika.nix b/nix/services/tika.nix
index 754330cb..9eac4a9f 100644
--- a/nix/services/tika.nix
+++ b/nix/services/tika.nix
@@ -58,6 +58,11 @@ in
                   host = config.host;
                   port = config.port;
                 };
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
             };
         };
diff --git a/nix/services/weaviate.nix b/nix/services/weaviate.nix
index 9b410cdd..94c6d0ce 100644
--- a/nix/services/weaviate.nix
+++ b/nix/services/weaviate.nix
@@ -73,6 +73,16 @@ in
                   path = "/v1/.well-known/ready";
                 };
                 initial_delay_seconds = 3;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
+              };
+
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability = {
+                restart = "on_failure";
+                max_restarts = 5;
               };
             };
         };
diff --git a/nix/services/zookeeper.nix b/nix/services/zookeeper.nix
index 2eb07768..0ddc1b3b 100644
--- a/nix/services/zookeeper.nix
+++ b/nix/services/zookeeper.nix
@@ -132,6 +132,11 @@ with lib;
 
               readiness_probe = {
                 exec.command = "echo stat | ${pkgs.netcat.nc}/bin/nc localhost ${toString config.port}";
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
               };
 
               availability = {

From 1c08b3bae3ba3e186daf77caafd3454345dd85f8 Mon Sep 17 00:00:00 2001
From: shivaraj-bh <sbh69840@gmail.com>
Date: Mon, 28 Oct 2024 01:26:26 +0530
Subject: [PATCH 2/3] fix(postgres): Move `pg_ctl stop` to trap during init

Ensuring init process from keep running because one of
`initialDatabases` or `initialScripts` failed to run
---
 nix/services/postgres/setup-script.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nix/services/postgres/setup-script.nix b/nix/services/postgres/setup-script.nix
index 705e0e05..627233dd 100644
--- a/nix/services/postgres/setup-script.nix
+++ b/nix/services/postgres/setup-script.nix
@@ -127,12 +127,13 @@ in
       }
       trap 'remove_tmp_pg_init_sock_dir "$PGHOST"' EXIT
 
+      trap 'pg_ctl -D "$PGDATA" -m fast -w stop' EXIT
+
       pg_ctl -D "$PGDATA" -w start -o "-c unix_socket_directories=$PGHOST -c listen_addresses= -p ${toString config.port}"
+
       ${runInitialScript.before}
       ${setupInitialDatabases}
       ${runInitialScript.after}
-      pg_ctl -D "$PGDATA" -m fast -w stop
-      remove_tmp_pg_init_sock_dir "$PGHOST"
     else
       echo
       echo "PostgreSQL database directory appears to contain a database; Skipping initialization"

From 75c55dce67e335ab3445942bb82e5e2276877cc7 Mon Sep 17 00:00:00 2001
From: shivaraj-bh <sbh69840@gmail.com>
Date: Mon, 28 Oct 2024 01:28:46 +0530
Subject: [PATCH 3/3] test(postgres): Error while init'ing database should fail
 the init process

---
 nix/services/postgres/bad_test.sql      |  3 +++
 nix/services/postgres/postgres_test.nix | 29 ++++++++++++++++++++++---
 test/flake.lock                         |  6 ++---
 test/flake.nix                          |  6 +++++
 4 files changed, 38 insertions(+), 6 deletions(-)
 create mode 100644 nix/services/postgres/bad_test.sql

diff --git a/nix/services/postgres/bad_test.sql b/nix/services/postgres/bad_test.sql
new file mode 100644
index 00000000..cbc29ee5
--- /dev/null
+++ b/nix/services/postgres/bad_test.sql
@@ -0,0 +1,3 @@
+-- Tests if the postgres process gets skipped when there is an error while setting up inital databases
+CREATE STABLE users (id INT PRIMARY KEY, user_name VARCHAR(25));
+
diff --git a/nix/services/postgres/postgres_test.nix b/nix/services/postgres/postgres_test.nix
index 1adf9789..c3ad015e 100644
--- a/nix/services/postgres/postgres_test.nix
+++ b/nix/services/postgres/postgres_test.nix
@@ -1,4 +1,4 @@
-{ pkgs, config, ... }: {
+{ pkgs, config, name, ... }: {
   services.postgres."pg1" = {
     enable = true;
     initialScript.before = "CREATE USER bar;";
@@ -26,16 +26,31 @@
       }
     ];
   };
+
+  # Tests if `pg4-init` fails due to `./bad_test.sql`
+  services.postgres."pg4" = {
+    enable = true;
+    socketDir = "./test/new/socket/path3";
+    listen_addresses = "";
+    initialDatabases = [
+      {
+        name = "test-db";
+        schemas = [ ./bad_test.sql ];
+      }
+    ];
+  };
   # avoid both the processes trying to create `data` directory at the same time
   settings.processes."pg2-init".depends_on."pg1-init".condition = "process_completed_successfully";
   settings.processes."pg3-init".depends_on."pg2-init".condition = "process_completed_successfully";
+  settings.processes."pg4-init".depends_on."pg3-init".condition = "process_completed_successfully";
+
   settings.processes.test =
     let
       cfg = config.services.postgres."pg1";
     in
     {
       command = pkgs.writeShellApplication {
-        runtimeInputs = [ cfg.package pkgs.gnugrep ];
+        runtimeInputs = [ cfg.package pkgs.gnugrep pkgs.curl pkgs.jq ];
         text = ''
           echo 'SELECT version();' | psql -h 127.0.0.1
           echo 'SHOW hba_file;' | psql -h 127.0.0.1 | ${pkgs.gawk}/bin/awk 'NR==3' | grep '^ /nix/store'
@@ -48,9 +63,17 @@
 
           # schemas test
           echo "SELECT * from users where user_name = 'test_user';" | psql -h 127.0.0.1 -p 5433 -d sample-db | grep -q test_user
-         
+ 
           # listen_addresses test
           echo "SELECT 1 FROM pg_database where datname = 'test-db';" | psql -h "$(readlink -f ${config.services.postgres.pg3.socketDir})" -d postgres | grep -q 1
+
+          # Test if `pg4-init` fails due to `bad_test.sql`
+          #
+          # The curl to process-compose server is documented in the swagger URL, http://localhost:8080, but since we are listening on unix socket here, you can use `socat` to temporarily pipe the port `8080` to the `pc-${name}.sock` (`socat TCP-LISTEN:8080,fork UNIX-CONNECT:/path/to/your/socket.sock
+ 
+          # OpenAPI documentation from the swagger URL:
+          # /process/logs/{name}/{endOffset}/{limit}
+          curl --unix-socket pc-${name}.sock http://localhost/process/logs/pg4-init/30/0 | jq '.logs | contains(["ERROR:  syntax error at or near \"STABLE\""])' | grep "true"
         '';
         name = "postgres-test";
       };
diff --git a/test/flake.lock b/test/flake.lock
index 12a8dae8..19d690e9 100644
--- a/test/flake.lock
+++ b/test/flake.lock
@@ -48,11 +48,11 @@
     },
     "process-compose-flake": {
       "locked": {
-        "lastModified": 1727723765,
-        "narHash": "sha256-/3az0h2Zp02bjJQKliR0PEQiT+FMLOR1GumhH42jA4w=",
+        "lastModified": 1729544733,
+        "narHash": "sha256-ir7WTVpG999N07wkOCs1kwZsQKitOv3CNDqNalCMK3c=",
         "owner": "Platonic-Systems",
         "repo": "process-compose-flake",
-        "rev": "22f2aeb7e0b88006d1d27065f8422f2790d99e47",
+        "rev": "7241e6b247b65cfa579d6f838a8efc18e5401924",
         "type": "github"
       },
       "original": {
diff --git a/test/flake.nix b/test/flake.nix
index f5a43b54..fbb1f732 100644
--- a/test/flake.nix
+++ b/test/flake.nix
@@ -30,6 +30,12 @@
                   inputs.services-flake.processComposeModules.default
                   mod
                 ];
+                cli = {
+                  options = {
+                    use-uds = true;
+                    unix-socket = "pc-${name}.sock";
+                  };
+                };
               };
           in
           builtins.listToAttrs (builtins.map mkPackageFor ([