add quickstart support for ha

.github/workflows/test-quickstart.yml

@@ -84,3 +84,25 @@ jobs:
           ls -lAn ${GOCACHE:-${HOME}/.cache/go-build}/ ${GOPATH:-${HOME}/go}/pkg/mod/
           docker compose --profile test logs
           exit 0
+
+  haQuickstartTest:
+    name: Test HA Quickstart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Shallow checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: ./go.mod
+
+      - name: Build ziti executable
+        shell: bash
+        run: |
+          mkdir -pv /tmp/build
+          go build -o /tmp/build ${GITHUB_WORKSPACE}/...
+
+      - name: Build and run a three quickstart in HA mode
+        shell: bash
+        run: ./quickstart/test/ha-test.sh

quickstart/test/ha-test.sh

@@ -0,0 +1,123 @@
+BUILD_DIR=/tmp/build
+
+ctrl_port=2001
+router_port=3001
+rm -rf "/tmp/quickstart-ha-test"
+ziti_home="/tmp/quickstart-ha-test"
+
+function _wait_for_controller {
+    local advertised_host_port="127.0.0.1:${1}"
+    local timeout=60
+    local elapsed=0
+
+    while [[ "$(curl -w "%{http_code}" -m 1 -s -k -o /dev/null https://${advertised_host_port}/edge/client/v1/version)" != "200" ]]; do
+        if (( elapsed >= timeout )); then
+            echo "Timeout waiting for https://${advertised_host_port}" >&2
+            exit 1
+        fi
+        echo "waiting for https://${advertised_host_port}"
+        sleep 3
+        (( elapsed += 3 ))
+    done
+    echo "CONTROLLER ONLINE AT: https://${advertised_host_port}"
+}
+
+function _stop_instances {
+  echo "killing...."
+  kill "$@" 2>/dev/null
+
+  for pid in "$@"; do
+      while kill -0 "$pid" 2>/dev/null; do
+          echo "Waiting for process $pid to stop..."
+          sleep 1
+      done
+      echo "Process $pid has stopped."
+  done
+}
+
+trap 'kill $inst001pid $inst002pid $inst003pid 2>/dev/null' EXIT
+
+"${BUILD_DIR}/ziti" edge quickstart ha \
+    --home "${ziti_home}" \
+    --trust-domain="quickstart-ha-test" \
+    --instance-id inst001 \
+    --ctrl-port "${ctrl_port}" \
+    --router-port "${router_port}" \
+    &
+inst001pid=$!
+
+_wait_for_controller "${ctrl_port}"
+sleep 5
+echo "controller online"
+
+"${BUILD_DIR}/ziti" edge quickstart join \
+    --home "${ziti_home}" \
+    --trust-domain="quickstart-ha-test" \
+    --ctrl-port 2002 \
+    --router-port 3002 \
+    --instance-id "inst002" \
+    --member-pid "${inst001pid}" &
+inst002pid=$!
+
+"${BUILD_DIR}/ziti" edge quickstart join \
+    --home "${ziti_home}" \
+    --trust-domain="quickstart-ha-test" \
+    --ctrl-port 2003 \
+    --router-port 3003 \
+    --instance-id "inst003" \
+    --member-pid "${inst001pid}" &
+inst003pid=$!
+
+count=0
+timeout=60  # Timeout in seconds
+elapsed=0
+
+while [[ $count -lt 3 ]]; do
+    results=$("${BUILD_DIR}/ziti" fabric list links -j | jq -r '.data[].state')
+    connected_count=$(echo "$results" | grep -c "Connected")
+
+    if [[ $connected_count -eq 3 ]]; then
+        echo "All three are connected."
+        break
+    else
+        echo "Waiting for three router links before continuing..."
+        sleep 3
+        ((elapsed+=3))
+
+        if [[ $elapsed -ge $timeout ]]; then
+            echo "Timeout reached; not all connections are 'Connected'."
+            exit 1
+        fi
+    fi
+done
+
+# three links == things are ready -- tests start below
+output=$("${BUILD_DIR}/ziti" agent cluster list --pid $inst001pid)
+
+echo ""
+echo "$output"
+echo ""
+
+# Extract the columns for LEADER and CONNECTED
+leaders=$(echo "$output" | grep inst | awk -F '│' '{print $5}')
+connected=$(echo "$output" | grep inst | awk -F '/│' '{print $6}')
+
+# Check there is only one leader
+leader_count=$(echo "$leaders" | grep -c "true")
+if [[ $leader_count -ne 1 ]]; then
+    echo "Test failed: Expected 1 leader, found $leader_count"
+    _stop_instances $inst001pid $inst002pid $inst003pid
+    exit 1
+fi
+
+# Check all are connected
+disconnected_count=$(echo "$connected" | grep -c "false")
+if [[ $disconnected_count -ne 0 ]]; then
+    echo "Test failed: Some instances are not connected"
+    _stop_instances $inst001pid $inst002pid $inst003pid
+    exit 1
+fi
+
+echo "Test passed: One leader found and all instances are connected"
+_stop_instances $inst001pid $inst002pid $inst003pid
+

ziti/cmd/create/config_templates/controller.yml

@@ -233,6 +233,8 @@ web:
         options: { }
       - binding: fabric
         options: { }
+      - binding: edge-oidc
+        options: { }
       {{ if not .Controller.Web.BindPoints.Console.Enabled }}#{{- end }}- binding: zac
       {{ if not .Controller.Web.BindPoints.Console.Enabled }}#{{- end }}  options:
       {{ if not .Controller.Web.BindPoints.Console.Enabled }}#{{- end }}    location: {{ .Controller.Web.BindPoints.Console.Location }}

ziti/cmd/create/config_templates/router.yml

@@ -16,6 +16,9 @@ identity:
   {{ if not .Router.AltCertsEnabled }}#{{ end }}  - server_cert:  "{{ .Router.AltServerCert }}"
   {{ if not .Router.AltCertsEnabled }}#{{ end }}    server_key:   "{{ .Router.AltServerKey }}"
 
+ha:
+  enabled: {{ .Router.IsHA }}
+
 ctrl:
   endpoint:             tls:{{ .Controller.Ctrl.AdvertisedAddress }}:{{ .Controller.Ctrl.AdvertisedPort }}
 

ziti/cmd/create/create_config.go

@@ -72,6 +72,7 @@ type CtrlValues struct {
 	BindAddress                string
 	AltAdvertisedAddress       string
 	MinClusterSize             int
+	InstanceId                 string
 }
 
 type HealthChecksValues struct {
@@ -112,8 +113,8 @@ type BindPointsValues struct {
 }
 
 type ConsoleValues struct {
-	Enabled       bool
-	Location      string
+	Enabled  bool
+	Location string
 }
 
 type IdentityValues struct {
@@ -165,6 +166,7 @@ type RouterTemplateValues struct {
 	Wss                WSSRouterTemplateValues
 	Forwarder          RouterForwarderTemplateValues
 	Listener           RouterListenerTemplateValues
+	IsHA               bool
 }
 
 type EdgeRouterTemplateValues struct {

ziti/cmd/create/create_config_router.go

@@ -38,6 +38,7 @@ type CreateConfigRouterOptions struct {
 	IsPrivate    bool
 	TunnelerMode string
 	LanInterface string
+	IsHA         bool
 }
 
 type NewCreateConfigRouterCmd struct {

ziti/cmd/create/create_config_router_edge.go

@@ -78,6 +78,7 @@ func NewCmdCreateConfigRouterEdge(routerOptions *CreateConfigRouterOptions, data
 			data.Router.Edge.LanInterface = routerOptions.LanInterface
 			data.Router.Edge.Resolver = cmdhelper.GetZitiEdgeRouterResolver()
 			data.Router.Edge.DnsSvcIpRange = cmdhelper.GetZitiEdgeRouterDnsSvcIpRange()
+			data.Router.IsHA = routerOptions.IsHA
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			routerOptions.Cmd = cmd