Status Propagation (Part 2)

Make it so the reconciler loops until everything is healthy, and thus
requires roll up of individual machine statuses.

go.mod

@@ -7,9 +7,10 @@ require (
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/oapi-codegen/runtime v1.1.1
 	github.com/spf13/pflag v1.0.5
+	github.com/spjmurray/go-util v0.1.3
 	github.com/unikorn-cloud/core v0.1.85
 	github.com/unikorn-cloud/identity v0.2.45
-	github.com/unikorn-cloud/region v0.1.47-rc2
+	github.com/unikorn-cloud/region v0.1.47-rc4
 	go.opentelemetry.io/otel/sdk v1.31.0
 	k8s.io/api v0.31.1
 	k8s.io/apimachinery v0.31.1
@@ -61,7 +62,6 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.60.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/spjmurray/go-util v0.1.3 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/otel v1.31.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect

go.sum

@@ -131,16 +131,16 @@ github.com/spjmurray/go-util v0.1.3/go.mod h1:fARcBeaHio/6h9H7Ht+egZPBZMNxEwxmHC
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/unikorn-cloud/core v0.1.85 h1:S4B0nr0jhxF8SCsKyCRVwcx8+kJsI8fQVONLJDf9aic=
 github.com/unikorn-cloud/core v0.1.85/go.mod h1:wEKzCwAnIyTbo27l++Wl+gK95TAxMsFS3y3jbFB03aw=
 github.com/unikorn-cloud/identity v0.2.45 h1:YFmw3uunwdZMuYiimv4lfU6s7iTtYoc22xJqLTrlOVg=
 github.com/unikorn-cloud/identity v0.2.45/go.mod h1:WzNNv05ReDMLfWsOtq53uzhX2GU2nXDw76Bdsf3BPnM=
-github.com/unikorn-cloud/region v0.1.47-rc2 h1:eIwFHSoMojjg7MSFirDuNEnaNSWCygtKQRdNhqFIqzw=
-github.com/unikorn-cloud/region v0.1.47-rc2/go.mod h1:cofPuJjseRKE95xAqNa23nSL7/+LDXOBxNIHwwODpys=
+github.com/unikorn-cloud/region v0.1.47-rc4 h1:rS5vyND9vqMCib5Vd+IktHB4+erRp/+oWzHO20quqsI=
+github.com/unikorn-cloud/region v0.1.47-rc4/go.mod h1:cofPuJjseRKE95xAqNa23nSL7/+LDXOBxNIHwwODpys=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/provisioners/managers/cluster/provisioner.go

@@ -21,6 +21,8 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"slices"
+	"strings"
 
 	"github.com/spf13/pflag"
 
@@ -83,6 +85,9 @@ type Provisioner struct {
 
 	// options are documented for the type.
 	options *Options
+
+	// resourceProvisioning tells whether any sub resource is in a provisioning state.
+	resourceProvisioning bool
 }
 
 // New returns a new initialized provisioner object.
@@ -269,11 +274,23 @@ func (p *Provisioner) Provision(ctx context.Context) error {
 		}
 	}
 
-	// TODO: if any of the machine statuses are not provisioned, we should return
-	// the correct error status here or a yield to force reconcilliation until
-	// the desired state.
-	// TODO: we need to sort the status stuff so it doesn't move around all the
-	// time for determinism on both the CLI and UI.
+	// Sort the statuses so they have a deterministic order up the stack, especially
+	// to things like the UI.
+	slices.SortFunc(p.cluster.Status.WorkloadPools, func(a, b unikornv1.WorkloadPoolStatus) int {
+		return strings.Compare(a.Name, b.Name)
+	})
+
+	for _, pool := range p.cluster.Status.WorkloadPools {
+		slices.SortFunc(pool.Machines, func(a, b unikornv1.MachineStatus) int {
+			return strings.Compare(a.Hostname, b.Hostname)
+		})
+	}
+
+	// Now once the UX is sorted, roll up any non-healthy statuses to the top level.
+	if p.resourceProvisioning {
+		return provisioners.ErrYield
+	}
+
 	return nil
 }