Skip to content

Commit

Permalink
Allow to omit non ready backends (#70)
Browse files Browse the repository at this point in the history
* Allow users to choose if they want to include or exclude not ready backend pods from VCL.

Signed-off-by: Tomash Sidei <[email protected]>

Co-authored-by: Craig Ingram <[email protected]>
  • Loading branch information
tomashibm and Craig Ingram authored May 19, 2022
1 parent e69f4f7 commit 490ed1c
Show file tree
Hide file tree
Showing 8 changed files with 46 additions and 6 deletions.
1 change: 1 addition & 0 deletions api/v1alpha1/varnishcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ type VarnishClusterBackend struct {
// +kubebuilder:validation:Required
Port *intstr.IntOrString `json:"port,omitempty"`
Namespaces []string `json:"namespaces,omitempty"`
OnlyReady bool `json:"onlyReady,omitempty"`
ZoneBalancing *VarnishClusterBackendZoneBalancing `json:"zoneBalancing,omitempty"`
}

Expand Down
2 changes: 2 additions & 0 deletions config/crd/bases/caching.ibm.com_varnishclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,8 @@ spec:
items:
type: string
type: array
onlyReady:
type: boolean
port:
anyOf:
- type: integer
Expand Down
1 change: 1 addition & 0 deletions docs/varnish-cluster-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|
| `affinity ` | [Affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) settings for the pods. It allows you to configure onto which nodes Varnish pods should prefer being scheduled. | `optional` |
| `backend.namespaces ` | Namespace(s) to look for backend pods. By default - namespace the VarnishCluster is deployed to. | `required` |
| `backend.onlyReady ` | Include (`false`, by default) or exclude (`true`) backend pods from the VCL (.Backends template var). Alters `.Backends` template variable based on Kubernetes health checks (by default not ready pods are also included in VCL) instead of [Varnish health probes](https://varnish-cache.org/docs/6.6/reference/vcl-probe.html#backend-health-probes). | `optional` |
| `backend.port ` | The port of the backend pods being cached by Varnish. Can be port name or port number. | `required` |
| `backend.selector ` | The selector used to identify the backend Pods. | `required` |
| `backend.zoneBalancing ` | Controls Varnish backend topology aware routing which can assign weights to backends according to their geographical location. | `optional` |
Expand Down
8 changes: 7 additions & 1 deletion docs/vcl-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,13 @@ backend nginx-backend-6f4c6cbc6c-ckqmv {
}
```

The `.Backends` array includes all backend nodes, no matter if the instance is ready or not. For that reason, it is strongly recommended to set [health checks](https://varnish-cache.org/docs/3.0/tutorial/advanced_backend_servers.html#health-checks) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends.
The `.Backends` array can include both all backend nodes (default behavior) and only ready nodes. It depends on `.spec.backends.onlyReady` field.
It depends on which readiness checks you want to respect:
* Kubernetes readiness checks - field set to `true`. In this case the `.Backends` template var in VCL will include only ready backend pods.
Keep in mind that every backend state change will trigger a Varnish VCL config reload.
* Varnish readiness probes - field set to `false` or omitted. This way the `.Backends` array will include all scheduled pods, so it is strongly recommended to set [health checks](https://varnish-cache.org/docs/6.6/reference/vcl-probe.html#backend-health-probes) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends.

You can also combine approaches and use both Kubernetes and Varnish health probes.

### Using User Defined VCL Code Versions

Expand Down
17 changes: 12 additions & 5 deletions pkg/varnishcontroller/controller/endpoints.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"fmt"
"sort"

"github.com/ibm/varnish-operator/pkg/varnishcontroller/podutil"

"github.com/ibm/varnish-operator/pkg/varnishcontroller/events"

"github.com/ibm/varnish-operator/api/v1alpha1"
Expand Down Expand Up @@ -34,12 +36,13 @@ func (r *ReconcileVarnish) getBackendEndpoints(ctx context.Context, vc *v1alpha1
actualLocalWeight := 1.0
actualRemoteWeight := 1.0

namespaces := []string{r.config.Namespace}
ns := []string{r.config.Namespace}
if len(vc.Spec.Backend.Namespaces) > 0 {
namespaces = vc.Spec.Backend.Namespaces
ns = vc.Spec.Backend.Namespaces
}

backendList, portNumber, err := r.getPodsInfo(ctx, vc, namespaces, labels.SelectorFromSet(vc.Spec.Backend.Selector), *vc.Spec.Backend.Port)
selector := labels.SelectorFromSet(vc.Spec.Backend.Selector)
backendList, portNumber, err := r.getPodsInfo(ctx, vc, ns, selector, *vc.Spec.Backend.Port, vc.Spec.Backend.OnlyReady)
if err != nil {
return nil, 0, 0, 0, errors.WithStack(err)
}
Expand Down Expand Up @@ -105,15 +108,15 @@ func (r *ReconcileVarnish) getVarnishEndpoints(ctx context.Context, vc *v1alpha1
varnishLables := labels.SelectorFromSet(vclabels.CombinedComponentLabels(vc, v1alpha1.VarnishComponentVarnish))
varnishPort := intstr.FromString(v1alpha1.VarnishPortName)

varnishEndpoints, _, err := r.getPodsInfo(ctx, vc, []string{r.config.Namespace}, varnishLables, varnishPort)
varnishEndpoints, _, err := r.getPodsInfo(ctx, vc, []string{r.config.Namespace}, varnishLables, varnishPort, false)
if err != nil {
return nil, errors.WithStack(err)
}

return varnishEndpoints, nil
}

func (r *ReconcileVarnish) getPodsInfo(ctx context.Context, vc *v1alpha1.VarnishCluster, namespaces []string, labels labels.Selector, validPort intstr.IntOrString) ([]PodInfo, int32, error) {
func (r *ReconcileVarnish) getPodsInfo(ctx context.Context, vc *v1alpha1.VarnishCluster, namespaces []string, labels labels.Selector, validPort intstr.IntOrString, onlyReady bool) ([]PodInfo, int32, error) {
var pods []v1.Pod
for _, namespace := range namespaces {
listOptions := []client.ListOption{
Expand Down Expand Up @@ -142,6 +145,10 @@ func (r *ReconcileVarnish) getPodsInfo(ctx context.Context, vc *v1alpha1.Varnish
continue
}

if onlyReady && !podutil.PodReady(pod) {
continue
}

portFound := false
for _, container := range pod.Spec.Containers {
for _, containerPort := range container.Ports {
Expand Down
16 changes: 16 additions & 0 deletions pkg/varnishcontroller/podutil/podutil.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package podutil

import v1 "k8s.io/api/core/v1"

func PodReady(pod v1.Pod) bool {
if len(pod.Status.ContainerStatuses) == 0 {
return false
}
for _, status := range pod.Status.ContainerStatuses {
if !status.Ready {
return false
}
}

return true
}
5 changes: 5 additions & 0 deletions pkg/varnishcontroller/predicates/label_matcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package predicates

import (
"github.com/ibm/varnish-operator/pkg/logger"
"github.com/ibm/varnish-operator/pkg/varnishcontroller/podutil"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/event"
Expand Down Expand Up @@ -55,6 +56,10 @@ func (p *LabelMatcherPredicate) Update(e event.UpdateEvent) bool {
return true
}

if podutil.PodReady(*newPod) != podutil.PodReady(*oldPod) {
return true
}

return false
}

Expand Down
2 changes: 2 additions & 0 deletions varnish-operator/crds/varnishcluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -871,6 +871,8 @@ spec:
items:
type: string
type: array
onlyReady:
type: boolean
port:
anyOf:
- type: integer
Expand Down

0 comments on commit 490ed1c

Please sign in to comment.