Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support to track the chunks by local objects for kubernetes workflow. #1606

Merged
merged 3 commits into from
Oct 31, 2023
Merged

Support to track the chunks by local objects for kubernetes workflow. #1606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19510f9
Support to track the chunks by local objects for kubernetes workflow.
dashanji Oct 25, 2023
4ff4c11
Revert the GraphAr module commit.
dashanji Oct 25, 2023
15f4b73
Bump up kubernetes version to 1.25.11 in the e2e test.
dashanji Oct 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 34 additions & 3 deletions k8s/pkg/schedulers/scheduling_strategy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,18 @@ func (b *BestEffortStrategy) TrackingChunksByCRD() *BestEffortStrategy {
_ = multierr.Append(errList, err)
}

// if there is no local objects, return error
if len(localObjects) == 0 && len(globalObjects) != 0 {
_ = multierr.Append(errList, errors.Errorf("No local chunks found"))
if len(localObjects) == 0 {
if len(globalObjects) == 0 {
localObjects, err = b.GetLocalObjectsByID(b.required)
// if there is no local objects, return error
if err != nil {
_ = multierr.Append(errList, err)
}
} else {
// if there is no local objects, return error
_ = multierr.Append(errList, errors.Errorf("Failed to get local objects"))
}

}

if errList != nil {
Expand Down Expand Up @@ -335,6 +344,28 @@ func (b *BestEffortStrategy) GetGlobalObjectsByID(
return objects, nil
}

// GetLocalObjectsByID returns the local objects by the given jobname.
func (b *BestEffortStrategy) GetLocalObjectsByID(
jobNames []string,
) ([]*v1alpha1.LocalObject, error) {
requiredJobs := make(map[string]bool)
for _, n := range jobNames {
requiredJobs[n] = true
}
objects := []*v1alpha1.LocalObject{}
localObjects := &v1alpha1.LocalObjectList{}
if err := b.List(context.TODO(), localObjects); err != nil {
return nil, err
}
for i, obj := range localObjects.Items {
if jobname, exist := obj.Labels[labels.VineyardObjectJobLabel]; exist && requiredJobs[jobname] {
objects = append(objects, &localObjects.Items[i])
}
}

return objects, nil
}

// CreateConfigmapForID creates a configmap for the object id and the nodes.
func (b *BestEffortStrategy) CreateConfigmapForID(
jobname []string,
Expand Down
8 changes: 4 additions & 4 deletions k8s/test/e2e/kind-with-local-registry.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ containerdConfigPatches:
endpoint = ["http://kind-registry:5000"]
nodes:
- role: control-plane
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
8 changes: 4 additions & 4 deletions k8s/test/e2e/kind.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
- role: worker
image: kindest/node:v1.24.0
image: kindest/node:v1.25.11
Loading