diff --git a/Makefile b/Makefile index 0460af0c..40ac4db1 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,14 @@ gofumpt := mvdan.cc/gofumpt@v0.5.0 gosimports := github.com/rinchsan/gosimports/cmd/gosimports@v0.3.8 golangci_lint := github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2 -%/main.wasm: %/main.go +examples/advanced/main.wasm: examples/advanced/main.go @(cd $(@D); tinygo build -o main.wasm -gc=custom -tags=custommalloc -scheduler=none --no-debug -target=wasi .) +%/main.wasm: %/main.go + @(cd $(@D); tinygo build -o main.wasm -scheduler=none --no-debug -target=wasi .) + .PHONY: build-tinygo -build-tinygo: examples/nodenumber/main.wasm guest/testdata/all/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm +build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm guest/testdata/all/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm %/main-debug.wasm: %/main.go @(cd $(@D); tinygo build -o main-debug.wasm -gc=custom -tags=custommalloc -scheduler=none -target=wasi .) @@ -16,7 +19,7 @@ build-tinygo: examples/nodenumber/main.wasm guest/testdata/all/main.wasm guest/t .PHONY: test-guest test-guest: guest/.tinygo-target.json @(cd guest; tinygo test -v -target .tinygo-target.json ./...) - @(cd examples; tinygo test -v -target ../guest/.tinygo-target.json ./nodenumber/plugin/...) + @(cd examples/advanced; tinygo test -v -target ../../guest/.tinygo-target.json ./plugin/...) # Benchmarking the guest code means running it with TinyGo, which internally # compiles the benchmarks to a wasm binary, then runs it with wazero. @@ -43,7 +46,7 @@ testdata: @$(MAKE) build-wat .PHONY: profile -profile: examples/nodenumber/main-debug.wasm +profile: examples/advanced/main-debug.wasm @cd ./internal/e2e; \ go run ./profiler/profiler.go ../../$^; \ go tool pprof -text cpu.pprof; \ @@ -117,7 +120,7 @@ format: # all_mods are the go modules including examples. Examples should also be # formatted, lint checked, etc. even if they are are built with TinyGo. -all_mods := ./internal/e2e/go.mod ./scheduler/go.mod ./guest/go.mod ./guest/testdata/go.mod ./kubernetes/proto/go.mod ./examples/go.mod +all_mods := ./internal/e2e/go.mod ./scheduler/go.mod ./guest/go.mod ./guest/testdata/go.mod ./kubernetes/proto/go.mod ./examples/go.mod ./examples/advanced/go.mod all_nottinygo := ./examples/go.mod ./guest/testdata/go.mod .PHONY: tidy @@ -139,7 +142,8 @@ build: test: @(cd scheduler; go test -v ./...) @(cd guest; go test -v ./...) - @(cd examples; go test -v ./nodenumber/plugin/...) + @(cd examples; go test -v ./...) + @(cd examples/advanced; go test -v ./plugin/...) @(cd internal/e2e; go test -v ./...) .PHONY: check # Pre-flight check for pull requests diff --git a/examples/advanced/README.md b/examples/advanced/README.md new file mode 100644 index 00000000..43823433 --- /dev/null +++ b/examples/advanced/README.md @@ -0,0 +1,18 @@ +# Advanced NodeNumber Plugin + +This is a WebAssembly port of the [Scheduler Simulator NodeNumber plugin][1]. + +This variant is more complicated than the [simple plugin][2] to program, but +more efficient and testable. + +* This manually configures lifecycle hooks to avoid no-op overhead. +* This uses the more efficient [nottinygc][3] garbage collector. +* The `plugin` package can be tested both with `go test` and + `tinygo test -target=wasi`. +* See [RATIONALE.md](../../guest/RATIONALE.md) for more notes on performance. + +[1]: https://github.com/kubernetes-sigs/kube-scheduler-simulator/blob/simulator/v0.1.0/simulator/docs/sample/nodenumber/plugin.go + +[2]: ../nodenumber + +[3]: https://github.com/wasilibs/nottinygc \ No newline at end of file diff --git a/examples/advanced/go.mod b/examples/advanced/go.mod new file mode 100644 index 00000000..0094371a --- /dev/null +++ b/examples/advanced/go.mod @@ -0,0 +1,18 @@ +module sigs.k8s.io/kube-scheduler-wasm-extension/examples/advanced + +go 1.20 + +require ( + github.com/wasilibs/nottinygc v0.4.0 + sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000 + sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/magefile/mage v1.14.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect +) + +replace sigs.k8s.io/kube-scheduler-wasm-extension/guest => ./../../guest + +replace sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ./../../kubernetes/proto diff --git a/examples/advanced/go.sum b/examples/advanced/go.sum new file mode 100644 index 00000000..f69bfb90 --- /dev/null +++ b/examples/advanced/go.sum @@ -0,0 +1,11 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= +github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/examples/advanced/main.go b/examples/advanced/main.go new file mode 100644 index 00000000..9b421139 --- /dev/null +++ b/examples/advanced/main.go @@ -0,0 +1,53 @@ +//go:build tinygo.wasm + +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package main is the entrypoint of the %.wasm file, compiled with +// 'tinygo build -target=wasi'. See /guest/RATIONALE.md for details. +package main + +// Override the default GC with a more performant one. +// Note: this requires tinygo flags: -gc=custom -tags=custommalloc +import ( + _ "github.com/wasilibs/nottinygc" + + "sigs.k8s.io/kube-scheduler-wasm-extension/examples/advanced/plugin" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/config" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/prescore" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/score" +) + +// main is compiled to an exported Wasm function named "_start", called by the +// Wasm scheduler plugin during initialization. +func main() { + // The plugin package uses only normal Go code, which allows it to be + // unit testable via `tinygo test -target=wasi` as well normal `go test`. + // + // The real implementations, such as `config.Get()` use Wasm host functions + // (go:wasmimport), which cannot be tested with `tinygo test -target=wasi`. + plugin, err := plugin.New(config.Get()) + if err != nil { + panic(err) + } + // Instead of using `plugin.Set`, this configures only the interfaces + // implemented by the plugin. The Wasm host only calls functions imported, + // so this prevents additional overhead. + enqueue.SetPlugin(plugin) + prescore.SetPlugin(plugin) + score.SetPlugin(plugin) +} diff --git a/examples/advanced/main.wasm b/examples/advanced/main.wasm new file mode 100755 index 00000000..d04bcd77 Binary files /dev/null and b/examples/advanced/main.wasm differ diff --git a/examples/nodenumber/plugin/plugin.go b/examples/advanced/plugin/plugin.go similarity index 100% rename from examples/nodenumber/plugin/plugin.go rename to examples/advanced/plugin/plugin.go diff --git a/examples/nodenumber/plugin/plugin_test.go b/examples/advanced/plugin/plugin_test.go similarity index 100% rename from examples/nodenumber/plugin/plugin_test.go rename to examples/advanced/plugin/plugin_test.go diff --git a/examples/go.mod b/examples/go.mod index 5ea8d53a..6fc897be 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -3,15 +3,11 @@ module sigs.k8s.io/kube-scheduler-wasm-extension/examples go 1.20 require ( - github.com/wasilibs/nottinygc v0.4.0 sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000 sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 ) -require ( - github.com/magefile/mage v1.14.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect -) +require google.golang.org/protobuf v1.30.0 // indirect replace sigs.k8s.io/kube-scheduler-wasm-extension/guest => ./../guest diff --git a/examples/go.sum b/examples/go.sum index f69bfb90..f0e0891f 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -1,10 +1,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= -github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= diff --git a/examples/nodenumber/README.md b/examples/nodenumber/README.md new file mode 100644 index 00000000..c157af17 --- /dev/null +++ b/examples/nodenumber/README.md @@ -0,0 +1,13 @@ +# NodeNumber Plugin + +This is a WebAssembly port of the [Scheduler Simulator NodeNumber plugin][1]. + +## Performance + +This example was made to be simple to program and test with Go, but there are +some tradeoffs: This cannot be tested with TinyGo, and has higher runtime +overhead than more [advanced][2] approaches. + +[1]: https://github.com/kubernetes-sigs/kube-scheduler-simulator/blob/simulator/v0.1.0/simulator/docs/sample/nodenumber/plugin.go + +[2]: ../advanced diff --git a/examples/nodenumber/main.go b/examples/nodenumber/main.go index e90dd802..f46bd138 100644 --- a/examples/nodenumber/main.go +++ b/examples/nodenumber/main.go @@ -1,5 +1,3 @@ -//go:build tinygo.wasm - /* Copyright 2023 The Kubernetes Authors. @@ -20,28 +18,109 @@ // '-target=wasi'. See /guest/RATIONALE.md for details. package main -// Override the default GC with a more performant one. -// Note: this requires tinygo flags: -gc=custom -tags=custommalloc import ( - _ "github.com/wasilibs/nottinygc" + "encoding/json" + "fmt" - "sigs.k8s.io/kube-scheduler-wasm-extension/examples/nodenumber/plugin" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/config" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/prescore" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/score" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/plugin" ) // main is compiled to a WebAssembly function named "_start", called by the // wasm scheduler plugin during initialization. func main() { - plugin, err := plugin.New(config.Get()) - if err != nil { - panic(err) - } - // Below is like `var _ api.EnqueueExtensions = plugin`, except it also - // wires up functions the host should provide (go:wasmimport). - enqueue.SetPlugin(plugin) - prescore.SetPlugin(plugin) - score.SetPlugin(plugin) + var args nodeNumberArgs + if jsonConfig := config.Get(); jsonConfig != nil { + if err := json.Unmarshal(jsonConfig, &args); err != nil { + panic(fmt.Errorf("decode arg into NodeNumberArgs: %w", err)) + } + } + plugin.Set(&NodeNumber{reverse: args.Reverse}) +} + +// NodeNumber is an example plugin that favors nodes that share a numerical +// suffix with the pod name. +// +// For example, when a pod named "Pod1" is scheduled, a node named "Node1" gets +// a higher score than a node named "Node9". +// +// # Notes +// +// - Only the last character in names are considered. This means "Node99" is +// treated the same as "Node9" +// - The reverse field inverts the score. For example, when `reverse == true` +// a numeric match gets a results in a lower score than a match. +type NodeNumber struct { + reverse bool +} + +type nodeNumberArgs struct { + Reverse bool `json:"reverse"` +} + +const ( + // Name is the name of the plugin used in the plugin registry and configurations. + Name = "NodeNumber" + preScoreStateKey = "PreScore" + Name +) + +// preScoreState computed at PreScore and used at Score. +type preScoreState struct { + podSuffixNumber uint8 +} + +// EventsToRegister implements api.EnqueueExtensions +func (pl *NodeNumber) EventsToRegister() []api.ClusterEvent { + return []api.ClusterEvent{ + {Resource: api.Node, ActionType: api.Add}, + } +} + +// PreScore implements api.PreScorePlugin +func (pl *NodeNumber) PreScore(state api.CycleState, pod proto.Pod, _ proto.NodeList) *api.Status { + podnum, ok := lastNumber(pod.Spec().GetNodeName()) + if !ok { + return nil // return success even if its suffix is non-number. + } + + state.Write(preScoreStateKey, &preScoreState{podSuffixNumber: podnum}) + return nil +} + +// Score implements api.ScorePlugin +func (pl *NodeNumber) Score(state api.CycleState, _ proto.Pod, nodeName string) (int32, *api.Status) { + var match bool + if data, ok := state.Read(preScoreStateKey); ok { + // Match is when there is a last digit, and it is the pod suffix. + nodenum, ok := lastNumber(nodeName) + match = ok && data.(*preScoreState).podSuffixNumber == nodenum + } else { + // Match is also when there is no pod spec node name. + match = true + } + + if pl.reverse { + match = !match // invert the condition. + } + + if match { + return 10, nil + } + return 0, nil +} + +// lastNumber returns the last number in the string or false. +func lastNumber(str string) (uint8, bool) { + if len(str) == 0 { + return 0, false + } + + // We have at least a single character name. See if the last is a digit. + lastChar := str[len(str)-1] + if '0' <= lastChar && lastChar <= '9' { + return lastChar - '0', true + } + return 0, false } diff --git a/examples/nodenumber/main.wasm b/examples/nodenumber/main.wasm index 83fcb31e..8353ee83 100755 Binary files a/examples/nodenumber/main.wasm and b/examples/nodenumber/main.wasm differ diff --git a/examples/nodenumber/main_test.go b/examples/nodenumber/main_test.go new file mode 100644 index 00000000..51585a31 --- /dev/null +++ b/examples/nodenumber/main_test.go @@ -0,0 +1,153 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" + protoapi "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api" +) + +func Test_NodeNumber(t *testing.T) { + tests := []struct { + name string + pod proto.Pod + nodeName string + expectedMatch bool + }{ + {name: "nil,empty", pod: &testPod{}, expectedMatch: true}, + {name: "empty,empty", pod: &testPod{}, nodeName: "", expectedMatch: true}, + {name: "empty,letter", pod: &testPod{}, nodeName: "a", expectedMatch: true}, + {name: "empty,digit", pod: &testPod{}, nodeName: "1", expectedMatch: true}, + {name: "letter,letter", pod: &testPod{nodeName: "a"}, nodeName: "a", expectedMatch: true}, + {name: "letter,digit", pod: &testPod{nodeName: "a"}, nodeName: "1", expectedMatch: true}, + {name: "digit,letter", pod: &testPod{nodeName: "1"}, nodeName: "a", expectedMatch: false}, + {name: "digit,digit", pod: &testPod{nodeName: "1"}, nodeName: "1", expectedMatch: true}, + {name: "digit,different digit", pod: &testPod{nodeName: "1"}, nodeName: "2", expectedMatch: false}, + } + + for _, reverse := range []bool{false, true} { + for _, tc := range tests { + name := tc.name + expectedMatch := tc.expectedMatch + if reverse { + name += ",reverse" + expectedMatch = !expectedMatch + } + t.Run(name, func(t *testing.T) { + plugin := &NodeNumber{reverse: reverse} + state := testCycleState{} + + status := plugin.PreScore(state, tc.pod, nil) + if status != nil { + t.Fatalf("unexpected status: %v", status) + } + + score, status := plugin.Score(state, nil, tc.nodeName) + if status != nil { + t.Fatalf("unexpected status: %v", status) + } + + if expectedMatch { + if want, have := int32(10), score; want != have { + t.Fatalf("unexpected score: %v != %v", want, have) + } + } else { + if want, have := int32(0), score; want != have { + t.Fatalf("unexpected score: %v != %v", want, have) + } + } + }) + } + } +} + +func Test_lastNumber(t *testing.T) { + tests := []struct { + name string + input string + expectedDigit uint8 + expectedOk bool + }{ + {name: "empty", input: ""}, + {name: "not digit", input: "a"}, + {name: "unicode", input: "ó"}, + {name: "middle digit", input: "a1a"}, + {name: "digit after letter", input: "a1", expectedDigit: 1, expectedOk: true}, + {name: "digit after digit", input: "12", expectedDigit: 2, expectedOk: true}, + {name: "digit after unicode", input: "ó2", expectedDigit: 2, expectedOk: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d, ok := lastNumber(tc.input) + if want, have := tc.expectedDigit, d; want != have { + t.Fatalf("unexpected digit: %v != %v", want, have) + } + if want, have := tc.expectedOk, ok; want != have { + t.Fatalf("unexpected ok: %v != %v", want, have) + } + }) + } +} + +var _ api.CycleState = testCycleState{} + +type testCycleState map[string]any + +func (c testCycleState) Read(key string) (val any, ok bool) { + val, ok = c[key] + return +} + +func (c testCycleState) Write(key string, val any) { + c[key] = val +} + +func (c testCycleState) Delete(key string) { + delete(c, key) +} + +var _ proto.Pod = &testPod{} + +// testPod is test data just to set the nodeName +type testPod struct { + nodeName string +} + +func (t testPod) GetUid() string { + return "" +} + +func (t testPod) GetName() string { + return "" +} + +func (t testPod) GetNamespace() string { + return "" +} + +func (t testPod) Spec() *protoapi.PodSpec { + nodeName := t.nodeName + return &protoapi.PodSpec{NodeName: &nodeName} +} + +func (t testPod) Status() *protoapi.PodStatus { + return nil +} diff --git a/guest/enqueue/enqueue.go b/guest/enqueue/enqueue.go index 0e57b75d..646caf62 100644 --- a/guest/enqueue/enqueue.go +++ b/guest/enqueue/enqueue.go @@ -45,10 +45,10 @@ var _ func() = _enqueue // //export enqueue func _enqueue() { - if enqueue == nil { - // If we got here, someone imported the package, but forgot to set the - // filter. Panic with what's wrong. - panic("enqueue imported, but enqueue.SetPlugin not called") + if enqueue == nil { // Then, the user didn't define one. + // This is likely caused by use of plugin.Set(p), where 'p' didn't + // implement EnqueueExtensions: return to use default events. + return } clusterEvents := enqueue.EventsToRegister() diff --git a/guest/filter/filter.go b/guest/filter/filter.go index 6f1d6211..635bc65a 100644 --- a/guest/filter/filter.go +++ b/guest/filter/filter.go @@ -61,10 +61,10 @@ var _ func() uint32 = _filter // //export filter func _filter() uint32 { //nolint - if filter == nil { - // If we got here, someone imported the package, but forgot to set the - // filter. Panic with what's wrong. - panic("filter imported, but filter.SetPlugin not called") + if filter == nil { // Then, the user didn't define one. + // This is likely caused by use of plugin.Set(p), where 'p' didn't + // implement FilterPlugin: return success. + return 0 } s := filter.Filter(cyclestate.Values, cyclestate.Pod, &nodeInfo{}) diff --git a/guest/plugin/plugin.go b/guest/plugin/plugin.go new file mode 100644 index 00000000..15c668b7 --- /dev/null +++ b/guest/plugin/plugin.go @@ -0,0 +1,60 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package plugin + +import ( + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/filter" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/prefilter" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/prescore" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/score" +) + +// Set is a convenience to assign lifecycle hooks based on which +// interfaces `plugin` defines. +// +// func main() { +// plugin.Set(myPlugin{}) +// } +// +// Note: Using this results in the host call this plugin for every hook, even +// when it isn't implemented. For more control and performance, set each hook +// individually: +// +// func main() { +// plugin := myPlugin{} +// prefilter.SetPlugin(plugin) +// filter.SetPlugin(plugin) +// } +func Set(plugin api.Plugin) { + if plugin, ok := plugin.(api.EnqueueExtensions); ok { + enqueue.SetPlugin(plugin) + } + if plugin, ok := plugin.(api.PreFilterPlugin); ok { + prefilter.SetPlugin(plugin) + } + if plugin, ok := plugin.(api.FilterPlugin); ok { + filter.SetPlugin(plugin) + } + if plugin, ok := plugin.(api.PreScorePlugin); ok { + prescore.SetPlugin(plugin) + } + if plugin, ok := plugin.(api.ScorePlugin); ok { + score.SetPlugin(plugin) + } +} diff --git a/guest/prescore/prescore.go b/guest/prescore/prescore.go index 5ffa4bd1..46873265 100644 --- a/guest/prescore/prescore.go +++ b/guest/prescore/prescore.go @@ -72,10 +72,10 @@ var _ func() uint32 = _prescore // //export prescore func _prescore() uint32 { - if prescore == nil { - // If we got here, someone imported the package, but forgot to set the - // filter. Panic with what's wrong. - panic("prescore imported, but prescore.SetPlugin not called") + if prescore == nil { // Then, the user didn't define one. + // This is likely caused by use of plugin.Set(p), where 'p' didn't + // implement PreScorePlugin: return success. + return 0 } // Pod is lazy and the same value for all plugins in a scheduling cycle. diff --git a/guest/score/score.go b/guest/score/score.go index ed920112..2564ca37 100644 --- a/guest/score/score.go +++ b/guest/score/score.go @@ -58,10 +58,10 @@ var _ func() uint64 = _score // //export score func _score() uint64 { - if score == nil { - // If we got here, someone imported the package, but forgot to set the - // filter. Panic with what's wrong. - panic("score imported, but score.SetPlugin not called") + if score == nil { // Then, the user didn't define one. + // This is likely caused by use of plugin.Set(p), where 'p' didn't + // implement ScorePlugin: return success and score zero. + return 0 } // Pod is lazy and the same value for all plugins in a scheduling cycle. diff --git a/guest/testdata/all/main.go b/guest/testdata/all/main.go index 2c9915c8..065060f5 100644 --- a/guest/testdata/all/main.go +++ b/guest/testdata/all/main.go @@ -16,13 +16,9 @@ package main -// Override the default GC with a more performant one. -// Note: this requires tinygo flags: -gc=custom -tags=custommalloc import ( "os" - _ "github.com/wasilibs/nottinygc" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" diff --git a/guest/testdata/all/main.wasm b/guest/testdata/all/main.wasm index 33689456..7de9924e 100755 Binary files a/guest/testdata/all/main.wasm and b/guest/testdata/all/main.wasm differ diff --git a/guest/testdata/cyclestate/main.go b/guest/testdata/cyclestate/main.go index f1a1998a..4b4bbf80 100644 --- a/guest/testdata/cyclestate/main.go +++ b/guest/testdata/cyclestate/main.go @@ -18,14 +18,10 @@ // See https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/#extension-points package main -// Override the default GC with a more performant one. -// Note: this requires tinygo flags: -gc=custom -tags=custommalloc import ( "os" "unsafe" - _ "github.com/wasilibs/nottinygc" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" diff --git a/guest/testdata/cyclestate/main.wasm b/guest/testdata/cyclestate/main.wasm index bdac6326..b9e1c031 100755 Binary files a/guest/testdata/cyclestate/main.wasm and b/guest/testdata/cyclestate/main.wasm differ diff --git a/guest/testdata/filter/main.go b/guest/testdata/filter/main.go index f9d1e6e3..05a31010 100644 --- a/guest/testdata/filter/main.go +++ b/guest/testdata/filter/main.go @@ -16,13 +16,9 @@ package main -// Override the default GC with a more performant one. -// Note: this requires tinygo flags: -gc=custom -tags=custommalloc import ( "os" - _ "github.com/wasilibs/nottinygc" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/filter" diff --git a/guest/testdata/filter/main.wasm b/guest/testdata/filter/main.wasm index f8aee622..17741fda 100755 Binary files a/guest/testdata/filter/main.wasm and b/guest/testdata/filter/main.wasm differ diff --git a/guest/testdata/go.mod b/guest/testdata/go.mod index 49985c36..b8f77cb7 100644 --- a/guest/testdata/go.mod +++ b/guest/testdata/go.mod @@ -3,15 +3,11 @@ module sigs.k8s.io/kube-scheduler-wasm-extension/guest/testdata go 1.20 require ( - github.com/wasilibs/nottinygc v0.3.0 sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000 sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 ) -require ( - github.com/magefile/mage v1.14.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect -) +require google.golang.org/protobuf v1.30.0 // indirect replace sigs.k8s.io/kube-scheduler-wasm-extension/guest => ./../ diff --git a/guest/testdata/go.sum b/guest/testdata/go.sum index 19dd6a24..f0e0891f 100644 --- a/guest/testdata/go.sum +++ b/guest/testdata/go.sum @@ -1,10 +1,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/wasilibs/nottinygc v0.3.0 h1:0L1jsJ1MsyN5tdinmFbLfuEA0TnHRcqaBM9pDTJVJmU= -github.com/wasilibs/nottinygc v0.3.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= diff --git a/guest/testdata/score/main.go b/guest/testdata/score/main.go index 752aa189..0aee5203 100644 --- a/guest/testdata/score/main.go +++ b/guest/testdata/score/main.go @@ -16,13 +16,9 @@ package main -// Override the default GC with a more performant one. -// Note: this requires tinygo flags: -gc=custom -tags=custommalloc import ( "os" - _ "github.com/wasilibs/nottinygc" - "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/prescore" diff --git a/guest/testdata/score/main.wasm b/guest/testdata/score/main.wasm index 35bd304f..7a884610 100755 Binary files a/guest/testdata/score/main.wasm and b/guest/testdata/score/main.wasm differ diff --git a/internal/e2e/scheduler/scheduler_test.go b/internal/e2e/scheduler/scheduler_test.go index 21081888..c65b9ee5 100644 --- a/internal/e2e/scheduler/scheduler_test.go +++ b/internal/e2e/scheduler/scheduler_test.go @@ -52,8 +52,17 @@ func TestCycleStateCoherence(t *testing.T) { } func TestExample_NodeNumber(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + testExample_NodeNumber(t, false) + }) + t.Run("Advanced", func(t *testing.T) { + testExample_NodeNumber(t, true) + }) +} + +func testExample_NodeNumber(t *testing.T, advanced bool) { ctx := context.Background() - plugin := newNodeNumberPlugin(ctx, t, false) + plugin := newNodeNumberPlugin(ctx, t, advanced, false) defer plugin.(io.Closer).Close() pod := &v1.Pod{Spec: v1.PodSpec{NodeName: "happy8"}} @@ -78,7 +87,7 @@ func TestExample_NodeNumber(t *testing.T) { t.Run("Reverse means score zero on match", func(t *testing.T) { // This proves we can read configuration. - reversed := newNodeNumberPlugin(ctx, t, true) + reversed := newNodeNumberPlugin(ctx, t, advanced, true) defer reversed.(io.Closer).Close() score := e2e.RunAll(ctx, t, reversed, pod, nodeInfoWithName("glad8")) @@ -89,15 +98,26 @@ func TestExample_NodeNumber(t *testing.T) { } func BenchmarkExample_NodeNumber(b *testing.B) { + b.Run("Simple", func(b *testing.B) { + benchmarkExample_NodeNumber(b, false) + }) + b.Run("Advanced", func(b *testing.B) { + benchmarkExample_NodeNumber(b, true) + }) +} + +func benchmarkExample_NodeNumber(b *testing.B, advanced bool) { + b.Helper() ctx := context.Background() + b.Run("New", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - newNodeNumberPlugin(ctx, b, false).(io.Closer).Close() + newNodeNumberPlugin(ctx, b, advanced, false).(io.Closer).Close() } }) - plugin := newNodeNumberPlugin(ctx, b, false) + plugin := newNodeNumberPlugin(ctx, b, advanced, false) defer plugin.(io.Closer).Close() pod := *test.PodReal // copy @@ -114,9 +134,14 @@ func BenchmarkExample_NodeNumber(b *testing.B) { }) } -func newNodeNumberPlugin(ctx context.Context, t e2e.Testing, reverse bool) framework.Plugin { +func newNodeNumberPlugin(ctx context.Context, t e2e.Testing, advanced, reverse bool) framework.Plugin { + t.Helper() + guestURL := test.URLExampleNodeNumber + if advanced { + guestURL = test.URLExampleAdvanced + } plugin, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{ - GuestURL: test.URLExampleNodeNumber, + GuestURL: guestURL, GuestConfig: fmt.Sprintf(`{"reverse": %v}`, reverse), }) if err != nil { diff --git a/scheduler/go.mod b/scheduler/go.mod index bee36856..25bd8f44 100644 --- a/scheduler/go.mod +++ b/scheduler/go.mod @@ -36,7 +36,6 @@ replace ( require ( github.com/google/uuid v1.3.0 - github.com/stretchr/testify v1.8.1 github.com/tetratelabs/wazero v1.3.1 k8s.io/api v0.27.3 k8s.io/apimachinery v0.27.3 @@ -91,7 +90,6 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/selinux v1.10.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/scheduler/plugin/http.go b/scheduler/plugin/http.go deleted file mode 100644 index 6f3b7c96..00000000 --- a/scheduler/plugin/http.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package wasm - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" -) - -// httpClient decorates an http.Client with convenience methods. -type httpClient struct { - c http.Client -} - -// newHTTPClient is a constructor for httpFetcher. -// -// It is possible to plug a custom http.RoundTripper to handle other concerns (e.g. retries) -// Compression is handled transparently and automatically by http.Client. -func newHTTPClient(transport http.RoundTripper) *httpClient { - return &httpClient{ - c: http.Client{Transport: transport}, - } -} - -// get returns a byte slice of the wasm module found at the given URL, or an error otherwise. -func (f *httpClient) get(ctx context.Context, u *url.URL) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - resp, err := f.c.Do(req.WithContext(ctx)) - defer func() { - io.Copy(io.Discard, resp.Body) //nolint - resp.Body.Close() - }() - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("received %v status code from %q", resp.StatusCode, u) - } - - bytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return bytes, nil -} diff --git a/scheduler/plugin/http_test.go b/scheduler/plugin/http_test.go deleted file mode 100644 index e2d56365..00000000 --- a/scheduler/plugin/http_test.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package wasm - -import ( - "compress/gzip" - "context" - "net/http" - "net/http/httptest" - "net/url" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -var wasmMagicNumber = []byte{0x00, 0x61, 0x73, 0x6d} - -func TestWasmHTTPFetch(t *testing.T) { - wasmBinary := wasmMagicNumber - wasmBinary = append(wasmBinary, 0x00, 0x00, 0x00, 0x00) - cases := []struct { - name string - handler http.HandlerFunc - expectedError string - }{ - { - name: "plain wasm binary", - handler: func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(wasmBinary) - }, - }, - // Compressed payloads are handled automatically by http.Client. - { - name: "compressed payload", - handler: func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Encoding", "gzip") - - gw := gzip.NewWriter(w) - defer gw.Close() - _, _ = gw.Write(wasmBinary) - }, - }, - { - name: "http error", - handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - }, - expectedError: "received 500 status code", - }, - } - - for _, proto := range []string{"http", "https"} { - t.Run(proto, func(t *testing.T) { - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - ts := httptest.NewServer(tc.handler) - defer ts.Close() - c := newHTTPClient(http.DefaultTransport) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - parse, err := url.Parse(ts.URL) - require.NoError(t, err) - _, err = c.get(ctx, parse) - if tc.expectedError != "" { - require.ErrorContains(t, err, tc.expectedError) - return - } - require.NoError(t, err, "Wasm download got an unexpected error: %v", err) - }) - } - }) - } -} diff --git a/scheduler/plugin/plugin.go b/scheduler/plugin/plugin.go index b08b63ae..8e9e56db 100644 --- a/scheduler/plugin/plugin.go +++ b/scheduler/plugin/plugin.go @@ -18,10 +18,8 @@ package wasm import ( "context" + "errors" "fmt" - "net/http" - "net/url" - "os" "sync/atomic" "time" @@ -44,7 +42,7 @@ var _ frameworkruntime.PluginFactory = New func New(configuration runtime.Object, frameworkHandle framework.Handle) (framework.Plugin, error) { config := PluginConfig{} if err := frameworkruntime.DecodeInto(configuration, &config); err != nil { - return nil, fmt.Errorf("failed to decode into %s PluginConfig: %w", PluginName, err) + return nil, fmt.Errorf("wasm: failed to decode into PluginConfig: %w", err) } return NewFromConfig(context.Background(), config) @@ -53,9 +51,13 @@ func New(configuration runtime.Object, frameworkHandle framework.Handle) (framew // NewFromConfig is like New, except it allows us to explicitly provide the // context and configuration of the plugin. This allows flexibility in tests. func NewFromConfig(ctx context.Context, config PluginConfig) (framework.Plugin, error) { - guestBin, err := readFromURI(ctx, config.GuestURL) + url := config.GuestURL + if url == "" { + return nil, errors.New("wasm: guestURL is required") + } + guestBin, err := getURL(ctx, url) if err != nil { - return nil, fmt.Errorf("wasm: error reading guest binary at %s: %w", config.GuestURL, err) + return nil, fmt.Errorf("wasm: error reading guestURL %s: %w", url, err) } runtime, guestModule, err := prepareRuntime(ctx, guestBin, config.GuestConfig) @@ -79,22 +81,6 @@ func NewFromConfig(ctx context.Context, config PluginConfig) (framework.Plugin, } } -func readFromURI(ctx context.Context, u string) ([]byte, error) { - uri, err := url.ParseRequestURI(u) - if err != nil { - return nil, err - } - switch uri.Scheme { - case "file": - return os.ReadFile(uri.Path) - case "http", "https": - c := newHTTPClient(http.DefaultTransport) - return c.get(ctx, uri) - default: - return nil, fmt.Errorf("unsupported URL scheme: %s", uri.Scheme) - } -} - // newWasmPlugin is extracted to prevent small bugs: The caller must close the // wazero.Runtime to avoid leaking mmapped files. func newWasmPlugin(ctx context.Context, runtime wazero.Runtime, guestModule wazero.CompiledModule, config PluginConfig) (*wasmPlugin, error) { diff --git a/scheduler/plugin/plugin_test.go b/scheduler/plugin/plugin_test.go index 2ff2ee33..9247c46f 100644 --- a/scheduler/plugin/plugin_test.go +++ b/scheduler/plugin/plugin_test.go @@ -188,7 +188,7 @@ func TestNew_maskInterfaces(t *testing.T) { }, { name: "prescore|score", - guestURL: test.URLExampleNodeNumber, + guestURL: test.URLExampleAdvanced, expectedScore: true, }, { @@ -197,8 +197,8 @@ func TestNew_maskInterfaces(t *testing.T) { expectedScore: true, }, { - name: "prefilter|filter|prescore|score", - guestURL: test.URLTestAllNoopWat, + name: "all", + guestURL: test.URLExampleNodeNumber, expectedFilter: true, expectedScore: true, }, @@ -223,25 +223,35 @@ func TestNew_maskInterfaces(t *testing.T) { t.Fatalf("expecteded BasePlugin %v", p) } if _, ok := p.(wasm.FilterPlugin); tc.expectedFilter != ok { - t.Fatalf("didn't expected FilterPlugin %v", p) + t.Fatalf("unexpected FilterPlugin %v", p) } if _, ok := p.(wasm.ScorePlugin); tc.expectedScore != ok { - t.Fatalf("didn't expected ScorePlugin %v", p) + t.Fatalf("unexpected ScorePlugin %v", p) } if _, ok := p.(wasm.ReservePlugin); tc.expectedReserve != ok { - t.Fatalf("didn't expected ReservePlugin %v", p) + t.Fatalf("unexpected ReservePlugin %v", p) } if _, ok := p.(wasm.PermitPlugin); tc.expectedPermit != ok { - t.Fatalf("didn't expected PermitPlugin %v", p) + t.Fatalf("unexpected PermitPlugin %v", p) } if _, ok := p.(wasm.BindPlugin); tc.expectedBind != ok { - t.Fatalf("didn't expected BindPlugin %v", p) + t.Fatalf("unexpected BindPlugin %v", p) } }) } } func TestNewFromConfig(t *testing.T) { + uri, _ := url.ParseRequestURI(test.URLTestFilter) + bytes, _ := os.ReadFile(uri.Path) + _, file := path.Split(uri.Path) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/"+file { + _, _ = w.Write(bytes) + } + })) + t.Cleanup(ts.Close) + type testcase struct { name string guestURL string @@ -249,9 +259,22 @@ func TestNewFromConfig(t *testing.T) { } tests := []testcase{ { - name: "valid wasm", + name: "file: valid", guestURL: test.URLTestFilter, }, + { + name: "http: valid", + guestURL: ts.URL + "/" + file, + }, + { + name: "missing guestURL", + expectedError: "wasm: guestURL is required", + }, + { + name: "invalid guestURL", + guestURL: "c:\\foo.wasm", + expectedError: "wasm: error reading guestURL c:\\foo.wasm: unsupported URL scheme: c", + }, { name: "not plugin", guestURL: test.URLErrorNotPlugin, @@ -267,7 +290,7 @@ wasm stack trace: }, } - testWithURL := func(t *testing.T, tc testcase) { + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{GuestURL: tc.guestURL}) if err != nil { @@ -282,28 +305,6 @@ wasm stack trace: } }) } - - t.Run("local", func(t *testing.T) { - for _, tc := range tests { - testWithURL(t, tc) - } - }) - - t.Run("remote (http)", func(t *testing.T) { - for _, tc := range tests { - uri, _ := url.ParseRequestURI(tc.guestURL) - bytes, _ := os.ReadFile(uri.Path) - _, file := path.Split(uri.Path) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/"+file { - _, _ = w.Write(bytes) - } - })) - defer ts.Close() - tc.guestURL = ts.URL + "/" + file - testWithURL(t, tc) - } - }) } func TestEnqueue(t *testing.T) { diff --git a/scheduler/plugin/runtime.go b/scheduler/plugin/runtime.go index abc2b240..8921baed 100644 --- a/scheduler/plugin/runtime.go +++ b/scheduler/plugin/runtime.go @@ -49,20 +49,19 @@ func prepareRuntime(ctx context.Context, guestBin []byte, guestConfig string) (r // Detect and handle any host imports or lack thereof. imports := detectImports(guest.ImportedFunctions()) - switch { - case imports&importWasiP1 != 0: + if imports&importWasiP1 != 0 { if _, err = wasi_snapshot_preview1.Instantiate(ctx, runtime); err != nil { err = fmt.Errorf("wasm: error instantiating wasi: %w", err) return } - fallthrough // proceed to more imports - case imports&importK8sApi != 0: + } + if imports&importK8sApi != 0 { if _, err = instantiateHostApi(ctx, runtime); err != nil { err = fmt.Errorf("wasm: error instantiating api host functions: %w", err) return } - fallthrough // proceed to more imports - case imports&importK8sScheduler != 0: + } + if imports&importK8sScheduler != 0 { if _, err = instantiateHostScheduler(ctx, runtime, guestConfig); err != nil { err = fmt.Errorf("wasm: error instantiating scheduler host functions: %w", err) return diff --git a/scheduler/plugin/wasm.go b/scheduler/plugin/wasm.go new file mode 100644 index 00000000..e8af4830 --- /dev/null +++ b/scheduler/plugin/wasm.go @@ -0,0 +1,68 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package wasm + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "strings" +) + +// getURL parses the URL manually, so that it can resolve relative file paths, +// such as "file://../../path/to/plugin.wasm" +func getURL(ctx context.Context, url string) ([]byte, error) { + firstColon := strings.IndexByte(url, ':') + if firstColon == -1 { + return nil, fmt.Errorf("invalid URL: %s", url) + } + + scheme := url[:firstColon] + switch scheme { + case "http", "https": + return httpGet(ctx, http.DefaultClient, url) + case "file": + guestPath := url[7:] // strip file:// + return os.ReadFile(guestPath) + default: + return nil, fmt.Errorf("unsupported URL scheme: %s", scheme) + } +} + +// httpGet returns a byte slice of the wasm module found at the given URL, or +// an error otherwise. +func httpGet(ctx context.Context, client *http.Client, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // golang/go#60240 recommends to just close the client body, instead of + // draining it first. + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received %v status code from %q", resp.StatusCode, url) + } + + return io.ReadAll(resp.Body) +} diff --git a/scheduler/plugin/wasm_test.go b/scheduler/plugin/wasm_test.go new file mode 100644 index 00000000..6b257d09 --- /dev/null +++ b/scheduler/plugin/wasm_test.go @@ -0,0 +1,201 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package wasm + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "sigs.k8s.io/kube-scheduler-wasm-extension/scheduler/test" +) + +var wasmMagicNumber = []byte{0x00, 0x61, 0x73, 0x6d} + +func Test_getURL(t *testing.T) { + testCtx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + bin, err := os.ReadFile(test.URLTestAllNoopWat[7:]) + if err != nil { + t.Fatal(err) + } + + tmpDir := t.TempDir() + tmpFile, err := filepath.Abs(path.Join(tmpDir, "all_noop.wasm")) + if err != nil { + t.Fatal(err) + } + + if err = os.WriteFile(tmpFile, bin, 0o0444); err != nil { + t.Fatal(err) + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(bin) + })) + t.Cleanup(ts.Close) + + type testCase struct { + name string + url string + expected []byte + expectedError string + } + + tests := []testCase{ + { + name: "relative file valid", + url: "file://" + path.Join("..", "test", "testdata", "test", "all_noop.wasm"), + expected: bin, + }, + { + name: "file valid", + url: "file://" + tmpFile, + expected: bin, + }, + { + name: "not url", + url: "http", + expectedError: "invalid URL: http", + }, + { + name: "http invalid", + url: "http:// ", + expectedError: `parse "http:// ": invalid character " " in host name`, + }, + { + name: "https invalid", + url: "https:// ", + expectedError: `parse "https:// ": invalid character " " in host name`, + }, + { + name: "http", + url: ts.URL, + expected: bin, + }, + { + name: "unsupported scheme", + url: "ldap://foo/bar.wasm", + expectedError: "unsupported URL scheme: ldap", + }, + { + name: "file not found", + url: "file://testduta", + expectedError: "open testduta: ", + }, + { + name: "relative dir not file", + url: "file://.", + // Below ends in "is a directory" in unix, and "The handle is invalid." in windows. + expectedError: "read .: ", + }, + { + name: "dir not file", + url: "file://" + tmpDir, + // Below ends in "is a directory" in unix, and "The handle is invalid." in windows. + expectedError: fmt.Sprintf("read %s: ", tmpDir), + }, + } + + for _, tt := range tests { + tc := tt + t.Run(tc.name, func(t *testing.T) { + bin, err := getURL(testCtx, tc.url) + if tc.expectedError != "" { + // Use substring match as the error can be different in Windows. + if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected err %v to contain %s", err, tc.expectedError) + } + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if want, have := tc.expected, bin; !bytes.Equal(want, have) { + t.Fatalf("unexpected binary: want %v, have %v", want, have) + } + }) + } +} + +func Test_httpGet(t *testing.T) { + wasmBinary := wasmMagicNumber + wasmBinary = append(wasmBinary, 0x00, 0x00, 0x00, 0x00) + cases := []struct { + name string + handler http.HandlerFunc + expectedError string + }{ + { + name: "plain wasm binary", + handler: func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(wasmBinary) + }, + }, + // Compressed payloads are handled automatically by http.Client. + { + name: "compressed payload", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + gw := gzip.NewWriter(w) + defer gw.Close() + _, _ = gw.Write(wasmBinary) + }, + }, + { + name: "http error", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + expectedError: "received 500 status code", + }, + } + + for _, proto := range []string{"http", "https"} { + t.Run(proto, func(t *testing.T) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(tc.handler) + defer ts.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := httpGet(ctx, ts.Client(), ts.URL) + + if tc.expectedError != "" { + // Use substring match as the error can be different in Windows. + if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected err %v to contain %s", err, tc.expectedError) + } + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } + }) + } +} diff --git a/scheduler/test/testdata.go b/scheduler/test/testdata.go index 59996d21..13b3e0e6 100644 --- a/scheduler/test/testdata.go +++ b/scheduler/test/testdata.go @@ -36,6 +36,8 @@ var URLErrorPanicOnStart = localURL(pathWatError("panic_on_start")) var URLExampleNodeNumber = localURL(pathTinyGoExample("nodenumber")) +var URLExampleAdvanced = localURL(pathTinyGoExample("advanced")) + var URLTestAll = localURL(pathTinyGoTest("all")) var URLTestAllNoopWat = localURL(pathWatTest("all_noop"))