Skip to content

Commit

Permalink
Add e2e-framework for kwok with kind
Browse files Browse the repository at this point in the history
Signed-off-by: wwwnay <[email protected]>
  • Loading branch information
wzshiming committed Sep 1, 2023
1 parent a43b132 commit db886a9
Show file tree
Hide file tree
Showing 33 changed files with 1,725 additions and 260 deletions.
25 changes: 24 additions & 1 deletion .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,32 @@ jobs:
run: |
BUILDER=${{ matrix.builder }} make build-cluster-image
test-kwok-e2e:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: "1.20"

- name: Test e2e default
shell: bash
run: |
./hack/e2e-test.sh e2e/kwok/default
- name: Test e2e single
shell: bash
run: |
./hack/e2e-test.sh e2e/kwok/single
test-kwok:
strategy:
fail-fast: false
matrix:
case:
- kwok
- kwok-with-cni
- kwok-single-node
continue-on-error: false
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -263,6 +281,11 @@ jobs:
run: |
./hack/e2e-test.sh kwokctl/kwokctl_with_dryrun
- name: Test e2e
shell: bash
run: |
./hack/e2e-test.sh e2e/kwokctl/${{ matrix.kwokctl-runtime }}
- name: Test Export Logs
shell: bash
run: |
Expand Down
7 changes: 5 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@ require (
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/controller-runtime v0.16.0
sigs.k8s.io/controller-tools v0.13.0
sigs.k8s.io/e2e-framework v0.3.0
sigs.k8s.io/kustomize/api v0.14.0
sigs.k8s.io/kustomize/kustomize/v5 v5.1.1
sigs.k8s.io/kustomize/kyaml v0.14.3
sigs.k8s.io/yaml v1.3.0
)

Expand All @@ -50,6 +53,7 @@ require (
github.com/distribution/distribution/v3 v3.0.0-20230214150026-36d8c594d7aa // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.4 // indirect
Expand Down Expand Up @@ -86,6 +90,7 @@ require (
github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/vladimirvivien/gexe v0.2.0 // indirect
github.com/wzshiming/trie v0.1.1 // indirect
github.com/wzshiming/winseq v0.0.0-20200112104235-db357dc107ae // indirect
github.com/xlab/treeprint v1.2.0 // indirect
Expand All @@ -110,8 +115,6 @@ require (
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.14.0 // indirect
sigs.k8s.io/kustomize/cmd/config v0.11.3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
)
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJI
github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand All @@ -46,6 +48,7 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
Expand Down Expand Up @@ -103,6 +106,7 @@ github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU=
github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
Expand Down Expand Up @@ -153,6 +157,7 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down Expand Up @@ -187,6 +192,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
github.com/wzshiming/cmux v0.3.2 h1:lBEWbfbRqUDdXB6Mro/g35kvCuUEmAgIdpGEuER3bis=
github.com/wzshiming/cmux v0.3.2/go.mod h1:lPhqJN2E3frzkxrPdjesxL09z7nTcuZ6i8Is+2G/Xw4=
github.com/wzshiming/ctc v1.2.3 h1:q+hW3IQNsjIlOFBTGZZZeIXTElFM4grF4spW/errh/c=
Expand All @@ -205,6 +212,8 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee33
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
Expand Down Expand Up @@ -275,6 +284,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ=
Expand Down Expand Up @@ -341,6 +351,8 @@ sigs.k8s.io/controller-runtime v0.16.0 h1:5koYaaRVBHDr0LZAJjO5dWzUjMsh6cwa7q1Mmu
sigs.k8s.io/controller-runtime v0.16.0/go.mod h1:77DnuwA8+J7AO0njzv3wbNlMOnGuLrwFr8JPNwx3J7g=
sigs.k8s.io/controller-tools v0.13.0 h1:NfrvuZ4bxyolhDBt/rCZhDnx3M2hzlhgo5n3Iv2RykI=
sigs.k8s.io/controller-tools v0.13.0/go.mod h1:5vw3En2NazbejQGCeWKRrE7q4P+CW8/klfVqP8QZkgA=
sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U=
sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.14.0 h1:6+QLmXXA8X4eDM7ejeaNUyruA1DDB3PVIjbpVhDOJRA=
Expand Down
16 changes: 16 additions & 0 deletions hack/e2e-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ function all_cases() {
cases="$(echo "${cases}" | grep -v "${skip}" || :)"
done
echo "${cases}"
go test -json -v sigs.k8s.io/kwok/test/e2e/... -args --dry-run |
jq -r '. | select( .Action == "pass" ) | select( .Test == null ) | .Package' |
sed 's|^sigs.k8s.io/kwok/test/||g'
}

function usage() {
Expand Down Expand Up @@ -73,6 +76,19 @@ function main() {
local failed=()
for target in "${TARGETS[@]}"; do
echo "================================================================================"
if [[ "${target}" == "e2e/"* ]]; then
echo "Testing ${target}..."
if ! go test -v "sigs.k8s.io/kwok/test/${target}" -args --v=6; then
failed+=("${target}")
echo "------------------------------"
echo "Test ${target} failed."
else
echo "------------------------------"
echo "Test ${target} passed."
fi
continue
fi

target="${target%.test.sh}"
test="${test_dir}/${target}.test.sh"
if [[ ! -x "${test}" ]]; then
Expand Down
54 changes: 54 additions & 0 deletions test/e2e/cases.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"sigs.k8s.io/e2e-framework/pkg/features"

"sigs.k8s.io/kwok/test/e2e/helper"
)

// CaseNode creates a feature that tests node creation and deletion
func CaseNode(nodeName string) *features.FeatureBuilder {
node := helper.NewNodeBuilder(nodeName).
Build()
return features.New("Node can be created and deleted").
Assess("create node", helper.CreateNode(node)).
Assess("delete node", helper.DeleteNode(node))
}

// CasePod creates a feature that tests pod creation and deletion
func CasePod(nodeName string, namespace string) *features.FeatureBuilder {
node := helper.NewNodeBuilder(nodeName).
Build()
pod0 := helper.NewPodBuilder("pod0").
WithNamespace(namespace).
WithNodeName(nodeName).
Build()
pod1 := helper.NewPodBuilder("pod1").
WithNamespace(namespace).
WithNodeName(nodeName).
WithHostNetwork(true).
Build()
return features.New("Pod can be created and deleted").
Setup(helper.CreateNode(node)).
Teardown(helper.DeleteNode(node)).
Assess("create pod", helper.CreatePod(pod0)).
Assess("create pod for host network", helper.CreatePod(pod1)).
Assess("delete pod", helper.DeletePod(pod0)).
Assess("delete pod for host network", helper.DeletePod(pod1))
}
18 changes: 18 additions & 0 deletions test/e2e/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package e2e is a test package for e2e test.
package e2e
85 changes: 85 additions & 0 deletions test/e2e/helper/build.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helper

import (
"context"
"fmt"
"runtime"
"strings"

"sigs.k8s.io/e2e-framework/pkg/env"
"sigs.k8s.io/e2e-framework/pkg/envconf"

"sigs.k8s.io/kwok/pkg/utils/exec"
)

// BuildKwokImage builds the kwok image and returns a function that can be used
func BuildKwokImage(rootDir string, image string, builder string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
ref := strings.SplitN(image, ":", 2)
if len(ref) != 2 {
return nil, fmt.Errorf("invalid image reference %q", image)
}
ctx = exec.WithStdIO(ctx)
ctx = exec.WithDir(ctx, rootDir)

err := exec.Exec(ctx, "bash", "./hack/releases.sh", "--bin", "kwok", "--platform", "linux/"+runtime.GOARCH)
if err != nil {
return nil, err
}

err = exec.Exec(ctx, "bash", "./images/kwok/build.sh", "--image", ref[0], "--builder", builder, "--version", ref[1], "--platform", "linux/"+runtime.GOARCH)
if err != nil {
return nil, err
}
return ctx, nil
}
}

// BuildKwokBinary builds the kwok binary and returns a function that can be used
func BuildKwokBinary(rootDir string) env.Func {
return buildBinary(rootDir, "kwok")
}

// BuildKwokctlBinary builds the kwokctl binary and returns a function that can be used
func BuildKwokctlBinary(rootDir string) env.Func {
return buildBinary(rootDir, "kwokctl")
}

// buildBinary builds the kwok binary and returns a function that can be used
func buildBinary(rootDir string, binary string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
ctx = exec.WithStdIO(ctx)
ctx = exec.WithDir(ctx, rootDir)

err := exec.Exec(ctx, "bash", "./hack/releases.sh", "--bin", binary, "--platform", runtime.GOOS+"/"+runtime.GOARCH)
if err != nil {
return nil, err
}
return ctx, nil
}
}

// BinSuffix is the suffix for binaries on Windows
var BinSuffix string

func init() {
if runtime.GOOS == "windows" {
BinSuffix = ".exe"
}
}
18 changes: 18 additions & 0 deletions test/e2e/helper/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package helper provides tools for e2e test.
package helper
Loading

0 comments on commit db886a9

Please sign in to comment.