From 404d84abe82136f8bef4e5e93ee2d30cae379fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A2=D0=BE=D0=B2=D0=B0=D1=80=D0=B8=D1=89=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B3=D1=80=D0=B0=D0=BC=D0=BC=D0=B8=D1=81=D1=82?= <2962928213@qq.com> Date: Mon, 14 Aug 2023 19:28:59 +0800 Subject: [PATCH] add github action for running minikube-image-benchmark --- .../workflows/minikube-image-benchmark.yml | 31 ++ .gitmodules | 3 + hack/benchmark/image-build/generate-chart.go | 279 ++++++++++++++++++ .../image-build/minikube-image-benchmark | 1 + hack/benchmark/image-build/publish-chart.sh | 57 ++++ 5 files changed, 371 insertions(+) create mode 100644 .github/workflows/minikube-image-benchmark.yml create mode 100644 hack/benchmark/image-build/generate-chart.go create mode 160000 hack/benchmark/image-build/minikube-image-benchmark create mode 100755 hack/benchmark/image-build/publish-chart.sh diff --git a/.github/workflows/minikube-image-benchmark.yml b/.github/workflows/minikube-image-benchmark.yml new file mode 100644 index 000000000000..e232808d6652 --- /dev/null +++ b/.github/workflows/minikube-image-benchmark.yml @@ -0,0 +1,31 @@ +name: "build-and-run Publish Chart" +on: + workflow_dispatch: + schedule: + # every day at 7am & 7pm pacific + - cron: "0 2,14 * * *" +env: + GOPROXY: https://proxy.golang.org + GO_VERSION: '1.20.6' +permissions: + contents: read + +jobs: + build-and-run-benchmark: + if: github.repository == 'kubernetes/minikube' + runs-on: ubuntu-20.04 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: 'us-west-1' + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 + with: + go-version: ${{env.GO_VERSION}} + cache-dependency-path: ./go.sum + - name: Run Benchmark + run: | + ./hack/benchmark/image-build/publish-chart.sh + + \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index d398a94cf9b5..880fd9cec906 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "hack/benchmark/time-to-k8s/time-to-k8s-repo"] path = hack/benchmark/time-to-k8s/time-to-k8s-repo url = https://github.com/tstromberg/time-to-k8s.git +[submodule "hack/benchmark/image-build/minikube-image-benchmark"] + path = hack/benchmark/image-build/minikube-image-benchmark + url = git@github.com:GoogleContainerTools/minikube-image-benchmark.git diff --git a/hack/benchmark/image-build/generate-chart.go b/hack/benchmark/image-build/generate-chart.go new file mode 100644 index 000000000000..c81cccb76ca0 --- /dev/null +++ b/hack/benchmark/image-build/generate-chart.go @@ -0,0 +1,279 @@ +/* +Copyright 2023 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "image/color" + "io" + "log" + "math" + "os" + "path/filepath" + "strconv" + "time" + + "gonum.org/v1/plot" + "gonum.org/v1/plot/plotter" + "gonum.org/v1/plot/plotutil" + "gonum.org/v1/plot/vg" + "gonum.org/v1/plot/vg/draw" +) + +var Images = []string{ + "buildpacksFewLargeFiles", + // to simplfy the output, the following images are omitted + // "buildpacksFewSmallFiles", + // "buildpacksManyLargeFiles", + // "buildpacksManySmallFiles", +} + +var Environments = []string{ + "MinikubeImageLoadDocker", + "MinikubeImageBuild", + "MinikubeDockerEnvDocker", + "MinikubeAddonRegistryDocker", + "MinikubeImageLoadContainerd", + "MinikubeImageContainerd", + "MinikubeAddonRegistryContainerd", + "MinikubeImageLoadCrio", + "MinikubeImageCrio", + "MinikubeAddonRegistryCrio", + "Kind", + "K3d", + "Microk8s", +} + +var RuntimeEnvironments = map[string][]string{ + "docker": []string{ + "MinikubeImageLoadDocker", + "MinikubeImageBuild", + "MinikubeDockerEnvDocker", + "MinikubeAddonRegistryDocker", + }, + + "containerd": []string{ + "MinikubeImageLoadContainerd", + "MinikubeImageContainerd", + "MinikubeAddonRegistryContainerd", + }, +} + +const ( + INTERATIVE = "Iterative" + NONINTERATIVE = "NonIterative" +) + +var Methods = []string{ + INTERATIVE, + // to simplfy the output, non-interative is omitted + // NONINTERATIVE, +} + +// env name-> test result +type TestResult map[string]float64 + +func NewTestResult(values []float64) TestResult { + res := make(TestResult) + for index, v := range values { + res[Environments[index]] = v + } + return res +} + +// imageName->TestResult +type ImageTestResults map[string]TestResult + +type MethodTestResults struct { + Date time.Time + // method name -> results + Results map[string]ImageTestResults +} + +type Records struct { + Records []MethodTestResults +} + +func main() { + latestTestResultPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result") + pastTestRecordsPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results") + chartsPath := flag.String("charts", "", "path to the folder to write the daily charts to") + flag.Parse() + + latestBenchmark := readInLatestTestResult(*latestTestResultPath) + latestBenchmark.Date = time.Now() + pastBenchmarks := readInPastTestResults(*pastTestRecordsPath) + pastBenchmarks.Records = append(pastBenchmarks.Records, latestBenchmark) + updatePastTestResults(pastBenchmarks, *pastTestRecordsPath) + createDailyChart(pastBenchmarks, *chartsPath) +} + +// readInLatestTestResult reads in the latest benchmark result from a CSV file +// and return the MethodTestResults object +func readInLatestTestResult(latestBenchmarkPath string) MethodTestResults { + + var res = MethodTestResults{ + Results: make(map[string]ImageTestResults), + } + res.Results[INTERATIVE] = make(ImageTestResults) + res.Results[NONINTERATIVE] = make(ImageTestResults) + + f, err := os.Open(latestBenchmarkPath) + if err != nil { + log.Fatal(err) + } + + r := csv.NewReader(f) + for { + line, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + // skip the first line of the CSV file + if line[0] == "image" { + continue + } + + valuesInterative := []float64{} + valuesNonInterative := []float64{} + // interative test results of each env are stored in the following columns + indicesInterative := []int{1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49} + // non-interative test results of each env are stored in the following columns + indicesNonInterative := []int{3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51} + + for _, i := range indicesInterative { + v, err := strconv.ParseFloat(line[i], 64) + if err != nil { + log.Fatal(err) + } + valuesInterative = append(valuesInterative, v) + } + + for _, i := range indicesNonInterative { + v, err := strconv.ParseFloat(line[i], 64) + if err != nil { + log.Fatal(err) + } + valuesNonInterative = append(valuesNonInterative, v) + } + + imageName := line[0] + + res.Results[INTERATIVE][imageName] = NewTestResult(valuesInterative) + res.Results[NONINTERATIVE][imageName] = NewTestResult(valuesNonInterative) + + } + + return res +} + +// readInPastTestResults reads in the past benchmark results from a JSON file +func readInPastTestResults(pastTestRecordPath string) Records { + + record := Records{} + data, err := os.ReadFile(pastTestRecordPath) + if os.IsNotExist(err) { + return record + } + if err != nil { + log.Fatal(err) + } + + if err := json.Unmarshal(data, &record); err != nil { + log.Fatal(err) + } + + return record +} + +// updateRunsFile overwrites the run file with the updated benchmarks list +func updatePastTestResults(h Records, pastTestRecordPath string) { + b, err := json.Marshal(h) + if err != nil { + log.Fatal(err) + } + + if err := os.WriteFile(pastTestRecordPath, b, 0600); err != nil { + log.Fatal(err) + } +} +func createDailyChart(record Records, outputFolder string) { + + for _, method := range Methods { + for _, image := range Images { + createChart(record, method, image, "docker", outputFolder) + createChart(record, method, image, "containerd", outputFolder) + } + } +} + +func createChart(record Records, methodName string, imageName string, runtime string, chartOutputPath string) { + p := plot.New() + p.Add(plotter.NewGrid()) + p.Legend.Top = true + p.Title.Text = fmt.Sprintf("%s-%s-%s-performance", methodName, imageName, runtime) + p.X.Label.Text = "date" + p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"} + p.Y.Label.Text = "time (seconds)" + yMaxTotal := float64(0) + + // gonum plot do not have enough default colors in any group + // so we combine different group of default colors + colors := append([]color.Color{}, plotutil.SoftColors...) + colors = append(colors, plotutil.DarkColors...) + + pointGroup := make(map[string]plotter.XYs) + for _, name := range RuntimeEnvironments[runtime] { + pointGroup[name] = make(plotter.XYs, len(record.Records)) + + } + + for i := 0; i < len(record.Records); i++ { + for _, envName := range RuntimeEnvironments[runtime] { + pointGroup[envName][i].X = float64(record.Records[i].Date.Unix()) + pointGroup[envName][i].Y = record.Records[i].Results[methodName][imageName][envName] + yMaxTotal = math.Max(yMaxTotal, pointGroup[envName][i].Y) + } + } + p.Y.Max = yMaxTotal + + i := 0 + for envName, xys := range pointGroup { + line, points, err := plotter.NewLinePoints(xys) + if err != nil { + log.Fatal(err) + } + line.Color = colors[i] + points.Color = colors[i] + points.Shape = draw.CircleGlyph{} + i++ + p.Add(line, points) + p.Legend.Add(envName, line) + } + + filename := filepath.Join(chartOutputPath, fmt.Sprintf("%s_%s_%s_chart.png", methodName, imageName, runtime)) + + if err := p.Save(12*vg.Inch, 8*vg.Inch, filename); err != nil { + log.Fatalf("failed creating png: %v", err) + } +} diff --git a/hack/benchmark/image-build/minikube-image-benchmark b/hack/benchmark/image-build/minikube-image-benchmark new file mode 160000 index 000000000000..feab1337c92e --- /dev/null +++ b/hack/benchmark/image-build/minikube-image-benchmark @@ -0,0 +1 @@ +Subproject commit feab1337c92e1cd01d29e24c085407ec5ebdc3d2 diff --git a/hack/benchmark/image-build/publish-chart.sh b/hack/benchmark/image-build/publish-chart.sh new file mode 100755 index 000000000000..0b8cb7889bfd --- /dev/null +++ b/hack/benchmark/image-build/publish-chart.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +BUCKET="s3://build-and-run" + +install_minikube() { + make + sudo install ./out/minikube /usr/local/bin/minikube +} + +run_benchmark() { + ( cd ./hack/benchmark/image-build/minikube-image-benchmark && + git submodule update --init && + make && + ./out/benchmark ) +} + +generate_chart() { + go run ./hack/benchmark/image-build/generate-chart.go --csv hack/benchmark/image-build/minikube-image-benchmark/out/results.csv --past-runs record.json +} + +copy() { + aws s3 cp "$1" "$2" +} + +cleanup() { + rm ./Iterative_buildpacksFewLargeFiles_containerd_chart.png + rm ./Iterative_buildpacksFewLargeFiles_docker_chart.png + rm hack/benchmark/image-build/minikube-image-benchmark/out/results.csv +} + + +install_minikube +copy "$BUCKET/record.json" ./record.json +set -e + +run_benchmark +generate_chart + +copy ./record.json "$BUCKET/record.json" +copy ./Iterative_buildpacksFewLargeFiles_containerd_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_containerd_chart.png" +copy ./Iterative_buildpacksFewLargeFiles_docker_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_docker_chart.png" +cleanup