diff --git a/.github/workflows/minikube-image-benchmark.yml b/.github/workflows/minikube-image-benchmark.yml new file mode 100644 index 000000000000..3941ec7e1bb4 --- /dev/null +++ b/.github/workflows/minikube-image-benchmark.yml @@ -0,0 +1,31 @@ +name: "build-and-run Publish Chart" +on: + workflow_dispatch: + schedule: + # every day at 7am & 7pm pacific + - cron: "0 2,14 * * *" +env: + GOPROXY: https://proxy.golang.org + GO_VERSION: '1.20.6' +permissions: + contents: read + +jobs: + build-and-run-benchmark: + if: github.repository == 'kubernetes/minikube' + runs-on: ubuntu-20.04 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: 'us-west-1' + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 + with: + go-version: ${{env.GO_VERSION}} + cache-dependency-path: ./go.sum + - name: Run Benchmark + run: | + ./hack/benchmark/build_and_run/publish-chart.sh + + \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index d398a94cf9b5..2c86683df4ef 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "hack/benchmark/time-to-k8s/time-to-k8s-repo"] path = hack/benchmark/time-to-k8s/time-to-k8s-repo url = https://github.com/tstromberg/time-to-k8s.git +[submodule "hack/benchmark/build_and_run/minikube-image-benchmark"] + path = hack/benchmark/build_and_run/minikube-image-benchmark + url = https://github.com/GoogleContainerTools/minikube-image-benchmark.git diff --git a/hack/benchmark/build_and_run/generate-chart.go b/hack/benchmark/build_and_run/generate-chart.go new file mode 100644 index 000000000000..298380d7f0ec --- /dev/null +++ b/hack/benchmark/build_and_run/generate-chart.go @@ -0,0 +1,260 @@ +/* +Copyright 2023 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "image/color" + "io" + "log" + "math" + "os" + "path/filepath" + "strconv" + "time" + + "gonum.org/v1/plot" + "gonum.org/v1/plot/plotter" + "gonum.org/v1/plot/plotutil" + "gonum.org/v1/plot/vg" + "gonum.org/v1/plot/vg/draw" +) + +var Images = []string{ + "buildpacksFewLargeFiles", + "buildpacksFewSmallFiles", + "buildpacksManyLargeFiles", + "buildpacksManySmallFiles", +} + +var Environments = []string{ + "MinikubeImageLoadDocker", + "MinikubeImageBuild", + "MinikubeDockerEnvDocker", + "MinikubeAddonRegistryDocker", + "MinikubeImageLoadContainerd", + "MinikubeImageContainerd", + "MinikubeAddonRegistryContainerd", + "MinikubeImageLoadCrio", + "MinikubeImageCrio", + "MinikubeAddonRegistryCrio", + "Kind", + "K3d", + "Microk8s", +} + +const ( + INTERATIVE = "Iterative" + NONINTERATIVE = "NonIterative" +) + +var Methods = []string{ + INTERATIVE, + NONINTERATIVE, +} + +// env name-> test result +type TestResult map[string]float64 + +func NewTestResult(values []float64) TestResult { + res := make(TestResult) + for index, v := range values { + res[Environments[index]] = v + } + return res +} + +// imageName->TestResult +type ImageTestResults map[string]TestResult + +type MethodTestResults struct { + Date time.Time + // method name -> results + Results map[string]ImageTestResults +} + +type Records struct { + Records []MethodTestResults +} + +func main() { + latestTestResultPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result") + pastTestRecordsPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results") + chartsPath := flag.String("charts", "", "path to the folder to write the daily charts to") + flag.Parse() + + latestBenchmark := readInLatestTestResult(*latestTestResultPath) + latestBenchmark.Date=time.Now() + pastBenchmarks := readInPastTestResults(*pastTestRecordsPath) + pastBenchmarks.Records = append(pastBenchmarks.Records, latestBenchmark) + updatePastTestResults(pastBenchmarks, *pastTestRecordsPath) + createDailyChart(pastBenchmarks, *chartsPath) +} + +// readInLatestTestResult reads in the latest benchmark result from a CSV file +// and return the MethodTestResults object +func readInLatestTestResult(latestBenchmarkPath string) MethodTestResults { + + var res = MethodTestResults{ + Results: make(map[string]ImageTestResults), + } + res.Results[INTERATIVE] = make(ImageTestResults) + res.Results[NONINTERATIVE] = make(ImageTestResults) + + f, err := os.Open(latestBenchmarkPath) + if err != nil { + log.Fatal(err) + } + + r := csv.NewReader(f) + for { + line, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + // skip the first line of the CSV file + if line[0] == "image" { + continue + } + + valuesInterative := []float64{} + valuesNonInterative := []float64{} + // interative test results of each env are stored in the following columns + indicesInterative := []int{1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49} + // non-interative test results of each env are stored in the following columns + indicesNonInterative := []int{3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51} + + for _, i := range indicesInterative { + v, err := strconv.ParseFloat(line[i], 64) + if err != nil { + log.Fatal(err) + } + valuesInterative = append(valuesInterative, v) + } + + for _, i := range indicesNonInterative { + v, err := strconv.ParseFloat(line[i], 64) + if err != nil { + log.Fatal(err) + } + valuesNonInterative = append(valuesNonInterative, v) + } + + imageName := line[0] + + res.Results[INTERATIVE][imageName] = NewTestResult(valuesInterative) + res.Results[NONINTERATIVE][imageName] = NewTestResult(valuesNonInterative) + + } + + return res +} + +// readInPastTestResults reads in the past benchmark results from a JSON file +func readInPastTestResults(pastTestRecordPath string) Records { + + record := Records{} + data, err := os.ReadFile(pastTestRecordPath) + if os.IsNotExist(err) { + return record + } + if err != nil { + log.Fatal(err) + } + + if err := json.Unmarshal(data, &record); err != nil { + log.Fatal(err) + } + + return record +} + +// updateRunsFile overwrites the run file with the updated benchmarks list +func updatePastTestResults(h Records, pastTestRecordPath string) { + b, err := json.Marshal(h) + if err != nil { + log.Fatal(err) + } + + if err := os.WriteFile(pastTestRecordPath, b, 0600); err != nil { + log.Fatal(err) + } +} +func createDailyChart(record Records, outputFolder string) { + for _, method := range Methods { + for _, image := range Images { + createChart(record, method, image, outputFolder) + } + } +} + +func createChart(record Records, methodName string, imageName string, chartOutputPath string) { + p := plot.New() + p.Add(plotter.NewGrid()) + p.Legend.Top = true + p.Title.Text =fmt.Sprintf("%s-%s-performance", methodName, imageName) + p.X.Label.Text = "date" + p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"} + p.Y.Label.Text = "time (seconds)" + yMaxTotal := float64(0) + + // gonum plot do not have enough default colors in any group + // so we combine different group of default colors + colors := append([]color.Color{}, plotutil.SoftColors...) + colors = append(colors, plotutil.DarkColors...) + + pointGroup := make(map[string]plotter.XYs) + for _, name := range Environments[0:7] { + pointGroup[name] = make(plotter.XYs, len(record.Records)) + + } + + for i := 0; i < len(record.Records); i++ { + for _, envName := range Environments[0:7] { + pointGroup[envName][i].X = float64(record.Records[i].Date.Unix()) + pointGroup[envName][i].Y = record.Records[i].Results[methodName][imageName][envName] + yMaxTotal = math.Max(yMaxTotal, pointGroup[envName][i].Y) + } + } + p.Y.Max = yMaxTotal + + i := 0 + for envName, xys := range pointGroup { + line, points, err := plotter.NewLinePoints(xys) + if err != nil { + log.Fatal(err) + } + line.Color = colors[i] + points.Color = colors[i] + points.Shape = draw.CircleGlyph{} + i++ + p.Add(line, points) + p.Legend.Add(envName, line) + } + + filename := filepath.Join(chartOutputPath, fmt.Sprintf("%s_%s_chart.png", methodName, imageName)) + + if err := p.Save(12*vg.Inch, 8*vg.Inch, filename); err != nil { + log.Fatalf("failed creating png: %v", err) + } +} diff --git a/hack/benchmark/build_and_run/minikube-image-benchmark b/hack/benchmark/build_and_run/minikube-image-benchmark new file mode 160000 index 000000000000..feab1337c92e --- /dev/null +++ b/hack/benchmark/build_and_run/minikube-image-benchmark @@ -0,0 +1 @@ +Subproject commit feab1337c92e1cd01d29e24c085407ec5ebdc3d2 diff --git a/hack/benchmark/build_and_run/publish-chart.sh b/hack/benchmark/build_and_run/publish-chart.sh new file mode 100755 index 000000000000..f04aec2ffe6f --- /dev/null +++ b/hack/benchmark/build_and_run/publish-chart.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +BUCKET="s3://build-and-run" + +install_minikube() { + make + sudo install ./out/minikube /usr/local/bin/minikube +} + +run_benchmark() { + ( cd ./hack/benchmark/build_and_run/minikube-image-benchmark && + git submodule update --init && + make && + ./out/benchmark ) +} + +generate_chart() { + go run ./hack/benchmark/build_and_run/generate-chart.go --csv hack/benchmark/build_and_run/minikube-image-benchmark/out/results.csv --past-runs record.json +} + +copy() { + aws s3 cp "$1" "$2" +} + +cleanup() { + rm ./Iterative_buildpacksFewLargeFiles_chart.png + rm ./Iterative_buildpacksFewSmallFiles_chart.png + rm ./Iterative_buildpacksManyLargeFiles_chart.png + rm ./Iterative_buildpacksManySmallFiles_chart.png + rm ./NonIterative_buildpacksFewLargeFiles_chart.png + rm ./NonIterative_buildpacksFewSmallFiles_chart.png + rm ./NonIterative_buildpacksManyLargeFiles_chart.png + rm ./NonIterative_buildpacksManySmallFiles_chart.png + rm hack/benchmark/build_and_run/minikube-image-benchmark/out/results.csv +} + + +install_minikube +copy "$BUCKET/record.json" ./record.json +set -e + +run_benchmark +generate_chart + +copy ./record.json "$BUCKET/record.json" +copy ./Iterative_buildpacksFewLargeFiles_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_chart.png" +copy ./Iterative_buildpacksFewSmallFiles_chart.png "$BUCKET/Iterative_buildpacksFewSmallFiles_chart.png" +copy ./Iterative_buildpacksManyLargeFiles_chart.png "$BUCKET/Iterative_buildpacksManyLargeFiles_chart.png" +copy ./Iterative_buildpacksManySmallFiles_chart.png "$BUCKET/Iterative_buildpacksManySmallFiles_chart.png" +copy ./NonIterative_buildpacksFewLargeFiles_chart.png "$BUCKET/NonIterative_buildpacksFewLargeFiles_chart.png" +copy ./NonIterative_buildpacksFewSmallFiles_chart.png "$BUCKET/NonIterative_buildpacksFewSmallFiles_chart.png" +copy ./NonIterative_buildpacksManyLargeFiles_chart.png "$BUCKET/NonIterative_buildpacksManyLargeFiles_chart.png" +copy ./NonIterative_buildpacksManySmallFiles_chart.png "$BUCKET/NonIterative_buildpacksManySmallFiles_chart.png" +cleanup