Development tools to enable pprof

Tools for building a controller with pprof enabed, and Make targets for accessing the http pprof endpoint while it is running on a Kubernetes cluster, with accompanying documentation. This will help in diagnosing #414
googleforgames · Nov 16, 2018 · 63cd0a4 · 63cd0a4
1 parent 88adba4
commit 63cd0a4
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 34 deletions.
diff --git a/build/Makefile b/build/Makefile
@@ -46,6 +46,7 @@ GCP_CLUSTER_ZONE ?= us-west1-c
 GCP_BUCKET_CHARTS ?= agones-chart
 # the profile to use when developing on minikube
 MINIKUBE_PROFILE ?= agones
+GO_BUILD_TAGS ?= none
 
 # Game Server image to use while doing end-to-end tests
 GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.4
@@ -185,7 +186,8 @@ uninstall: $(ensure-build-image)
 # Build a static binary for the gameserver controller
 build-controller-binary: $(ensure-build-image)
 	docker run --rm -e "CGO_ENABLED=0" $(common_mounts) $(build_tag) go build \
-		-o $(mount_path)/cmd/controller/bin/controller -a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/controller
+	-tags $(GO_BUILD_TAGS) -o $(mount_path)/cmd/controller/bin/controller \
+	-a $(go_version_flags) -installsuffix cgo $(agones_package)/cmd/controller
 
 # Lint the go source code.
 # use LINT_TIMEOUT to manipulate the linter timeout
@@ -293,6 +295,18 @@ push-build-image:
 	docker tag $(build_tag) $(build_remote_tag)
 	docker push $(build_remote_tag)
 
+# port forward the agones controller.
+# useful for pprof and stats viewing, etc
+controller-portforward: PORT ?= 6060
+controller-portforward:
+	docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) -p $(PORT):$(PORT) $(build_tag) \
+		kubectl port-forward deployments/agones-controller -n agones-system $(PORT)
+
+# start pprof with a web ui
+pprof-web:
+	docker run --rm -it --network=host $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) \
+		go tool pprof -http :6061 http://localhost:6060
+
 # generate a changelog using github-changelog-generator
 gen-changelog: RELEASE_VERSION ?= $(base_version)
 gen-changelog:
@@ -471,3 +485,7 @@ minikube-transfer-image:
 # Runs e2e tests against our minikube
 minikube-test-e2e: DOCKER_RUN_ARGS=--network=host -v $(minikube_cert_mount)
 minikube-test-e2e: minikube-agones-profile test-e2e
+
+# minikube port forwarding
+minikube-controller-portforward:
+	$(MAKE) controller-portforward DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)"
diff --git a/build/README.md b/build/README.md
@@ -7,37 +7,70 @@ Rather than installing all the dependencies locally, you can test and build Agon
 built from the Dockerfile in this directory. There is an accompanying Makefile for all the common
 tasks you may wish to accomplish.
 
-<!-- ToC start -->
-# Table of Contents
-
-   1. [Table of Contents](#table-of-contents)
-   1. [Building on Different Platforms](#building-on-different-platforms)
-      1. [Linux](#linux)
-      1. [Windows](#windows)
-      1. [macOS](#macOS)
-   1. [GOPATH](#gopath)
-   1. [Testing and Building](#testing-and-building)
-      1. [Running a Test Google Kubernetes Engine Cluster](#running-a-test-google-kubernetes-engine-cluster)
-      1. [Running a Test Minikube cluster](#running-a-test-minikube-cluster)
-      1. [Running a Custom Test Environment](#running-a-custom-test-environment)
-      1. [Next Steps](#next-steps)
-   1. [Make Variable Reference](#make-variable-reference)
-      1. [VERSION](#version)
-      1. [REGISTRY](#registry)
-      1. [KUBECONFIG](#kubeconfig)
-      1. [CLUSTER_NAME](#cluster_name)
-      1. [IMAGE_PULL_SECRET](#image_pull_secret)
-      1. [IMAGE_PULL_SECRET_FILE](#image_pull_secret_file)
-   1. [Make Target Reference](#make-target-reference)
-      1. [Development Targets](#development-targets)
-      1. [Build Image Targets](#build-image-targets)
-      1. [Google Cloud Platform](#google-cloud-platform)
-      1. [Minikube](#minikube)
-      1. [Custom Environment](#custom-environment)
-   1. [Dependencies](#dependencies)
-   1. [Troubleshooting](#troubleshooting)
-
-<!-- ToC end -->
+Table of Contents
+=================
+
+  * [Building on Different Platforms](#building-on-different-platforms)
+     * [Linux](#linux)
+     * [Windows](#windows)
+     * [macOS](#macos)
+  * [GOPATH](#gopath)
+  * [Testing and Building](#testing-and-building)
+     * [Running a Test Google Kubernetes Engine Cluster](#running-a-test-google-kubernetes-engine-cluster)
+     * [Running a Test Minikube cluster](#running-a-test-minikube-cluster)
+     * [Running a Custom Test Environment](#running-a-custom-test-environment)
+     * [Next Steps](#next-steps)
+  * [Make Variable Reference](#make-variable-reference)
+     * [VERSION](#version)
+     * [REGISTRY](#registry)
+     * [KUBECONFIG](#kubeconfig)
+     * [CLUSTER_NAME](#cluster_name)
+     * [IMAGE_PULL_SECRET](#image_pull_secret)
+     * [IMAGE_PULL_SECRET_FILE](#image_pull_secret_file)
+  * [Make Target Reference](#make-target-reference)
+     * [Development Targets](#development-targets)
+        * [make build](#make-build)
+        * [make build-images](#make-build-images)
+        * [make build-sdks](#make-build-sdks)
+        * [make build-sdk-cpp](#make-build-sdk-cpp)
+        * [make test](#make-test)
+        * [make push](#make-push)
+        * [make install](#make-install)
+        * [make uninstall](#make-uninstall)
+        * [make test-e2e](#make-test-e2e)
+        * [make controller-portforward](#make-controller-portforward)
+        * [make pprof-web](#make-pprof-web)
+        * [make shell](#make-shell)
+        * [make godoc](#make-godoc)
+        * [make build-agones-controller-image](#make-build-agones-controller-image)
+        * [make build-agones-sdk-image](#make-build-agones-sdk-image)
+        * [make gen-install](#make-gen-install)
+        * [make gen-crd-client](#make-gen-crd-client)
+        * [make gen-gameservers-sdk-grpc](#make-gen-gameservers-sdk-grpc)
+     * [Build Image Targets](#build-image-targets)
+        * [make clean-config](#make-clean-config)
+        * [make clean-build-image](#make-clean-build-image)
+        * [make build-build-image](#make-build-build-image)
+     * [Google Cloud Platform](#google-cloud-platform)
+        * [make gcloud-init](#make-gcloud-init)
+        * [make gcloud-test-cluster](#make-gcloud-test-cluster)
+        * [make gcloud-auth-cluster](#make-gcloud-auth-cluster)
+        * [make gcloud-auth-docker](#make-gcloud-auth-docker)
+     * [Minikube](#minikube)
+        * [make minikube-test-cluster](#make-minikube-test-cluster)
+        * [make minikube-push](#make-minikube-push)
+        * [make minikube-install](#make-minikube-install)
+     * [make minikube-test-e2e](#make-minikube-test-e2e)
+        * [make minikube-shell](#make-minikube-shell)
+        * [make minikube-transfer-image](#make-minikube-transfer-image)
+        * [make minikube-controller-portforward](#make-minikube-controller-portforward)
+     * [Custom Environment](#custom-environment)
+        * [make setup-custom-test-cluster](#make-setup-custom-test-cluster)
+        * [make clean-custom-test-cluster](#make-clean-custom-test-cluster)
+  * [Dependencies](#dependencies)
+  * [Troubleshooting](#troubleshooting)
+        * [$GOPATH/$GOROOT error when building in WSL](#gopathgoroot-error-when-building-in-wsl)
+        * [I want to use pprof to profile the controller.](#i-want-to-use-pprof-to-profile-the-controller)
 
 ## Building on Different Platforms
 
@@ -330,14 +363,22 @@ Installs the current development version of Agones into the Kubernetes cluster
 #### `make uninstall`
 Removes Agones from the Kubernetes cluster
 
-### `make test-e2e`
+#### `make test-e2e`
 Runs end-to-end tests on the previously installed version of Agones.
 These tests validate Agones flow from start to finish.
 
 It uses the KUBECONFIG to target a Kubernetes cluster.
 
 See [`make minikube-test-e2e`](#make-minikube-test-e2e) to run end-to-end tests on minikube.
 
+#### `make controller-portforward`
+Sets up port forwarding to a specified PORT var (defaults to 6060 for pprof) to the
+controller deployment.
+
+#### `make pprof-web`
+
+Start the web interface for pprof.
+
 #### `make shell`
 Run a bash shell with the developer tools (go tooling, kubectl, etc) and source code in it.
 
@@ -425,6 +466,10 @@ instead of `make shell` to start an interactive shell for development on Minikub
 Convenience target for transferring images into minikube.
 Use TAG to specify the image to transfer into minikube
 
+#### `make minikube-controller-portforward`
+The minikube version of [`make controller-portforward`](#make-controller-portforward) to setup
+port forwarding to the controller deployment.
+
 ### Custom Environment
 
 #### `make setup-custom-test-cluster`
@@ -453,4 +498,14 @@ If you get this error when building Agones in WSL (`make build`, `make test` or
 - Are your project files on a different folder than C? If yes, then you should either move them on drive C or set up Docker for Windows to share your project drive as well
 - Did you set up the volume mount for Docker correctly? By default, drive C is mapped by WSL as /mnt/c, but Docker expects it as /c. You can test by executing `ls /c` in your linux shell. If you get an error, then follow the instructions for [setting up volume mount for Docker](https://nickjanetakis.com/blog/setting-up-docker-for-windows-and-wsl-to-work-flawlessly#ensure-volume-mounts-work)
 
+#### I want to use pprof to profile the controller.
+
+Run `make build-images GO_BUILD_TAGS=profile` and this will build images with [pprof](https://golang.org/pkg/net/http/pprof/)
+enabled in the controller, which you can then push and install on your cluster.
+
+To get the pprof ui working, run `make controller-portforward` (or `minikube-controller-portforward` if you are on minikube),
+which will setup the port forwarding to the pprof http endpoint.
+
+Run `make pprof-web`, which will start the web interface. It may take a few minutes to start up, but it can be opened on
+[http://localhost:6060/ui](http://localhost:6060/ui).
 
diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile
@@ -19,7 +19,7 @@
 FROM gcr.io/agones-images/grpc-cxx:1.12
 
 RUN apt-get update && \
-    apt-get install -y wget psmisc rsync make python bash-completion zip nano jq && \
+    apt-get install -y wget psmisc rsync make python bash-completion zip nano jq graphviz && \
     apt-get clean
 
 # install go

diff --git a/cmd/controller/pprof.go b/cmd/controller/pprof.go
@@ -0,0 +1,30 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build profile
+
+package main
+
+import (
+	"github.com/sirupsen/logrus"
+	"net/http"
+	_ "net/http/pprof"
+)
+
+func init() {
+	go func() {
+		logrus.WithError(http.ListenAndServe(":6060", nil)).Info("Closed pprof server")
+	}()
+	logrus.Info("*** PPROF PROFILER STARTED on :6060 ***")
+}