Skip to content

Commit

Permalink
Add graceful pod termination/grpc health checks
Browse files Browse the repository at this point in the history
  • Loading branch information
ChrisRx committed Feb 25, 2024
1 parent 9eb408e commit b57e37d
Show file tree
Hide file tree
Showing 29 changed files with 637 additions and 186 deletions.
5 changes: 4 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,13 @@ COPY pkg pkg/
# RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on taskset -c 1 /usr/local/go/bin/go build -a -o q3 ./cmd/q3
RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -o q3 ./cmd/q3

RUN CGO_ENABLED=0 go install github.com/grpc-ecosystem/grpc-health-probe@latest

FROM alpine:3

COPY --from=builder /workspace/q3 /usr/local/bin
COPY --from=builder /go/bin/grpc-health-probe /usr/local/bin/grpc-health-probe
COPY --from=quake-n-bake /usr/local/bin/ioq3ded /usr/local/bin
COPY --from=quake-n-bake /lib/ld-musl-*.so.1 /lib

ENTRYPOINT ["/usr/local/bin/q3"]
CMD ["/usr/local/bin/q3", "/usr/local/bin/grpc-health-probe"]
3 changes: 3 additions & 0 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,7 @@ k8s_yaml(
]
)
)

# Using tilt port_forward doesn't work with graceful pod termination, the
# port_forward is closed as soon as the deployment changes.
k8s_resource('quake-kube-chart', port_forwards='30001:8080')
66 changes: 28 additions & 38 deletions chart/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ spec:
selector:
matchLabels:
{{- include "chart.selectorLabels" . | nindent 6 }}
run: quake-kube
app: quake-kube
strategy:
rollingUpdate:
# This should be set to 0 if we only have 1 replica defined
maxUnavailable: 0
template:
metadata:
{{- with .Values.podAnnotations }}
Expand All @@ -21,6 +25,8 @@ spec:
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
version: "2"
app: quake-kube
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
Expand All @@ -29,56 +35,40 @@ spec:
serviceAccountName: {{ include "chart.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
terminationGracePeriodSeconds: 180
containers:
# - name: {{ .Chart.Name }}
- name: server
command:
- q3
- server
- run
- --config=/config/config.yaml
- --content-server=http://127.0.0.1:9090
- --agree-eula
- --shutdown-delay=10s
- --seed-content-url=http://content.quakejs.com
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- containerPort: 8080
# livenessProbe:
# httpGet:
# path: /
# port: http
livenessProbe:
exec:
command:
- grpc-health-probe
- -addr=localhost:8080
initialDelaySeconds: 30
failureThreshold: 1
successThreshold: 1
periodSeconds: 10 # this is 3 times the period on readinessProbe
readinessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 15
periodSeconds: 5
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
- name: content-server
command:
- q3
- content
- --seed-content-url=http://content.quakejs.com
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- containerPort: 9090
# livenessProbe:
# httpGet:
# path: /
# port: http
# readinessProbe:
# tcpSocket:
# port: 8080
# initialDelaySeconds: 15
# periodSeconds: 5
exec:
command:
- grpc-health-probe
- -addr=localhost:8080
initialDelaySeconds: 5
failureThreshold: 3
successThreshold: 1
periodSeconds: 3
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.volumeMounts }}
Expand Down
13 changes: 3 additions & 10 deletions chart/templates/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,9 @@ spec:
ports:
- port: 8080
targetPort: 8080
nodePort: 30001
nodePort: 30000
protocol: TCP
name: client
- port: 27960
targetPort: 27960
nodePort: 30003
name: server
- port: 9090
targetPort: 9090
nodePort: 30002
name: content
selector:
{{- include "chart.selectorLabels" . | nindent 4 }}
run: quake-kube
app: quake-kube
11 changes: 9 additions & 2 deletions cmd/q3/app/content/content.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package content

import (
"context"
"fmt"
"net"
"net/url"
Expand All @@ -18,6 +19,7 @@ import (

var opts struct {
Addr string
ServerAddr string
AssetsDir string
SeedContentURL string
}
Expand Down Expand Up @@ -50,16 +52,21 @@ func NewCommand() *cobra.Command {
}
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

m := mux.New(must.Must(net.Listen("tcp", opts.Addr)))
m.Register(quakecontent.NewRPCServer(opts.AssetsDir)).
m.Register(quakecontent.NewRPCServer(ctx, opts.AssetsDir, opts.ServerAddr)).
Match(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"))
m.Register(quakecontent.NewHTTPContentServer(opts.AssetsDir)).
m.Register(quakecontent.NewHTTPContentServer(ctx, opts.AssetsDir)).
Any()
fmt.Printf("Starting server %s\n", opts.Addr)
return m.Serve()
},
}
cmd.Flags().StringVarP(&opts.Addr, "addr", "a", ":9090", "address <host>:<port>")
cmd.Flags().
StringVar(&opts.ServerAddr, "server-addr", "", "(optional) dedicated server <host>:<port>")
cmd.Flags().StringVarP(&opts.AssetsDir, "assets-dir", "d", "assets", "assets directory")
cmd.Flags().StringVar(&opts.SeedContentURL, "seed-content-url", "", "seed content from another content server")
return cmd
Expand Down
7 changes: 6 additions & 1 deletion cmd/q3/app/proxy/proxy.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package proxy

import (
"context"
"fmt"
"net/http"

Expand Down Expand Up @@ -29,7 +30,11 @@ func NewCommand() *cobra.Command {
}
opts.ClientAddr = fmt.Sprintf("%s:8080", hostIPv4)
}
p, err := quakeclient.NewProxy(opts.ServerAddr)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

p, err := quakeclient.NewProxy(ctx, opts.ServerAddr)
if err != nil {
return err
}
Expand Down
115 changes: 115 additions & 0 deletions cmd/q3/app/run/run.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package server

import (
"context"
"errors"
"fmt"
"log"
"net"
"net/url"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"

"github.com/soheilhy/cmux"
"github.com/spf13/cobra"

quakeclient "github.com/ChrisRx/quake-kube/internal/quake/client"
"github.com/ChrisRx/quake-kube/internal/quake/content"
quakecontentutil "github.com/ChrisRx/quake-kube/internal/quake/content/util"
quakeserver "github.com/ChrisRx/quake-kube/internal/quake/server"
. "github.com/ChrisRx/quake-kube/pkg/must"
"github.com/ChrisRx/quake-kube/pkg/mux"
)

var opts struct {
ClientAddr string
ServerAddr string
ContentServer string
AcceptEula bool
AssetsDir string
ConfigFile string
WatchInterval time.Duration
ShutdownDelay time.Duration
SeedContentURL string
}

func NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "run",
Short: "run QuakeKube",
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) (err error) {
if !filepath.IsAbs(opts.AssetsDir) {
opts.AssetsDir, err = filepath.Abs(opts.AssetsDir)
if err != nil {
return err
}
}

if !opts.AcceptEula {
fmt.Println(quakeserver.Q3DemoEULA)
return errors.New("You must agree to the EULA to continue")
}

// Create the assets directory using the embedded game files first, and
// then any seeded content.
if err := quakeserver.ExtractGameFiles(opts.AssetsDir); err != nil {
return err
}
if opts.SeedContentURL != "" {
if err := quakecontentutil.DownloadAssets(Must(url.Parse(opts.SeedContentURL)), opts.AssetsDir); err != nil {
return err
}
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

sctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
defer stop()

qs := quakeserver.Server{
Addr: opts.ServerAddr,
ConfigFile: opts.ConfigFile,
Dir: opts.AssetsDir,
WatchInterval: opts.WatchInterval,
ShutdownDelay: opts.ShutdownDelay,
}
go func() {
defer cancel()

if err := qs.Start(sctx); err != nil {
log.Printf("quakeserver: %v\n", err)
}
}()

m := mux.New(Must(net.Listen("tcp", opts.ClientAddr)))
m.Register(content.NewRPCServer(ctx, opts.AssetsDir, opts.ServerAddr)).
Match(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"))
m.Register(content.NewHTTPContentServer(ctx, opts.AssetsDir)).
Match(cmux.PrefixMatcher("GET /assets"))
m.Register(Must(quakeclient.NewProxy(ctx, opts.ServerAddr))).
Match(cmux.HTTP1HeaderField("Upgrade", "websocket"))
m.Register(Must(quakeclient.NewHTTPClientServer(ctx, &quakeclient.Config{
ContentServerURL: opts.ContentServer,
ServerAddr: opts.ServerAddr,
}))).
Any()
fmt.Printf("Starting server %s\n", opts.ClientAddr)
return m.ServeAndWait()
},
}
cmd.Flags().StringVarP(&opts.ConfigFile, "config", "c", "", "server configuration file")
cmd.Flags().StringVar(&opts.ContentServer, "content-server", "http://127.0.0.1:8080", "content server url")
cmd.Flags().BoolVar(&opts.AcceptEula, "agree-eula", false, "agree to the Quake 3 demo EULA")
cmd.Flags().StringVar(&opts.AssetsDir, "assets-dir", "assets", "location for game files")
cmd.Flags().StringVar(&opts.ClientAddr, "client-addr", "0.0.0.0:8080", "client address <host>:<port>")
cmd.Flags().StringVar(&opts.ServerAddr, "server-addr", "0.0.0.0:27960", "dedicated server <host>:<port>")
cmd.Flags().DurationVar(&opts.WatchInterval, "watch-interval", 15*time.Second, "watch interval for config file")
cmd.Flags().DurationVar(&opts.ShutdownDelay, "shutdown-delay", 1*time.Minute, "delay for graceful shutdown")
cmd.Flags().StringVar(&opts.SeedContentURL, "seed-content-url", "", "seed content from another content server")
return cmd
}
4 changes: 2 additions & 2 deletions cmd/q3/app/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ func NewCommand() *cobra.Command {
}()

m := mux.New(must.Must(net.Listen("tcp", opts.ClientAddr)))
m.Register(must.Must(quakeclient.NewProxy(opts.ServerAddr))).
m.Register(must.Must(quakeclient.NewProxy(ctx, opts.ServerAddr))).
Match(cmux.HTTP1HeaderField("Upgrade", "websocket"))
m.Register(must.Must(quakeclient.NewHTTPClientServer(&quakeclient.Config{
m.Register(must.Must(quakeclient.NewHTTPClientServer(ctx, &quakeclient.Config{
ContentServerURL: opts.ContentServer,
ServerAddr: opts.ServerAddr,
}))).
Expand Down
2 changes: 2 additions & 0 deletions cmd/q3/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
q3cmd "github.com/ChrisRx/quake-kube/cmd/q3/app/cmd"
q3content "github.com/ChrisRx/quake-kube/cmd/q3/app/content"
q3proxy "github.com/ChrisRx/quake-kube/cmd/q3/app/proxy"
q3run "github.com/ChrisRx/quake-kube/cmd/q3/app/run"
q3server "github.com/ChrisRx/quake-kube/cmd/q3/app/server"
q3upload "github.com/ChrisRx/quake-kube/cmd/q3/app/upload"
)
Expand All @@ -25,6 +26,7 @@ func main() {
q3cmd.NewCommand(),
q3content.NewCommand(),
q3proxy.NewCommand(),
q3run.NewCommand(),
q3server.NewCommand(),
q3upload.NewCommand(),
)
Expand Down
4 changes: 2 additions & 2 deletions docs/src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ QuakeKube is a Kubernetes-ified version of [Quake 3](https://en.wikipedia.org/wi

This uses files from the Quake 3 Demo. The demo doesn't allow custom games, so while you can add new maps, you couldn't say load up [Urban Terror](https://www.moddb.com/mods/urban-terror). I think with pak files from a full version of the game this would be possible, but I haven't tried it (maybe one day).

Another caveat is that the copy running in the browser is using [QuakeJS](https://github.com/inolen/quakejs). This version is an older verison of [ioquake3](https://github.com/ioquake/ioq3) built with [emscripten](https://emscripten.org/) and it does not appear to be supported, nor does it still compile with any newer versions of empscripten. I believe this could be made to work again, but I haven't personally looked at how involved it would be. It is worth noting that any non-browser versions of the Quake 3 could connect to the dedicated servers.
Another caveat is that the copy running in the browser is using [QuakeJS](https://github.com/inolen/quakejs). This version is an older verison of [ioquake3](https://github.com/ioquake/ioq3) built with [emscripten](https://emscripten.org/) and it does not appear to be supported, nor does it still compile with any newer versions of emscripten. I believe this could be made to work again, but I haven't personally looked at how involved it would be. It is worth noting that any non-browser versions of the Quake 3 could connect to the dedicated servers.

## What is this project for?

This was just made for fun and learning. It isn't trying to be a complete solution for managing Quake 3 on Kubernetes, and I am using it now as a repo of common patterns and best practices (IMO) for Go/Kubernetes projects. I think some fun additions though might be adding code to work as a Quake 3 master server, a server that exchanges information with the game client about what dedicated game servers, and making a controller/crd that ties it all together.
This was just made for fun and learning. It isn't trying to be a complete solution for managing Quake 3 on Kubernetes, and I am using it now as a repo of common patterns and best practices (IMO) for Go/Kubernetes projects. I think some fun additions though might be adding code to work as a Quake 3 master server, a server that exchanges information with the game client about what dedicated game servers are available, and making a controller/crd that ties it all together.
2 changes: 1 addition & 1 deletion docs/src/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ QuakeKube makes use of [ioquake](https://www.ioquake.org) for the Quake 3 dedica

The client/server protocol of Quake 3 uses UDP to synchronize game state. Browsers do not natively support sending UDP packets so QuakeJS wraps the client and dedicated server net code in websockets, allowing the browser-based clients to send messages and enable multiplayer for other clients. This ends up preventing the browser client from using any other Quake 3 dedicated server. In order to use other Quake 3 dedicated servers, a proxy handles websocket traffic coming from browser clients and translates that into UDP to the backend. This gives the flexibility of being able to talk to other existing Quake 3 servers, but also allows using ioquake (instead of the javascript translation of it), which uses *considerably* less CPU and memory.

QuakeKube also uses a cool trick with [cmux](https://github.com/cockroachdb/cmux) to multiplex the client and websocket traffic into the same connection. Having all the traffic go through the same address makes routing a client to its backend much easier (since it can just use its `document.location.host`).
QuakeKube also uses a cool trick with [cmux](https://github.com/soheilhy/cmux) to multiplex the client and websocket traffic into the same connection. Having all the traffic go through the same address makes routing a client to its backend much easier (since it can just use its `document.location.host`).

## Quake 3 demo EULA

Expand Down
Loading

0 comments on commit b57e37d

Please sign in to comment.