From 4b51f109f0a1f155e3e7afd63dadecd30f22129b Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Wed, 30 Jan 2019 17:37:45 -0800 Subject: [PATCH 01/21] quay standalone git URL fix, CI yaml for kubernetes added, push to trigger build --- Dockerfile_standalone | 3 +- README.md | 2 +- build.sh | 1 + ci_k8s.yaml | 215 ++++++++++++++++++++++++++++++++++++++++++ docs/quay_io_ci.md | 9 ++ 5 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 ci_k8s.yaml create mode 100644 docs/quay_io_ci.md diff --git a/Dockerfile_standalone b/Dockerfile_standalone index d321683e3..eb234b6a5 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -26,7 +26,8 @@ WORKDIR /project/src/github.com/leaf-ai/studio-go-runner RUN mkdir $GOPATH/bin && \ (curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh) && \ - git config --global url."git://github.com".insteadOf "https://github.com" && \ + git config --global url."https://github.com/".insteadOf git@github.com: && \ + git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ rm /usr/bin/nvidia-* diff --git a/README.md b/README.md index eef02acb0..0653979fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # studio-go-runner -Version: 0.9.9 +Version: 0.9.10-feature-184-flux-trial-1goyyl [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/leaf-ai/studio-go-runner/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/leaf-ai/studio-go-runner)](https://goreportcard.com/report/leaf-ai/studio-go-runner)[![DepShield Badge](https://depshield.sonatype.org/badges/leaf-ai/studio-go-runner/depshield.svg)](https://depshield.github.io) diff --git a/build.sh b/build.sh index 453fabc7a..b1b0469e6 100755 --- a/build.sh +++ b/build.sh @@ -90,6 +90,7 @@ trap Tidyup 1 2 3 15 export SEMVER=`semver` export GIT_BRANCH=`echo '{{.duat.gitBranch}}'|stencil - | tr '_' '-' | tr '\/' '-'` +GIT_COMMIT=`git rev-parse HEAD` export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 diff --git a/ci_k8s.yaml b/ci_k8s.yaml new file mode 100644 index 000000000..a5b06dbef --- /dev/null +++ b/ci_k8s.yaml @@ -0,0 +1,215 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ci-go-runner-{{.duat.gitHash}} +--- +# The following role allows the runner under test to interact with the working state config map +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-cluster-admin +subjects: + - kind: ServiceAccount + name: default + namespace: ci-go-runner-{{.duat.gitHash}} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: build-env + namespace: ci-go-runner-{{.duat.gitHash}} +data: + LOGXI: "*=INF" + RABBITMQ_DEFAULT_USER: "UserUser" + RABBITMQ_DEFAULT_PASS: "PasswordPassword" + MINIO_ACCESS_KEY: "UserUser" + MINIO_SECRET_KEY: "PasswordPassword" + MINIO_TEST_SERVER: "${MINIO_SERVICE_SERVICE_HOST}:${MINIO_SERVICE_SERVICE_PORT}" + AMQP_URL: "amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_SERVICE_SERVICE_HOST}:${RABBITMQ_SERVICE_SERVICE_PORT}/%2f?connection_attempts=2&retry_delay=.5&socket_timeout=5" +--- +# This service exposes rabbit MQ to the cluster members +apiVersion: v1 +kind: Service +metadata: + labels: + component: rabbitmq + name: rabbitmq-service + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + ports: + - name: rmq-client + port: 5672 + - name: rmq-admin + port: 15672 + selector: + app: taskQueue + component: rabbitmq +--- +# The replication controller encapsulates the pod(s) used to run RabbitMQ +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + component: rabbitmq + name: rabbitmq-controller + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + replicas: 1 + template: + metadata: + labels: + app: taskQueue + component: rabbitmq + spec: + containers: + - image: rabbitmq + name: rabbitmq + ports: + - containerPort: 5672 + - containerPort: 15672 + resources: + limits: + cpu: 1 + envFrom: + - configMapRef: + name: build-env + lifecycle: + postStart: + exec: + command: + - "/bin/bash" + - "-c" + - > + set -euo pipefail ; + IFS=$'\n\t' ; + echo "Starting the install of the management plugin" ; + sleep 30 ; + rabbitmq-plugins enable rabbitmq_management ; + apt-get -y update ; apt-get install -y wget python ; + wget -q -O /usr/local/bin/rabbitmqadmin http://localhost:15672/cli/rabbitmqadmin ; + chmod +x /usr/local/bin/rabbitmqadmin +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + # This name uniquely identifies the PVC. Will be used in deployment below. + name: minio-pv-claim + labels: + app: minio-storage-claim + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + # Read more about access modes here: https://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes + accessModes: + - ReadWriteOnce + resources: + # This is the request for storage. Should be available in the cluster. + requests: + storage: 10Gi + # Uncomment and add storageClass specific to your requirements below. Read more https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1 + #storageClassName: +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + # This name uniquely identifies the Deployment + name: minio-deployment + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + strategy: + type: Recreate + template: + metadata: + labels: + # Label is used as selector in the service. + app: minio + spec: + # Refer to the PVC created earlier + volumes: + - name: storage + persistentVolumeClaim: + # Name of the PVC created earlier + claimName: minio-pv-claim + containers: + - name: minio + # Pulls the default Minio image from Docker Hub + image: minio/minio + args: + - server + - /storage + envFrom: + - configMapRef: + name: build-env + ports: + - containerPort: 9000 + # Mount the volume into the pod + volumeMounts: + - name: storage # must match the volume name, above + mountPath: "/storage" +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-service + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio +--- +# This job starts a run to completion runner to perform the integration and end-to-end testing +apiVersion: batch/v1 +kind: Job +metadata: + name: build + namespace: ci-go-runner-{{.duat.gitHash}} +spec: + template: + spec: + volumes: + - name: podinfo + downwardAPI: + items: + - path: "namespace" + fieldRef: + fieldPath: metadata.namespace + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - path: "labels" + fieldRef: + fieldPath: metadata.labels + containers: + - name: build + env: + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + envFrom: + - configMapRef: + name: build-env + image: quay.io/leaf_ai_dockerhub/studio-go-runner-standalone-build:{{.duat.version}} + imagePullPolicy: Always + resources: + limits: + memory: "1024Mi" + cpu: 1 + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + readOnly: false + restartPolicy: Never + backoffLimit: 4 diff --git a/docs/quay_io_ci.md b/docs/quay_io_ci.md new file mode 100644 index 000000000..75731423c --- /dev/null +++ b/docs/quay_io_ci.md @@ -0,0 +1,9 @@ +# Continuous Integration Setup + +This document describes setting up a CI pipline that can be used to prepare releases for studio go runner. + +studio go runner is designed to run in resource intensive environments using GPU enabled machines and so providing a free hosted pipeline for CI/CD is cost prohibitive. As an alternative parties interested in studio go runner can make use of quay.io hosted images built automatically to then trigger test and integration downstream in their own Kubernetes provisioned clusters. + +This document contains instructions detailing how a quay.io and private Kubernetes cluster can be created using the same method that the project maintainers utilize. + +stencil -input ci_k8s.yaml | kubectl apply -f - From f65ce27f476a41eedeeb71369300fd96a55be682 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Fri, 1 Feb 2019 17:25:54 -0800 Subject: [PATCH 02/21] start work on the CI keel implementation --- Dockerfile_standalone | 3 +- README.md | 2 +- build.sh | 101 ++---------------- ci.sh | 204 ++++++++++++++++++++++++++++++++++++ ci_k8s.yaml => ci_keel.yaml | 0 docs/quay_io_ci.md | 42 +++++++- docs/workstation_k8s.md | 2 +- 7 files changed, 253 insertions(+), 101 deletions(-) create mode 100755 ci.sh rename ci_k8s.yaml => ci_keel.yaml (100%) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index eb234b6a5..42e240341 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -29,11 +29,12 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://github.com/".insteadOf git@github.com: && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ + go get github.com/karlmutch/petname && \ rm /usr/bin/nvidia-* COPY . /project/src/github.com/leaf-ai/studio-go-runner/ -CMD /bin/bash -c 'set -e ; set -o pipefail ; (dep ensure && go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Build Success" || echo "Build Failure") 2>&1 ' +CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions LABEL vendor="Open Source" diff --git a/README.md b/README.md index 0653979fc..248076e77 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # studio-go-runner -Version: 0.9.10-feature-184-flux-trial-1goyyl +Version: 0.9.10-feature-184-flux-trial-1gpk3e [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/leaf-ai/studio-go-runner/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/leaf-ai/studio-go-runner)](https://goreportcard.com/report/leaf-ai/studio-go-runner)[![DepShield Badge](https://depshield.sonatype.org/badges/leaf-ai/studio-go-runner/depshield.svg)](https://depshield.github.io) diff --git a/build.sh b/build.sh index b1b0469e6..dab66a724 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,5 @@ #!/bin/bash -e -[ -z "$USER" ] && echo "Error: env variable USER must be set" && exit 1; [ -z "$GOPATH" ] && echo "Error: env variable GOPATH must be set" && exit 1; [ -z "$GITHUB_TOKEN" ] && echo "Warning : env variable GITHUB_TOKEN should be set in the event that a release is to be generated" ; [ -z ${azure_registry_name+x} ] && echo "Warning : env variable azure_registry_name not set"; @@ -62,6 +61,11 @@ go get -u github.com/golang/dep/cmd/dep dep ensure +wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 +wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 +chmod +x $GOPATH/bin/semver +chmod +x $GOPATH/bin/stencil + bash -c "while true; do echo \$(date) - building ...; sleep 180s; done" & PING_LOOP_PID=$! @@ -94,27 +98,9 @@ GIT_COMMIT=`git rev-parse HEAD` export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 -# Build the base image that other images will derive from for development style images -docker build -t leafai/studio-go-runner-dev-base:0.0.0 -f Dockerfile_base . - travis_fold start "build.image" travis_time_start - stencil -input Dockerfile | docker build -t leafai/studio-go-runner-build:$GIT_BRANCH - - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - # Information about safely working with temporary files in shell scripts can be found at - # https://dev.to/philgibbs/avoiding-temporary-files-in-shell-scripts - { - stencil -input Dockerfile_standalone > $working_file - [[ $? != 0 ]] && ExitWithError "stencil processing of Dockerfile_standalone failed" - } | tee $working_file > /dev/null - [[ $? != 0 ]] && ExitWithError "Error writing to $working_file" - docker build -t leafai/studio-go-runner-standalone-build:$GIT_BRANCH -f $working_file . - rm -f $working_file - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH leafai/studio-go-runner-standalone-build - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH localhost:32000/leafai/studio-go-runner-standalone-build + set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 exit_code=$? if [ $exit_code -ne 0 ]; then exit $exit_code @@ -126,79 +112,4 @@ if [ $exit_code -ne 0 ]; then exit $exit_code fi -# Running build.go inside of a container will result in a compilation, light testing, and release however no docker images -travis_fold start "build" - travis_time_start - docker run -e TERM="$TERM" -e LOGXI="$LOGXI" -e LOGXI_FORMAT="$LOGXI_FORMAT" -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GOPATH:/project leafai/studio-go-runner-build:$GIT_BRANCH - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - travis_time_finish -travis_fold end "build" - -if [ $exit_code -ne 0 ]; then - exit $exit_code -fi - -# Automatically produces images without compilation, or releases when run outside of a container -travis_fold start "image.build" - travis_time_start - go run -tags=NO_CUDA ./build.go -image-only -r -dirs cmd - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - travis_time_finish -travis_fold end "image.build" - -if [ $exit_code -ne 0 ]; then - exit $exit_code -fi - -travis_fold start "image.push" - travis_time_start - if docker image inspect leaf-ai/studio-go-runner/runner:$SEMVER 2>/dev/null 1>/dev/null; then - if type docker 2>/dev/null ; then - docker login docker.io - if [ $? -eq 0 ]; then - docker tag leaf-ai/studio-go-runner/runner:$SEMVER leafai/studio-go-runner:$SEMVER - docker tag leafai/studio-go-runner-dev-base:0.0.0 leafai/studio-go-runner-dev-base:$GIT_BRANCH - - docker push leafai/studio-go-runner:$SEMVER - docker push leafai/studio-go-runner-dev-base:0.0.0 - docker push leafai/studio-go-runner-dev-base:$GIT_BRANCH - docker push leafai/studio-go-runner-standalone-build:$GIT_BRANCH - fi - fi - if type aws 2>/dev/null ; then - `aws ecr get-login --no-include-email` - if [ $? -eq 0 ]; then - account=`aws sts get-caller-identity --output text --query Account` - if [ $? -eq 0 ]; then - docker tag leafai/studio-go-runner:$SEMVER $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER - docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER - - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH - docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH - fi - fi - fi - if [ -z ${azure_registry_name+x} ]; then - : - else - if type az 2>/dev/null; then - if az acr login --name $azure_registry_name; then - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH - docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH - - docker tag leafai/studio-go-runner:$SEMVER $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER - docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER - fi - fi - fi - fi - travis_time_finish -travis_fold end "image.push" - exit 0 diff --git a/ci.sh b/ci.sh new file mode 100755 index 000000000..b1b0469e6 --- /dev/null +++ b/ci.sh @@ -0,0 +1,204 @@ +#!/bin/bash -e + +[ -z "$USER" ] && echo "Error: env variable USER must be set" && exit 1; +[ -z "$GOPATH" ] && echo "Error: env variable GOPATH must be set" && exit 1; +[ -z "$GITHUB_TOKEN" ] && echo "Warning : env variable GITHUB_TOKEN should be set in the event that a release is to be generated" ; +[ -z ${azure_registry_name+x} ] && echo "Warning : env variable azure_registry_name not set"; + +if [[ ":$PATH:" != *":$GOPATH/bin:"* ]]; then + export PATH=$PATH:$GOPATH/bin +fi + +export LOGXI="*=DBG" +export LOGXI_FORMAT="happy,maxcol=1024" + +[ -z "$TERM" ] && export TERM=xterm+256color; + +if [ -n "$(type -t travis_fold)" ] && [ "$(type -t travis_fold)" = function ]; then + type travis_fold + type travis_nanoseconds + type travis_time_start + type travis_time_finish +: +else +declare -i travis_start_time +declare -i travis_end_time + + function travis_nanoseconds () { + local cmd="date"; + local format="+%s%N"; + local os=$(uname); + if hash gdate > /dev/null 2>&1; then + cmd="gdate"; + else + if [[ "$os" = Darwin ]]; then + format="+%s000000000"; + fi; + fi; + $cmd -u $format + } + + function travis_fold() { + local action=$1; + local name=$2; + echo -en "travis_fold:${action}:${name}\r${ANSI_CLEAR}" + } + function travis_time_start() { + travis_timer_id=$(printf %08x $(( RANDOM * RANDOM ))); + travis_start_time=$(travis_nanoseconds); + echo -en "travis_time:start:$travis_timer_id\r${ANSI_CLEAR}" + } + function travis_time_finish() { + local result=$?; + travis_end_time=$(travis_nanoseconds); + local duration=$(($travis_end_time-$travis_start_time)); + echo -en "\ntravis_time:end:$travis_timer_id:start=$travis_start_time,finish=$travis_end_time,duration=$duration\r${ANSI_CLEAR}"; + return $result + + } +fi + +go get -u github.com/golang/dep/cmd/dep + +dep ensure + +bash -c "while true; do echo \$(date) - building ...; sleep 180s; done" & +PING_LOOP_PID=$! + +function cleanup { + # nicely terminate the ping output loop + kill $PING_LOOP_PID +} +trap cleanup EXIT + +function ExitWithError +{ + echo "$*">&2 + rm -f $working_file + exit 1 +} + +function Tidyup +{ + ExitWithError "Abort" +} + +umask 077 +working_file=$$.studio-go-runner-working +rm -f $working_file +trap Tidyup 1 2 3 15 + +export SEMVER=`semver` +export GIT_BRANCH=`echo '{{.duat.gitBranch}}'|stencil - | tr '_' '-' | tr '\/' '-'` +GIT_COMMIT=`git rev-parse HEAD` +export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log +exit_code=0 + +# Build the base image that other images will derive from for development style images +docker build -t leafai/studio-go-runner-dev-base:0.0.0 -f Dockerfile_base . + +travis_fold start "build.image" + travis_time_start + stencil -input Dockerfile | docker build -t leafai/studio-go-runner-build:$GIT_BRANCH - + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + # Information about safely working with temporary files in shell scripts can be found at + # https://dev.to/philgibbs/avoiding-temporary-files-in-shell-scripts + { + stencil -input Dockerfile_standalone > $working_file + [[ $? != 0 ]] && ExitWithError "stencil processing of Dockerfile_standalone failed" + } | tee $working_file > /dev/null + [[ $? != 0 ]] && ExitWithError "Error writing to $working_file" + docker build -t leafai/studio-go-runner-standalone-build:$GIT_BRANCH -f $working_file . + rm -f $working_file + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH leafai/studio-go-runner-standalone-build + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH localhost:32000/leafai/studio-go-runner-standalone-build + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + travis_time_finish +travis_fold end "build.image" + +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + +# Running build.go inside of a container will result in a compilation, light testing, and release however no docker images +travis_fold start "build" + travis_time_start + docker run -e TERM="$TERM" -e LOGXI="$LOGXI" -e LOGXI_FORMAT="$LOGXI_FORMAT" -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GOPATH:/project leafai/studio-go-runner-build:$GIT_BRANCH + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + travis_time_finish +travis_fold end "build" + +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + +# Automatically produces images without compilation, or releases when run outside of a container +travis_fold start "image.build" + travis_time_start + go run -tags=NO_CUDA ./build.go -image-only -r -dirs cmd + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + travis_time_finish +travis_fold end "image.build" + +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + +travis_fold start "image.push" + travis_time_start + if docker image inspect leaf-ai/studio-go-runner/runner:$SEMVER 2>/dev/null 1>/dev/null; then + if type docker 2>/dev/null ; then + docker login docker.io + if [ $? -eq 0 ]; then + docker tag leaf-ai/studio-go-runner/runner:$SEMVER leafai/studio-go-runner:$SEMVER + docker tag leafai/studio-go-runner-dev-base:0.0.0 leafai/studio-go-runner-dev-base:$GIT_BRANCH + + docker push leafai/studio-go-runner:$SEMVER + docker push leafai/studio-go-runner-dev-base:0.0.0 + docker push leafai/studio-go-runner-dev-base:$GIT_BRANCH + docker push leafai/studio-go-runner-standalone-build:$GIT_BRANCH + fi + fi + if type aws 2>/dev/null ; then + `aws ecr get-login --no-include-email` + if [ $? -eq 0 ]; then + account=`aws sts get-caller-identity --output text --query Account` + if [ $? -eq 0 ]; then + docker tag leafai/studio-go-runner:$SEMVER $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER + docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER + + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH + docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH + fi + fi + fi + if [ -z ${azure_registry_name+x} ]; then + : + else + if type az 2>/dev/null; then + if az acr login --name $azure_registry_name; then + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH + docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH + + docker tag leafai/studio-go-runner:$SEMVER $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER + docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER + fi + fi + fi + fi + travis_time_finish +travis_fold end "image.push" + +exit 0 diff --git a/ci_k8s.yaml b/ci_keel.yaml similarity index 100% rename from ci_k8s.yaml rename to ci_keel.yaml diff --git a/docs/quay_io_ci.md b/docs/quay_io_ci.md index 75731423c..431f05b64 100644 --- a/docs/quay_io_ci.md +++ b/docs/quay_io_ci.md @@ -2,8 +2,44 @@ This document describes setting up a CI pipline that can be used to prepare releases for studio go runner. -studio go runner is designed to run in resource intensive environments using GPU enabled machines and so providing a free hosted pipeline for CI/CD is cost prohibitive. As an alternative parties interested in studio go runner can make use of quay.io hosted images built automatically to then trigger test and integration downstream in their own Kubernetes provisioned clusters. +studio go runner is designed to run in resource intensive environments using GPU enabled machines and so providing a free hosted pipeline for CI/CD is cost prohibitive. As an alternative parties interested in studio go runner can make use of quay.io hosted images built automatically and are then pulled into a test and integration downstream Kubernetes provisioned cluster they own. This allows testing to be done using the CI pipeline on both local laptops, workstations and in cloud or data center environments. -This document contains instructions detailing how a quay.io and private Kubernetes cluster can be created using the same method that the project maintainers utilize. +This document contains instructions that can be used for hardware configurations that individual users to large scale enterprises can use without incuring monthly charges from third party providers. These instructions first detail how a quay.io trigger can be setup to trigger builds on github commits. Instructions then detail how to make use of Keel, https://keel.sh/, to pull CI images into a cluster and run the pipeline. -stencil -input ci_k8s.yaml | kubectl apply -f - +Optional GITHUB_TOKEN secrets are added to the cluster + +Annotations updated via stencil with gitHash etc and also with desired regular expression or keel semver policy +namespace is generated and used for the bootstrapped build +stencil -input ci_keel.yaml | kubectl apply -f - +git commit and push to start things rolling +Keel repo polling triggers build + +built container in build pod removes itself from keel using Kubernetes preStartHook by renaming annotations +``` +Using downward API +metadata.annotations['myannotation'] +``` + +build pod starts +new namespace generated for next listener +``` +github.com/docker/docker/pkg/namesgenerator +Loop creating namespace with uuid annotation and then validating we owned it +``` + +container used the included ci_keel and injects the annotations from itself to create the next listening deployment +``` +stencil with variables in a file for all annotations now renamed for their real keys +``` + +new namspace with deployment using ci_keel.yaml is dispatched +build starts in our now liberated namespace + +build finishes +set ReplicationControllers and deployment .spec.replicas to 0 +``` +kubectl scale --namespace build-test-k8s-local --replicas=0 deployment/minio-deployment +kubectl scale --namespace build-test-k8s-local --replicas=0 rc/rabbitmq-controller +``` + +and the build then sits until such time as we decide on a policy for self destruction like push results back to github, at which point we dispose of the unique namespace used for the build diff --git a/docs/workstation_k8s.md b/docs/workstation_k8s.md index 965efadf1..75ffa0b25 100644 --- a/docs/workstation_k8s.md +++ b/docs/workstation_k8s.md @@ -53,7 +53,7 @@ Having obtained a copy of the studio go runner code the next step is to build an ``` cd ~/projects/src/github.com/leaf-ai/studio-go-runner -docker build -t leafai/studio-go-runner-standalone-build Dockerfile_standalone +docker build -t leafai/studio-go-runner-standalone-build -f Dockerfile_standalone . ``` ### Kubernetes test deployment and results collection From e49317d45e0ac50b6417d7452f99a62fe2a9809d Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Fri, 1 Feb 2019 17:32:58 -0800 Subject: [PATCH 03/21] ci.sh and build.sh got switched, switch them back --- build.sh | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++---- ci.sh | 101 ++++--------------------------------------------------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/build.sh b/build.sh index dab66a724..b1b0469e6 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,6 @@ #!/bin/bash -e +[ -z "$USER" ] && echo "Error: env variable USER must be set" && exit 1; [ -z "$GOPATH" ] && echo "Error: env variable GOPATH must be set" && exit 1; [ -z "$GITHUB_TOKEN" ] && echo "Warning : env variable GITHUB_TOKEN should be set in the event that a release is to be generated" ; [ -z ${azure_registry_name+x} ] && echo "Warning : env variable azure_registry_name not set"; @@ -61,11 +62,6 @@ go get -u github.com/golang/dep/cmd/dep dep ensure -wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 -wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 -chmod +x $GOPATH/bin/semver -chmod +x $GOPATH/bin/stencil - bash -c "while true; do echo \$(date) - building ...; sleep 180s; done" & PING_LOOP_PID=$! @@ -98,9 +94,27 @@ GIT_COMMIT=`git rev-parse HEAD` export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 +# Build the base image that other images will derive from for development style images +docker build -t leafai/studio-go-runner-dev-base:0.0.0 -f Dockerfile_base . + travis_fold start "build.image" travis_time_start - set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 + stencil -input Dockerfile | docker build -t leafai/studio-go-runner-build:$GIT_BRANCH - + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + # Information about safely working with temporary files in shell scripts can be found at + # https://dev.to/philgibbs/avoiding-temporary-files-in-shell-scripts + { + stencil -input Dockerfile_standalone > $working_file + [[ $? != 0 ]] && ExitWithError "stencil processing of Dockerfile_standalone failed" + } | tee $working_file > /dev/null + [[ $? != 0 ]] && ExitWithError "Error writing to $working_file" + docker build -t leafai/studio-go-runner-standalone-build:$GIT_BRANCH -f $working_file . + rm -f $working_file + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH leafai/studio-go-runner-standalone-build + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH localhost:32000/leafai/studio-go-runner-standalone-build exit_code=$? if [ $exit_code -ne 0 ]; then exit $exit_code @@ -112,4 +126,79 @@ if [ $exit_code -ne 0 ]; then exit $exit_code fi +# Running build.go inside of a container will result in a compilation, light testing, and release however no docker images +travis_fold start "build" + travis_time_start + docker run -e TERM="$TERM" -e LOGXI="$LOGXI" -e LOGXI_FORMAT="$LOGXI_FORMAT" -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GOPATH:/project leafai/studio-go-runner-build:$GIT_BRANCH + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + travis_time_finish +travis_fold end "build" + +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + +# Automatically produces images without compilation, or releases when run outside of a container +travis_fold start "image.build" + travis_time_start + go run -tags=NO_CUDA ./build.go -image-only -r -dirs cmd + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + travis_time_finish +travis_fold end "image.build" + +if [ $exit_code -ne 0 ]; then + exit $exit_code +fi + +travis_fold start "image.push" + travis_time_start + if docker image inspect leaf-ai/studio-go-runner/runner:$SEMVER 2>/dev/null 1>/dev/null; then + if type docker 2>/dev/null ; then + docker login docker.io + if [ $? -eq 0 ]; then + docker tag leaf-ai/studio-go-runner/runner:$SEMVER leafai/studio-go-runner:$SEMVER + docker tag leafai/studio-go-runner-dev-base:0.0.0 leafai/studio-go-runner-dev-base:$GIT_BRANCH + + docker push leafai/studio-go-runner:$SEMVER + docker push leafai/studio-go-runner-dev-base:0.0.0 + docker push leafai/studio-go-runner-dev-base:$GIT_BRANCH + docker push leafai/studio-go-runner-standalone-build:$GIT_BRANCH + fi + fi + if type aws 2>/dev/null ; then + `aws ecr get-login --no-include-email` + if [ $? -eq 0 ]; then + account=`aws sts get-caller-identity --output text --query Account` + if [ $? -eq 0 ]; then + docker tag leafai/studio-go-runner:$SEMVER $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER + docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER + + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH + docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH + fi + fi + fi + if [ -z ${azure_registry_name+x} ]; then + : + else + if type az 2>/dev/null; then + if az acr login --name $azure_registry_name; then + docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH + docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH + + docker tag leafai/studio-go-runner:$SEMVER $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER + docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER + fi + fi + fi + fi + travis_time_finish +travis_fold end "image.push" + exit 0 diff --git a/ci.sh b/ci.sh index b1b0469e6..dab66a724 100755 --- a/ci.sh +++ b/ci.sh @@ -1,6 +1,5 @@ #!/bin/bash -e -[ -z "$USER" ] && echo "Error: env variable USER must be set" && exit 1; [ -z "$GOPATH" ] && echo "Error: env variable GOPATH must be set" && exit 1; [ -z "$GITHUB_TOKEN" ] && echo "Warning : env variable GITHUB_TOKEN should be set in the event that a release is to be generated" ; [ -z ${azure_registry_name+x} ] && echo "Warning : env variable azure_registry_name not set"; @@ -62,6 +61,11 @@ go get -u github.com/golang/dep/cmd/dep dep ensure +wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 +wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 +chmod +x $GOPATH/bin/semver +chmod +x $GOPATH/bin/stencil + bash -c "while true; do echo \$(date) - building ...; sleep 180s; done" & PING_LOOP_PID=$! @@ -94,27 +98,9 @@ GIT_COMMIT=`git rev-parse HEAD` export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 -# Build the base image that other images will derive from for development style images -docker build -t leafai/studio-go-runner-dev-base:0.0.0 -f Dockerfile_base . - travis_fold start "build.image" travis_time_start - stencil -input Dockerfile | docker build -t leafai/studio-go-runner-build:$GIT_BRANCH - - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - # Information about safely working with temporary files in shell scripts can be found at - # https://dev.to/philgibbs/avoiding-temporary-files-in-shell-scripts - { - stencil -input Dockerfile_standalone > $working_file - [[ $? != 0 ]] && ExitWithError "stencil processing of Dockerfile_standalone failed" - } | tee $working_file > /dev/null - [[ $? != 0 ]] && ExitWithError "Error writing to $working_file" - docker build -t leafai/studio-go-runner-standalone-build:$GIT_BRANCH -f $working_file . - rm -f $working_file - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH leafai/studio-go-runner-standalone-build - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH localhost:32000/leafai/studio-go-runner-standalone-build + set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 exit_code=$? if [ $exit_code -ne 0 ]; then exit $exit_code @@ -126,79 +112,4 @@ if [ $exit_code -ne 0 ]; then exit $exit_code fi -# Running build.go inside of a container will result in a compilation, light testing, and release however no docker images -travis_fold start "build" - travis_time_start - docker run -e TERM="$TERM" -e LOGXI="$LOGXI" -e LOGXI_FORMAT="$LOGXI_FORMAT" -e GITHUB_TOKEN=$GITHUB_TOKEN -v $GOPATH:/project leafai/studio-go-runner-build:$GIT_BRANCH - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - travis_time_finish -travis_fold end "build" - -if [ $exit_code -ne 0 ]; then - exit $exit_code -fi - -# Automatically produces images without compilation, or releases when run outside of a container -travis_fold start "image.build" - travis_time_start - go run -tags=NO_CUDA ./build.go -image-only -r -dirs cmd - exit_code=$? - if [ $exit_code -ne 0 ]; then - exit $exit_code - fi - travis_time_finish -travis_fold end "image.build" - -if [ $exit_code -ne 0 ]; then - exit $exit_code -fi - -travis_fold start "image.push" - travis_time_start - if docker image inspect leaf-ai/studio-go-runner/runner:$SEMVER 2>/dev/null 1>/dev/null; then - if type docker 2>/dev/null ; then - docker login docker.io - if [ $? -eq 0 ]; then - docker tag leaf-ai/studio-go-runner/runner:$SEMVER leafai/studio-go-runner:$SEMVER - docker tag leafai/studio-go-runner-dev-base:0.0.0 leafai/studio-go-runner-dev-base:$GIT_BRANCH - - docker push leafai/studio-go-runner:$SEMVER - docker push leafai/studio-go-runner-dev-base:0.0.0 - docker push leafai/studio-go-runner-dev-base:$GIT_BRANCH - docker push leafai/studio-go-runner-standalone-build:$GIT_BRANCH - fi - fi - if type aws 2>/dev/null ; then - `aws ecr get-login --no-include-email` - if [ $? -eq 0 ]; then - account=`aws sts get-caller-identity --output text --query Account` - if [ $? -eq 0 ]; then - docker tag leafai/studio-go-runner:$SEMVER $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER - docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/runner:$SEMVER - - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH - docker push $account.dkr.ecr.us-west-2.amazonaws.com/leafai/studio-go-runner/standalone-build:$GIT_BRANCH - fi - fi - fi - if [ -z ${azure_registry_name+x} ]; then - : - else - if type az 2>/dev/null; then - if az acr login --name $azure_registry_name; then - docker tag leafai/studio-go-runner-standalone-build:$GIT_BRANCH $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH - docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner-standalone-build:$GIT_BRANCH - - docker tag leafai/studio-go-runner:$SEMVER $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER - docker push $azure_registry_name.azurecr.io/leafai/studio-go-runner:$SEMVER - fi - fi - fi - fi - travis_time_finish -travis_fold end "image.push" - exit 0 From da7c39173fcb16b366046d8714f719cb2a691a03 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 13:46:34 -0800 Subject: [PATCH 04/21] Keel test --- Dockerfile_standalone | 2 +- build.sh | 2 ++ ci.sh | 8 ++++++++ ci_keel.yaml | 18 ++++++++++++------ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 42e240341..149ed2ea6 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,4 +37,4 @@ COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source" +LABEL vendor="Open Source " diff --git a/build.sh b/build.sh index b1b0469e6..67393a2bb 100755 --- a/build.sh +++ b/build.sh @@ -58,6 +58,8 @@ declare -i travis_end_time } fi +go get github.com/karlmutch/petname + go get -u github.com/golang/dep/cmd/dep dep ensure diff --git a/ci.sh b/ci.sh index dab66a724..09eb2aee9 100755 --- a/ci.sh +++ b/ci.sh @@ -98,6 +98,14 @@ GIT_COMMIT=`git rev-parse HEAD` export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 +# Determine if we are running under a keel based CI build and if so ... +export +if [ -z ${KeelCI+x} ]; then + echo "Look for deployments to scale to 0 to lighten the load and footprint from this process" + kubectl --namespace $K8S_NAMESPACE get deployments + kubectl --namespace $K8S_NAMESPACE -o go-template --template="{{range .items}}kubectl scale --namespace {{.metadata.namespace}} --replicas=0 rc/{{.metadata.name}}{{end}}" get rc +fi + travis_fold start "build.image" travis_time_start set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 diff --git a/ci_keel.yaml b/ci_keel.yaml index a5b06dbef..23d2c77f8 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -24,6 +24,7 @@ metadata: name: build-env namespace: ci-go-runner-{{.duat.gitHash}} data: + KeelCI: "KeelCI" LOGXI: "*=INF" RABBITMQ_DEFAULT_USER: "UserUser" RABBITMQ_DEFAULT_PASS: "PasswordPassword" @@ -165,14 +166,21 @@ spec: selector: app: minio --- -# This job starts a run to completion runner to perform the integration and end-to-end testing -apiVersion: batch/v1 -kind: Job +# Run the integration build as a deployment, the lifecycle will be dealt with by the CMD entry +apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: build namespace: ci-go-runner-{{.duat.gitHash}} + labels: + keel.sh/policy: all + keel.sh/trigger: poll spec: template: + metadata: + labels: + # Label is used as selector in the service. + app: build spec: volumes: - name: podinfo @@ -201,7 +209,7 @@ spec: envFrom: - configMapRef: name: build-env - image: quay.io/leaf_ai_dockerhub/studio-go-runner-standalone-build:{{.duat.version}} + image: quay.io/leaf_ai_dockerhub/studio-go-runner-standalone-build:0.9.10-feature-184-flux-trial-1gpk3e imagePullPolicy: Always resources: limits: @@ -211,5 +219,3 @@ spec: - name: podinfo mountPath: /etc/podinfo readOnly: false - restartPolicy: Never - backoffLimit: 4 From 0ae7de6e204dad18873b642e5c306c84aa6c4f44 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 14:14:35 -0800 Subject: [PATCH 05/21] Keel test --- Dockerfile_standalone | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 149ed2ea6..42e240341 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,4 +37,4 @@ COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source " +LABEL vendor="Open Source" diff --git a/README.md b/README.md index 248076e77..4c4dd9374 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # studio-go-runner -Version: 0.9.10-feature-184-flux-trial-1gpk3e +Version: 0.9.10-feature-184-flux-trial-1gq3YS [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/leaf-ai/studio-go-runner/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/leaf-ai/studio-go-runner)](https://goreportcard.com/report/leaf-ai/studio-go-runner)[![DepShield Badge](https://depshield.sonatype.org/badges/leaf-ai/studio-go-runner/depshield.svg)](https://depshield.github.io) From 0d29052e86230e400416e09d461c60ca4d47a92e Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 14:35:34 -0800 Subject: [PATCH 06/21] Keel test --- Dockerfile_standalone | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 42e240341..149ed2ea6 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,4 +37,4 @@ COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source" +LABEL vendor="Open Source " diff --git a/README.md b/README.md index 4c4dd9374..215bb5ca4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # studio-go-runner -Version: 0.9.10-feature-184-flux-trial-1gq3YS +Version: 0.9.10-feature-184-flux-trial-1gq3sS [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/leaf-ai/studio-go-runner/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/leaf-ai/studio-go-runner)](https://goreportcard.com/report/leaf-ai/studio-go-runner)[![DepShield Badge](https://depshield.sonatype.org/badges/leaf-ai/studio-go-runner/depshield.svg)](https://depshield.github.io) From 3bafd4e5be74635274a25416fe26d959b868318d Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 15:03:35 -0800 Subject: [PATCH 07/21] Keel test --- Dockerfile_standalone | 2 +- ci_keel.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 149ed2ea6..42e240341 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,4 +37,4 @@ COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source " +LABEL vendor="Open Source" diff --git a/ci_keel.yaml b/ci_keel.yaml index 23d2c77f8..f9b0d7519 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -209,7 +209,7 @@ spec: envFrom: - configMapRef: name: build-env - image: quay.io/leaf_ai_dockerhub/studio-go-runner-standalone-build:0.9.10-feature-184-flux-trial-1gpk3e + image: quay.io/leaf_ai_dockerhub/studio-go-runner-standalone-build:feature_184_flux_trial imagePullPolicy: Always resources: limits: From 3eac1a94f1dee674086d35383250113b7f488da3 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 15:21:11 -0800 Subject: [PATCH 08/21] Keel test --- Dockerfile_standalone | 2 +- ci_keel.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 42e240341..149ed2ea6 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,4 +37,4 @@ COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source" +LABEL vendor="Open Source " diff --git a/ci_keel.yaml b/ci_keel.yaml index f9b0d7519..839b89e85 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -173,7 +173,8 @@ metadata: name: build namespace: ci-go-runner-{{.duat.gitHash}} labels: - keel.sh/policy: all + keel.sh/policy: force + keel.sh/match-tag: "true" keel.sh/trigger: poll spec: template: From ceab4a9d1abe2294ff6d86399d38db5939293801 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sat, 2 Feb 2019 17:16:39 -0800 Subject: [PATCH 09/21] Keel test --- Dockerfile_standalone | 4 ++++ ci_keel.yaml | 44 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 149ed2ea6..6dd9e70e0 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -30,6 +30,10 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ go get github.com/karlmutch/petname && \ + wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 && \ + wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 && \ + chmod +x $GOPATH/bin/semver && \ + chmod +x $GOPATH/bin/stencil && \ rm /usr/bin/nvidia-* COPY . /project/src/github.com/leaf-ai/studio-go-runner/ diff --git a/ci_keel.yaml b/ci_keel.yaml index 839b89e85..89ec0336c 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: ci-go-runner-{{.duat.gitHash}} + name: {{ default "ci-go-runner-{{.duat.gitHash}}" .Namespace }} --- # The following role allows the runner under test to interact with the working state config map kind: ClusterRoleBinding @@ -12,7 +12,7 @@ metadata: subjects: - kind: ServiceAccount name: default - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} roleRef: kind: ClusterRole name: cluster-admin @@ -22,7 +22,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: build-env - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} data: KeelCI: "KeelCI" LOGXI: "*=INF" @@ -40,7 +40,7 @@ metadata: labels: component: rabbitmq name: rabbitmq-service - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} spec: ports: - name: rmq-client @@ -58,7 +58,7 @@ metadata: labels: component: rabbitmq name: rabbitmq-controller - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} spec: replicas: 1 template: @@ -102,7 +102,7 @@ metadata: name: minio-pv-claim labels: app: minio-storage-claim - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} spec: # Read more about access modes here: https://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes accessModes: @@ -119,7 +119,7 @@ kind: Deployment metadata: # This name uniquely identifies the Deployment name: minio-deployment - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} spec: strategy: type: Recreate @@ -156,7 +156,7 @@ apiVersion: v1 kind: Service metadata: name: minio-service - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} spec: type: LoadBalancer ports: @@ -171,7 +171,7 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: build - namespace: ci-go-runner-{{.duat.gitHash}} + namespace: {{ .Namespace }} labels: keel.sh/policy: force keel.sh/match-tag: "true" @@ -220,3 +220,29 @@ spec: - name: podinfo mountPath: /etc/podinfo readOnly: false + lifecycle: + postStart: + exec: + command: + - "/bin/bash" + - "-c" + - > + set -euo pipefail ; + IFS=$'\n\t' ; + echo "Starting the keel modifications" ; + go install github.com/karlmutch/petname/cmd/petname ; + stencil -input ci_keel.yaml -values Namespace=ci-go-runner-`petname` > /tmp/new_ci_cluster.yaml ; + kubectl --namespace $K8S_NAMESPACE get pods ; + curl -v --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $(cat /var/runsecrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$K8S_NAMESPACE/pods/$K8S_POD_NAME + preStop: + exec: + command: + - "/bin/bash" + - "-c" + - > + set -euo pipefail ; + IFS=$'\n\t' ; + echo "Starting the namespace injections etc" ; + kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller ; + kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment ; + kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/build From 131d7dc7291d3fd5646f34b39ce18627edfe86e6 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 11:34:29 -0800 Subject: [PATCH 10/21] Keel test --- Dockerfile_standalone | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 6dd9e70e0..e58a4e7a7 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -36,9 +36,11 @@ RUN mkdir $GOPATH/bin && \ chmod +x $GOPATH/bin/stencil && \ rm /usr/bin/nvidia-* +LABEL author="Karl Mutch" + COPY . /project/src/github.com/leaf-ai/studio-go-runner/ CMD /bin/bash -c './ci.sh' # Done last to prevent lots of disruption when bumping versions -LABEL vendor="Open Source " +LABEL vendor="Open Source" From 10b62dcf8573528f713b3c4f154bf6560467ac8b Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 11:52:54 -0800 Subject: [PATCH 11/21] Keel test --- Dockerfile_standalone | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index e58a4e7a7..729f02a7b 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -4,7 +4,14 @@ MAINTAINER karlmutch@gmail.com ENV LANG C.UTF-8 -ENV GO_VERSION 1.11.4 +ENV GO_VERSION 1.11.5 + +RUN \ + apt-get update && apt-get install -y apt-transport-https && \ + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list && \ + apt-get update && \ + apt-get install -y kubectl RUN \ mkdir -p /project/go && \ @@ -30,8 +37,8 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ go get github.com/karlmutch/petname && \ - wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 && \ - wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 && \ + wget -q -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 && \ + wget -q -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 && \ chmod +x $GOPATH/bin/semver && \ chmod +x $GOPATH/bin/stencil && \ rm /usr/bin/nvidia-* From 74341f72beedd0e1a563633525576ae7e8bedcdf Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 12:32:41 -0800 Subject: [PATCH 12/21] Keel test, removed unused components --- ci.sh | 14 -------------- ci_keel.yaml | 8 +++----- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/ci.sh b/ci.sh index 09eb2aee9..9b9737e06 100755 --- a/ci.sh +++ b/ci.sh @@ -61,11 +61,6 @@ go get -u github.com/golang/dep/cmd/dep dep ensure -wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 -wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 -chmod +x $GOPATH/bin/semver -chmod +x $GOPATH/bin/stencil - bash -c "while true; do echo \$(date) - building ...; sleep 180s; done" & PING_LOOP_PID=$! @@ -92,19 +87,10 @@ working_file=$$.studio-go-runner-working rm -f $working_file trap Tidyup 1 2 3 15 -export SEMVER=`semver` -export GIT_BRANCH=`echo '{{.duat.gitBranch}}'|stencil - | tr '_' '-' | tr '\/' '-'` -GIT_COMMIT=`git rev-parse HEAD` -export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log exit_code=0 # Determine if we are running under a keel based CI build and if so ... export -if [ -z ${KeelCI+x} ]; then - echo "Look for deployments to scale to 0 to lighten the load and footprint from this process" - kubectl --namespace $K8S_NAMESPACE get deployments - kubectl --namespace $K8S_NAMESPACE -o go-template --template="{{range .items}}kubectl scale --namespace {{.metadata.namespace}} --replicas=0 rc/{{.metadata.name}}{{end}}" get rc -fi travis_fold start "build.image" travis_time_start diff --git a/ci_keel.yaml b/ci_keel.yaml index 89ec0336c..e28b2ff8f 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -230,9 +230,8 @@ spec: set -euo pipefail ; IFS=$'\n\t' ; echo "Starting the keel modifications" ; - go install github.com/karlmutch/petname/cmd/petname ; - stencil -input ci_keel.yaml -values Namespace=ci-go-runner-`petname` > /tmp/new_ci_cluster.yaml ; - kubectl --namespace $K8S_NAMESPACE get pods ; + kubectl scale --namespace $K8S_NAMESPACE --replicas=1 rc/rabbitmq-controller ; + kubectl scale --namespace $K8S_NAMESPACE --replicas=1 deployment/minio-deployment ; curl -v --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $(cat /var/runsecrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$K8S_NAMESPACE/pods/$K8S_POD_NAME preStop: exec: @@ -244,5 +243,4 @@ spec: IFS=$'\n\t' ; echo "Starting the namespace injections etc" ; kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller ; - kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment ; - kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/build + kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment From d47545bfc37adcb05901ad9ac6135e8df88fe0a4 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 16:11:20 -0800 Subject: [PATCH 13/21] Support ssh access --- Dockerfile_standalone | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 729f02a7b..2eab2b56c 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,8 +37,8 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ go get github.com/karlmutch/petname && \ - wget -q -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 && \ - wget -q -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 && \ + wget -q -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.3/semver-linux-amd64 && \ + wget -q -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.3/stencil-linux-amd64 && \ chmod +x $GOPATH/bin/semver && \ chmod +x $GOPATH/bin/stencil && \ rm /usr/bin/nvidia-* From c1aa788e7d4be8752e419ba65dfabc742d5c434b Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 17:45:39 -0800 Subject: [PATCH 14/21] Support ssh access --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 215bb5ca4..112215b36 100644 --- a/README.md +++ b/README.md @@ -130,9 +130,9 @@ To install the tools on Ubuntu use the following commands: ```shell mkdir -p $GOPATH/bin -wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.2/semver-linux-amd64 -wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.2/stencil-linux-amd64 -wget -O $GOPATH/bin/github-release https://github.com/karlmutch/duat/releases/download/0.9.2/github-release-linux-amd64 +wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.3/semver-linux-amd64 +wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.3/stencil-linux-amd64 +wget -O $GOPATH/bin/github-release https://github.com/karlmutch/duat/releases/download/0.9.3/github-release-linux-amd64 chmod +x $GOPATH/bin/semver chmod +x $GOPATH/bin/stencil chmod +x $GOPATH/bin/github-release From 41904f7c867f04e88df7a312be938adccd51ff28 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Sun, 3 Feb 2019 17:57:33 -0800 Subject: [PATCH 15/21] Upgrade duat --- Gopkg.lock | 4 ++-- vendor/github.com/karlmutch/duat/Dockerfile | 2 +- vendor/github.com/karlmutch/duat/README.md | 2 +- vendor/github.com/karlmutch/duat/git.go | 8 ++++++-- vendor/github.com/karlmutch/duat/metadata.go | 8 ++++++++ vendor/github.com/karlmutch/duat/template.go | 17 ++++++++++------- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d4852710d..7b9bb860e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -444,14 +444,14 @@ [[projects]] branch = "master" - digest = "1:761c94921a498e50ad21fdfc3748e96c998ada0b75531548679c13169238372d" + digest = "1:cb39001cf3c1e09e22bd8a0d95af76b6428586bec756681425b22cd5443b1490" name = "github.com/karlmutch/duat" packages = [ ".", "version", ] pruneopts = "UT" - revision = "f03105608dfc7d38642337b9bcf649353f705cc7" + revision = "41b536cb50e1d24656096ba018b8e71879f0a45a" [[projects]] branch = "master" diff --git a/vendor/github.com/karlmutch/duat/Dockerfile b/vendor/github.com/karlmutch/duat/Dockerfile index a961fbff2..9210ed3be 100644 --- a/vendor/github.com/karlmutch/duat/Dockerfile +++ b/vendor/github.com/karlmutch/duat/Dockerfile @@ -20,7 +20,7 @@ RUN apt-get -y install git software-properties-common wget openssl ssh curl jq a groupadd -f -g ${USER_GROUP_ID} ${USER} && \ useradd -g ${USER_GROUP_ID} -u ${USER_ID} -ms /bin/bash ${USER} -ENV GO_VERSION 1.10.3 +ENV GO_VERSION 1.11.2 RUN cd /home/${USER} && \ mkdir -p /home/${USER}/go && \ diff --git a/vendor/github.com/karlmutch/duat/README.md b/vendor/github.com/karlmutch/duat/README.md index 457a93c2f..75fdbca38 100644 --- a/vendor/github.com/karlmutch/duat/README.md +++ b/vendor/github.com/karlmutch/duat/README.md @@ -1,6 +1,6 @@ # Developer utilities and tools (duat) Alpha -Version : 0.9.1 +Version : 0.9.3 duat is a set of tools useful for automating workflows operating on common software artifacts such as git branches and tags, semantic versioning, and docker image delivery. duat is a work in progress experiment in using Go to manage the entire software lifecycle removing scripting and other DSLs typically used for building, releasing, and deploying software. diff --git a/vendor/github.com/karlmutch/duat/git.go b/vendor/github.com/karlmutch/duat/git.go index 697733a6b..c681efa37 100644 --- a/vendor/github.com/karlmutch/duat/git.go +++ b/vendor/github.com/karlmutch/duat/git.go @@ -75,9 +75,13 @@ func (md *MetaData) LoadGit(dir string, scanParents bool) (err errors.Error) { return md.Git.Err } - gitURL, errGo := url.Parse(refs[0].Config().URLs[0]) + urlLoc := refs[0].Config().URLs[0] + if strings.HasPrefix(urlLoc, "git@github.com:") { + urlLoc = strings.Replace(urlLoc, "git@github.com:", "https://github.com/", 1) + } + gitURL, errGo := url.Parse(urlLoc) if errGo != nil { - md.Git.Err = errors.Wrap(errGo).With("url", refs[0].Config().URLs[0]).With("git", gitDir).With("stack", stack.Trace().TrimRuntime()) + md.Git.Err = errors.Wrap(errGo).With("url", urlLoc).With("git", gitDir).With("stack", stack.Trace().TrimRuntime()) return md.Git.Err } md.Git.URL = *gitURL diff --git a/vendor/github.com/karlmutch/duat/metadata.go b/vendor/github.com/karlmutch/duat/metadata.go index c5f7eba61..4090f5d13 100644 --- a/vendor/github.com/karlmutch/duat/metadata.go +++ b/vendor/github.com/karlmutch/duat/metadata.go @@ -3,6 +3,7 @@ package duat import ( "net/url" "os" + "os/user" "path/filepath" // The following packages are forked to retain copies in the event github accounts are shutdown @@ -31,6 +32,7 @@ type GitInfo struct { } type MetaData struct { + user *user.User Dockers map[string]docker.Client SemVer *semver.Version Module string // A string name for the software component that is being handled @@ -54,6 +56,12 @@ func NewMetaData(dir string, verFile string) (md *MetaData, err errors.Error) { md = &MetaData{} + usr, errGo := user.Current() + if errGo != nil { + return nil, errors.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) + } + md.user = usr + cwd, errGo := os.Getwd() if errGo != nil { return nil, errors.Wrap(errGo, "current directory unknown").With("stack", stack.Trace().TrimRuntime()) diff --git a/vendor/github.com/karlmutch/duat/template.go b/vendor/github.com/karlmutch/duat/template.go index ad2bc0819..229758988 100644 --- a/vendor/github.com/karlmutch/duat/template.go +++ b/vendor/github.com/karlmutch/duat/template.go @@ -93,13 +93,16 @@ func (md *MetaData) NewTemplateVariables(jsonVals string, loadFiles []string, ov vars["Env"] = envs duatVars := map[string]interface{}{ - "version": md.SemVer.String(), - "module": md.Module, - "gitTag": md.Git.Tag, - "gitHash": md.Git.Hash, - "gitBranch": md.Git.Branch, - "gitURL": md.Git.URL, - "gitDir": md.Git.Dir, + "version": md.SemVer.String(), + "module": md.Module, + "gitTag": md.Git.Tag, + "gitHash": md.Git.Hash, + "gitBranch": md.Git.Branch, + "gitURL": md.Git.URL, + "gitDir": md.Git.Dir, + "userID": md.user.Uid, + "userName": md.user.Username, + "userGroupID": md.user.Gid, } if runtime, err := md.ContainerRuntime(); err == nil { duatVars["runtime"] = runtime From 8efedcd9af9a1dce56e00f711c154f09e7de76b8 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Mon, 4 Feb 2019 09:29:51 -0800 Subject: [PATCH 16/21] Handle container lifecycle within the scripts inside the container to prevent spawning many children --- ci.sh | 9 +++++++++ ci_keel.yaml | 12 +++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ci.sh b/ci.sh index 9b9737e06..af88e31f1 100755 --- a/ci.sh +++ b/ci.sh @@ -102,6 +102,15 @@ travis_fold start "build.image" travis_time_finish travis_fold end "build.image" +echo "Starting the namespace injections etc" $K8S_POD_NAME +kubectl label deployment build keel.sh/policy=force --namespace=$K8S_NAMESPACE +kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller +kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment +for (( ; ; )) +do + sleep 10 +done + if [ $exit_code -ne 0 ]; then exit $exit_code fi diff --git a/ci_keel.yaml b/ci_keel.yaml index e28b2ff8f..dadcacadd 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -229,7 +229,8 @@ spec: - > set -euo pipefail ; IFS=$'\n\t' ; - echo "Starting the keel modifications" ; + echo "Starting the keel modifications" $K8S_POD_NAME ; + kubectl label deployment build keel.sh/policy- --namespace=$K8S_NAMESPACE ; kubectl scale --namespace $K8S_NAMESPACE --replicas=1 rc/rabbitmq-controller ; kubectl scale --namespace $K8S_NAMESPACE --replicas=1 deployment/minio-deployment ; curl -v --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $(cat /var/runsecrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$K8S_NAMESPACE/pods/$K8S_POD_NAME @@ -241,6 +242,11 @@ spec: - > set -euo pipefail ; IFS=$'\n\t' ; - echo "Starting the namespace injections etc" ; + echo "Starting the namespace injections etc" $K8S_POD_NAME ; + kubectl label deployment build keel.sh/policy=force --namespace=$K8S_NAMESPACE ; kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller ; - kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment + kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment ; + for (( ; ; )) ; + do ; + sleep 10 ; + done From ead522e20d88d7e5a368a8fc2d06c747d771147b Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Mon, 4 Feb 2019 17:08:11 -0800 Subject: [PATCH 17/21] Trigger downstream container build and then test if keel picksup the changes --- docs/quay_io_ci.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quay_io_ci.md b/docs/quay_io_ci.md index 431f05b64..0eb81748f 100644 --- a/docs/quay_io_ci.md +++ b/docs/quay_io_ci.md @@ -2,10 +2,11 @@ This document describes setting up a CI pipline that can be used to prepare releases for studio go runner. -studio go runner is designed to run in resource intensive environments using GPU enabled machines and so providing a free hosted pipeline for CI/CD is cost prohibitive. As an alternative parties interested in studio go runner can make use of quay.io hosted images built automatically and are then pulled into a test and integration downstream Kubernetes provisioned cluster they own. This allows testing to be done using the CI pipeline on both local laptops, workstations and in cloud or data center environments. +studio go runner is designed to run in resource intensive environments using GPU enabled machines and so providing a free hosted pipeline for CI/CD is cost prohibitive. As an alternative, parties interested in studio go runner can make use of quay.io hosted images built automatically on github commit triggers to then trigger their own downstream build, test and deploy automation. Downstream automation can be hosted on a self provisioned Kubernetes provisioned cluster either within the cloud or on private infrastructure. This allows testing to be done using the CI pipeline on both local laptops, workstations and in cloud or data center environments. This document contains instructions that can be used for hardware configurations that individual users to large scale enterprises can use without incuring monthly charges from third party providers. These instructions first detail how a quay.io trigger can be setup to trigger builds on github commits. Instructions then detail how to make use of Keel, https://keel.sh/, to pull CI images into a cluster and run the pipeline. +# Optional GITHUB_TOKEN secrets are added to the cluster Annotations updated via stencil with gitHash etc and also with desired regular expression or keel semver policy From 3ef13d5aab819f8ed68e8b54980f023e20c389be Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Tue, 5 Feb 2019 09:24:29 -0800 Subject: [PATCH 18/21] Dont continue to pump out meaningless messages after quiesing --- ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index af88e31f1..5052a9530 100755 --- a/ci.sh +++ b/ci.sh @@ -102,13 +102,15 @@ travis_fold start "build.image" travis_time_finish travis_fold end "build.image" +cleanup + echo "Starting the namespace injections etc" $K8S_POD_NAME kubectl label deployment build keel.sh/policy=force --namespace=$K8S_NAMESPACE kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment for (( ; ; )) do - sleep 10 + sleep 600 done if [ $exit_code -ne 0 ]; then From 078b3aa6a0e0f56c8c9da0025336768cad6ea00d Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Tue, 5 Feb 2019 13:07:55 -0800 Subject: [PATCH 19/21] Release -h --- Dockerfile_standalone | 2 ++ README.md | 2 ++ ci_keel.yaml | 14 ++++++++++ docs/quay_io_ci.md | 65 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 2eab2b56c..728448c9f 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,6 +37,8 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ go get github.com/karlmutch/petname && \ + go get github.com/karlmutch/petname && \ + go install github.com/karlmutch/petname/cmd/petname && \ wget -q -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.3/semver-linux-amd64 && \ wget -q -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.3/stencil-linux-amd64 && \ chmod +x $GOPATH/bin/semver && \ diff --git a/README.md b/README.md index 112215b36..91ed9a91d 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ To install the tools on Ubuntu use the following commands: ```shell mkdir -p $GOPATH/bin +go get github.com/karlmutch/petname +go install github.com/karlmutch/petname/cmd/petname wget -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.3/semver-linux-amd64 wget -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.3/stencil-linux-amd64 wget -O $GOPATH/bin/github-release https://github.com/karlmutch/duat/releases/download/0.9.3/github-release-linux-amd64 diff --git a/ci_keel.yaml b/ci_keel.yaml index dadcacadd..656500f89 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -153,6 +153,15 @@ spec: mountPath: "/storage" --- apiVersion: v1 +kind: Secret +metadata: + name: release-github-token + namespace: {{ .Namespace }} +type: Opaque +data: + github_token: '' +--- +apiVersion: v1 kind: Service metadata: name: minio-service @@ -199,6 +208,11 @@ spec: containers: - name: build env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: release-github-token + key: github_token - name: K8S_POD_NAME valueFrom: fieldRef: diff --git a/docs/quay_io_ci.md b/docs/quay_io_ci.md index 0eb81748f..b066477d4 100644 --- a/docs/quay_io_ci.md +++ b/docs/quay_io_ci.md @@ -6,8 +6,69 @@ studio go runner is designed to run in resource intensive environments using GPU This document contains instructions that can be used for hardware configurations that individual users to large scale enterprises can use without incuring monthly charges from third party providers. These instructions first detail how a quay.io trigger can be setup to trigger builds on github commits. Instructions then detail how to make use of Keel, https://keel.sh/, to pull CI images into a cluster and run the pipeline. -# -Optional GITHUB_TOKEN secrets are added to the cluster +# A word about privacy + +Many of the services that provide image hosting use Single Sign On and credentials management with your source code control platform of choice. As a consequence of this often these services will gain access to any and all repositories private or otherwise that you might have access to within your account. In order to preserve privacy and maintain fine grained control over the visibility of your private repositories it is recommended that when using quay.io and other services that you create a service account that has the minimal level of access to repositories as nessasary to implement your CI/CD features. + +# CI Image building + +The studio go runner project uses Docker images to encapsulate builds within an immutable archive format. Using internet accessible hosted registries it is possible to configure a registry to actively build an image from the git repository at that commit and to then host the resulting image. A number of internet registries offer hosting for open source projects for free, and also offer paid hosted plans for users requiring privacy. These instructions give a summary of what needs to be done in order to use the quay.io service to provision an image repository that auto-builds images from the studio go runner project. + +The first step is to create or login to an account on quay.io. When creating an account on quay.io it is best to ensure before starting that you have a browser window open to giuthub.com using the account that you wish to use for accessing code on github to prevent any unintended accesses to private repositories. As you create the account on you can choose to link it automatically to github granting application access from quay to your github authorized applications. This is needed in order that quay can poll your projects for any changes in order to trigger image building. + +Having logged in you can now create a repository using the label at the top right corner of your web page underneath the account related drop down menu. + +The first screen will allow you to specify tgar you wish to create an image repository and assign it a name, also set the visibility to public, and to 'Link to a GitHub Repository Push', this indicates that any push of a commit or tag will result in a container build being triggered. + +Pushing the next button will then cause the browser to request github to authorize access from quay to github and will prompt you to allow this authorization to be setup for future interactions between the two platform. Again, be sure you are assuming the role of the most recently logged in github user and that the one being authorized is the one you intend to allow Quay to obtain access to. + +After the auhtorization is enabled the next web page is displayed which allows the organization and account to be choosen from which the image will be built. Step through the next two screen to drill down to the repository that will be used and then push the continue button. + +You can then specify the branch(es) that can then be used for the builds to meet you own needs. Pushing con tinue will then allow you to select the Dockerfile that will act as your source for the new image. When using studio go runner a Dockerfile called Dockerfile_standalone is versioned in the source code repository that will allow a fully standalone container to be created that can be perform the entire build, test, release life cycle for the software. usign a slash indicates the top level of the go runner repo. + +Using continue will then prompt for the Context of the build which should be set to '/'. You can now click through the rest of the selections and will end up with a fully populated trigger for the repository. + +You can now trigger the first build and test cycle for the repository. Once the repository has been built you can proceed to setting up a Kubernetes test cluster than can pull the image(s) from the repository as they are updated via git commits followed by a git push. + +# Continuous Integration + +The presence of a publically accesible repository allows a suitably configured Kubernetes cluster to query for the presence of build images for testing and integration. + +The studio go runner standalone build image can be used within a go runner deployment to perform testing and validation against a live minio(s3 server) and a RabbitMQ (queue server) instances deployed within a single Kubernetes namespace. The definition of the deployment is stored within the source code repository, as ci_keel.yaml. + +The build deployment contains an annotated deployment of the build image that when deployed concurrently with http://keel.sh/ can react to freshly created build images to cycle through build, deploy, test cycles automatically. + +Keel is documented at https://keel.sh/, installation instruction can also be found there, https://keel.sh/guide/installation.html. Once deploy keel can be left to run as a background service observing Kubernetes deployments that contain annotations it is designed to react to. Keel will watch for changes to image repositories that Deployments have annotations for and will automatically upgrade the Deployment pods as new images are seen. + +The studio go runner ci_keel.yaml contains annotations for a studio go runner Deployment that the user should look into and configure to select the branches for which they want to watch and perform tests and releases for. The keel labels within the ci_keel.yaml file dictate under what circumstances the keel server will trigger a new pod for the build and test to be created in response to the reference build image changing as git commit and push operations are performed. Information about these labels can be found at, https://keel.sh/v1/guide/documentation.html#Policies. + +The commands that you might performed to this point in order to deploy keel into an existing Kubernetes deploy might well appear as follows: + +``` +mkdir -p ~/project/src/github.com/keel-hq +cd ~/project/src/github.com/keel-hq +git clone https://github.com/keel-hq/keel.git +cd keel +kubectl create -f deployment-rbac.yaml +mkdir -p ~/project/src/github.com/leaf-ai +cd ~/project/src/github.com/leaf-ai +git clone https://github.com/leaf-ai/studio-go-runner.git +cd studio-go-runner +git checkout [branch name] +# Follow the instructions for setting up the Prerequisites for compilation in the main README.md file +``` + +The next step would be to edit the ci_keel.yaml file to reflect the branch name on which the development is being performed or the release prepared, and then deploy the integration stack. + +``` +stencil -input ci_keel.yaml -values Namespace=ci-go-runner | kubectl apply -f - +``` + +This will deploy a stack capable of builds and testing. As a build finishes the stack will scale down the dependencies it uses for queuing and storage and will keep the build container alive so that logs can be examined. The build activities will disable container upgrades while the build is running and will then open for upgrades once the build steps have completed to prevent premature termination. When the build, and test has completed and pushed commits have been seen for the code base then the pod will be shutdown for the latest build and a new pod created. + +When deploying an integration stack it is possible to specify a GithubToken for performing releases. If the token is present as a Kubernetes secret then upon successful build and test cycles the running container will attempt to create and deploy a release using the github release pages. + +OPtional GITHUB_TOKEN secrets are added to the cluster Annotations updated via stencil with gitHash etc and also with desired regular expression or keel semver policy namespace is generated and used for the bootstrapped build From 41c10ac81a6cadc909e147c6ce8cdac8ca0bcd49 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Tue, 5 Feb 2019 16:58:43 -0800 Subject: [PATCH 20/21] Get the build log back in play and auto inject the GITHUB_TOKEN as a kubernetes secret, this is safe as long as we use private clusters when the yaml is deployed to the CI system --- Dockerfile_standalone | 1 - ci.sh | 6 +++++- ci_keel.yaml | 2 +- docs/quay_io_ci.md | 40 ++-------------------------------------- 4 files changed, 8 insertions(+), 41 deletions(-) diff --git a/Dockerfile_standalone b/Dockerfile_standalone index 728448c9f..59bd020e3 100644 --- a/Dockerfile_standalone +++ b/Dockerfile_standalone @@ -37,7 +37,6 @@ RUN mkdir $GOPATH/bin && \ git config --global url."https://".insteadOf git:// && \ go get github.com/karlmutch/enumer && \ go get github.com/karlmutch/petname && \ - go get github.com/karlmutch/petname && \ go install github.com/karlmutch/petname/cmd/petname && \ wget -q -O $GOPATH/bin/semver https://github.com/karlmutch/duat/releases/download/0.9.3/semver-linux-amd64 && \ wget -q -O $GOPATH/bin/stencil https://github.com/karlmutch/duat/releases/download/0.9.3/stencil-linux-amd64 && \ diff --git a/ci.sh b/ci.sh index 5052a9530..3576a029d 100755 --- a/ci.sh +++ b/ci.sh @@ -87,6 +87,9 @@ working_file=$$.studio-go-runner-working rm -f $working_file trap Tidyup 1 2 3 15 +export GIT_BRANCH=`echo '{{.duat.gitBranch}}' | stencil - | tr '_' '-' | tr '\/' '-'` +export RUNNER_BUILD_LOG=build-$GIT_BRANCH.log + exit_code=0 # Determine if we are running under a keel based CI build and if so ... @@ -94,7 +97,7 @@ export travis_fold start "build.image" travis_time_start - set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 + set -o pipefail ; (go run build.go -r -dirs=internal && go run build.go -r -dirs=cmd && echo "Success" || echo "Failure") 2>&1 | tee $RUNNER_BUILD_LOG exit_code=$? if [ $exit_code -ne 0 ]; then exit $exit_code @@ -108,6 +111,7 @@ echo "Starting the namespace injections etc" $K8S_POD_NAME kubectl label deployment build keel.sh/policy=force --namespace=$K8S_NAMESPACE kubectl scale --namespace $K8S_NAMESPACE --replicas=0 rc/rabbitmq-controller kubectl scale --namespace $K8S_NAMESPACE --replicas=0 deployment/minio-deployment + for (( ; ; )) do sleep 600 diff --git a/ci_keel.yaml b/ci_keel.yaml index 656500f89..5b1484b13 100644 --- a/ci_keel.yaml +++ b/ci_keel.yaml @@ -159,7 +159,7 @@ metadata: namespace: {{ .Namespace }} type: Opaque data: - github_token: '' + github_token: '{{ expandenv "$GITHUB_TOKEN" | b64enc }}' --- apiVersion: v1 kind: Service diff --git a/docs/quay_io_ci.md b/docs/quay_io_ci.md index b066477d4..dae0b3917 100644 --- a/docs/quay_io_ci.md +++ b/docs/quay_io_ci.md @@ -66,42 +66,6 @@ stencil -input ci_keel.yaml -values Namespace=ci-go-runner | kubectl apply -f - This will deploy a stack capable of builds and testing. As a build finishes the stack will scale down the dependencies it uses for queuing and storage and will keep the build container alive so that logs can be examined. The build activities will disable container upgrades while the build is running and will then open for upgrades once the build steps have completed to prevent premature termination. When the build, and test has completed and pushed commits have been seen for the code base then the pod will be shutdown for the latest build and a new pod created. -When deploying an integration stack it is possible to specify a GithubToken for performing releases. If the token is present as a Kubernetes secret then upon successful build and test cycles the running container will attempt to create and deploy a release using the github release pages. +If the env variable GITHUB_TOKEN is present when deploying an integration stack it will be placed as a Kubernetes secret into the integration stack. If the secret is present then upon successful build and test cycles the running container will attempt to create and deploy a release using the github release pages. -OPtional GITHUB_TOKEN secrets are added to the cluster - -Annotations updated via stencil with gitHash etc and also with desired regular expression or keel semver policy -namespace is generated and used for the bootstrapped build -stencil -input ci_keel.yaml | kubectl apply -f - -git commit and push to start things rolling -Keel repo polling triggers build - -built container in build pod removes itself from keel using Kubernetes preStartHook by renaming annotations -``` -Using downward API -metadata.annotations['myannotation'] -``` - -build pod starts -new namespace generated for next listener -``` -github.com/docker/docker/pkg/namesgenerator -Loop creating namespace with uuid annotation and then validating we owned it -``` - -container used the included ci_keel and injects the annotations from itself to create the next listening deployment -``` -stencil with variables in a file for all annotations now renamed for their real keys -``` - -new namspace with deployment using ci_keel.yaml is dispatched -build starts in our now liberated namespace - -build finishes -set ReplicationControllers and deployment .spec.replicas to 0 -``` -kubectl scale --namespace build-test-k8s-local --replicas=0 deployment/minio-deployment -kubectl scale --namespace build-test-k8s-local --replicas=0 rc/rabbitmq-controller -``` - -and the build then sits until such time as we decide on a policy for self destruction like push results back to github, at which point we dispose of the unique namespace used for the build +When the build completes the pods that are present that are only useful during the actual build and test steps will be scaled back to 0 instances. From 507d2e1dc9107d57f7f0b8bbc818fbbfd0f01948 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Tue, 5 Feb 2019 17:34:01 -0800 Subject: [PATCH 21/21] Mint a release --- CHANGELOG.md | 5 +++-- README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0178edb79..b66ef2869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ FIXES: IMPROVMENTS -* Image repository naming modified to work with dockerhub, images now pushed to the docker hub leafai account +* Image repository naming modified to work with dockerhub, images can now be pushed to the docker hub leafai account * Git actions ready, changes to allow larger base containers to be prebuilt reducing build requirements in the Git infrastructure -* Test quay.io based builds from github commit/push on any branch. +* quay.io based builds from github commit/push on any branch +* keel.sh based CI with automatted builds and tests using git commit notifications diff --git a/README.md b/README.md index 91ed9a91d..698c037f2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # studio-go-runner -Version: 0.9.10-feature-184-flux-trial-1gq3sS +Version: 0.9.10 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/leaf-ai/studio-go-runner/blob/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/leaf-ai/studio-go-runner)](https://goreportcard.com/report/leaf-ai/studio-go-runner)[![DepShield Badge](https://depshield.sonatype.org/badges/leaf-ai/studio-go-runner/depshield.svg)](https://depshield.github.io)