From 9529640a8294364b7dac50129f1d811969b453a5 Mon Sep 17 00:00:00 2001 From: Frederic Giloux Date: Wed, 8 Jun 2022 11:48:39 +0200 Subject: [PATCH] Add gateway deployment for Pipelines as Code Signed-off-by: Frederic Giloux --- .../workflows/gateway-deployment-image.yaml | 47 ++++++++ docs/gateway.md | 17 ++- .../kcp}/gateway/haproxy-cfg-cm.yaml | 12 +- .../kcp}/gateway/haproxy-deployment.yaml | 8 +- .../kcp}/gateway/haproxy-ingress.yaml | 8 +- .../kcp}/gateway/haproxy-service.yaml | 2 +- gitops/kcp/gateway/kustomization.yaml | 5 + gitops/pac/README.md | 6 +- ...c-registration.yaml => cluster-setup.yaml} | 0 gitops/sre/.tekton/gateway-deployment.yaml | 61 ++++++++++ images/gateway/Dockerfile | 13 +++ images/gateway/install.sh | 109 ++++++++++++++++++ images/kcp-registrar/register.sh | 2 +- 13 files changed, 270 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/gateway-deployment-image.yaml rename {exploration/triggers => gitops/kcp}/gateway/haproxy-cfg-cm.yaml (71%) rename {exploration/triggers => gitops/kcp}/gateway/haproxy-deployment.yaml (81%) rename {exploration/triggers => gitops/kcp}/gateway/haproxy-ingress.yaml (63%) rename {exploration/triggers => gitops/kcp}/gateway/haproxy-service.yaml (90%) create mode 100644 gitops/kcp/gateway/kustomization.yaml rename gitops/sre/.tekton/{pln-svc-registration.yaml => cluster-setup.yaml} (100%) create mode 100644 gitops/sre/.tekton/gateway-deployment.yaml create mode 100644 images/gateway/Dockerfile create mode 100755 images/gateway/install.sh diff --git a/.github/workflows/gateway-deployment-image.yaml b/.github/workflows/gateway-deployment-image.yaml new file mode 100644 index 000000000..2dfc3cb0a --- /dev/null +++ b/.github/workflows/gateway-deployment-image.yaml @@ -0,0 +1,47 @@ +name: Build and Publish the Gateway Deployment Image + +on: + push: + branches: + - main + paths: + - "images/gateway/**" + workflow_dispatch: + +jobs: + build-push: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + sha_short: ${{ steps.vars.outputs.sha_short }} + + steps: + - uses: actions/checkout@v3 + + - name: Get the short sha + id: vars + run: echo "::set-output name=sha_short::$(echo ${{ github.sha }} | cut -b -7)" + + # Build and push a gateway-deployment image, tagged with latest and the commit SHA. + - name: Build gateway-deployment Image + id: build-image + uses: redhat-actions/buildah-build@v2 + with: + image: gateway-deployment + context: ./images/gateway + tags: latest ${{ steps.vars.outputs.sha_short }} ${{ github.ref_name }} + containerfiles: | + ./images/gateway-deployment/Dockerfile + + - name: Push to ghcr.io + id: push-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: ${{ steps.build-image.outputs.image }} + tags: ${{ steps.build-image.outputs.tags }} ${{ github.ref_name }} + registry: ghcr.io/${{ github.repository_owner }} + username: ${{ github.actor }} + password: ${{ github.token }} diff --git a/docs/gateway.md b/docs/gateway.md index 56e9fff72..38543055a 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -41,23 +41,23 @@ HAProxy configuration can be amended through the ConfigMap. See the section belo Connection settings, support for https can be configured by amending the HAProxy configuration contained in the ConfigMap. This will get streamlined with the automation of the installation of Tekton triggers. -The first use case for the gateway is to forward requests to the listeners of Tekton triggers. This may also be used to proxy other services. +The first use case for the gateway is to forward requests to the EventListener for Pipelines as Code. This may also be used to proxy other services. Therefore, additional frontends can be configured so that queries with other paths are forwarded to other backend servers. Path-based routing is configured in this snippet: ```bash -acl PATH_trigger path_beg -i /trigger -use_backend be_trigger if PATH_trigger +acl PATH_pac path_beg -i /pac/ +use_backend be_el_pac if PATH_pac ``` The backend in charge of processing the query can be specified in the referenced section: ```bash -server s1 httpecho.trigger.svc.cluster.local:80 +server el-pac el-pipelines-as-code-interceptor.openshift-pipelines.svc.cluster.local:8080 ``` -here the backend is the service `httpecho` in the `trigger` namespace listening to port 80. +here the backend is the service `el-pac el-pipelines-as-code-interceptor` in the `openshift-pipelines` namespace listening to port 8080. ## Demo @@ -70,3 +70,10 @@ here the backend is the service `httpecho` in the `trigger` namespace listening **_NOTE:_** This is only needed for phase 1. This component will get removed when we move to phase 2 and have the event listeners provisioned through kcp. --- + +## Limitations + +- There is currently no controller watching EventListeners to configure the gateway dynamically. This means that the gateway would work for Pipelines as Code, which offers a stable entrypoint but not for pure Tekton Triggers. +- PipelineRuns are not visible in any kcp workspace. +- Ingress is currently broken with kcp 0.5 and may not get fixed before 0.7 + diff --git a/exploration/triggers/gateway/haproxy-cfg-cm.yaml b/gitops/kcp/gateway/haproxy-cfg-cm.yaml similarity index 71% rename from exploration/triggers/gateway/haproxy-cfg-cm.yaml rename to gitops/kcp/gateway/haproxy-cfg-cm.yaml index 8b229eb9a..929e44147 100644 --- a/exploration/triggers/gateway/haproxy-cfg-cm.yaml +++ b/gitops/kcp/gateway/haproxy-cfg-cm.yaml @@ -19,18 +19,18 @@ data: timeout http-keep-alive 300s timeout check 10s - frontend trigger_gateway-http - bind :80 + frontend fe_el_pac_http + bind :8080 mode http tcp-request inspect-delay 5s tcp-request content accept if HTTP - acl PATH_trigger path_beg -i /trigger - use_backend be_trigger if PATH_trigger + acl PATH_pac path_beg -i /pac/ + use_backend be_el_pac if PATH_pac - backend be_trigger + backend be_el_pac mode http option redispatch option forwardfor timeout check 5000ms - server s1 httpecho.trigger.svc.cluster.local:80 \ No newline at end of file + server el-pac el-pipelines-as-code-interceptor.openshift-pipelines.svc.cluster.local:8080 diff --git a/exploration/triggers/gateway/haproxy-deployment.yaml b/gitops/kcp/gateway/haproxy-deployment.yaml similarity index 81% rename from exploration/triggers/gateway/haproxy-deployment.yaml rename to gitops/kcp/gateway/haproxy-deployment.yaml index 5b532bbb4..5d8f7aa52 100644 --- a/exploration/triggers/gateway/haproxy-deployment.yaml +++ b/gitops/kcp/gateway/haproxy-deployment.yaml @@ -17,10 +17,14 @@ spec: containers: - name: gateway image: docker.io/haproxytech/haproxy-debian@sha256:7086aaf61dfe7f07fa36eafd4bb567a60564f7dd6fcf7cdafdfbbab19c01f0ba + resources: + requests: + cpu: 100m + memory: 256Mi ports: - name: http-port protocol: TCP - containerPort: 80 + containerPort: 8080 volumeMounts: - name: config mountPath: /usr/local/etc/haproxy/haproxy.cfg @@ -29,4 +33,4 @@ spec: volumes: - name: config configMap: - name: haproxy-config \ No newline at end of file + name: haproxy-config diff --git a/exploration/triggers/gateway/haproxy-ingress.yaml b/gitops/kcp/gateway/haproxy-ingress.yaml similarity index 63% rename from exploration/triggers/gateway/haproxy-ingress.yaml rename to gitops/kcp/gateway/haproxy-ingress.yaml index 56b041936..7cf1710f0 100644 --- a/exploration/triggers/gateway/haproxy-ingress.yaml +++ b/gitops/kcp/gateway/haproxy-ingress.yaml @@ -4,13 +4,13 @@ metadata: name: trigger spec: rules: - - host: trigger-gateway.kcp-apps.127.0.0.1.nip.io - http: + - http: paths: - path: / pathType: Prefix backend: service: - name: httpecho + name: gateway port: - number: 80 \ No newline at end of file + number: 80 + diff --git a/exploration/triggers/gateway/haproxy-service.yaml b/gitops/kcp/gateway/haproxy-service.yaml similarity index 90% rename from exploration/triggers/gateway/haproxy-service.yaml rename to gitops/kcp/gateway/haproxy-service.yaml index e827f2784..0f3b71ee6 100644 --- a/exploration/triggers/gateway/haproxy-service.yaml +++ b/gitops/kcp/gateway/haproxy-service.yaml @@ -9,4 +9,4 @@ spec: targetPort: http-port protocol: TCP selector: - app: gateway \ No newline at end of file + app: gateway diff --git a/gitops/kcp/gateway/kustomization.yaml b/gitops/kcp/gateway/kustomization.yaml new file mode 100644 index 000000000..845d8252c --- /dev/null +++ b/gitops/kcp/gateway/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - haproxy-cfg-cm.yaml + - haproxy-deployment.yaml + - haproxy-ingress.yaml + - haproxy-service.yaml diff --git a/gitops/pac/README.md b/gitops/pac/README.md index 8e2953a03..cbe593c6c 100644 --- a/gitops/pac/README.md +++ b/gitops/pac/README.md @@ -36,4 +36,8 @@ KUBECONFIG="/pathto/kubeconfig" GITOPS_REPO="https://gitops.org.com/org/pipeline ## Pipelines -Pipelines for the registration of new clusters via ArgoCD and kcp are available. +Pipelines are available for: + +- the registration of new workload clusters to kcp +- the installation of the Tekton components on the workload cluster leveraging Argo CD +- the deployment of the gateway through kcp diff --git a/gitops/sre/.tekton/pln-svc-registration.yaml b/gitops/sre/.tekton/cluster-setup.yaml similarity index 100% rename from gitops/sre/.tekton/pln-svc-registration.yaml rename to gitops/sre/.tekton/cluster-setup.yaml diff --git a/gitops/sre/.tekton/gateway-deployment.yaml b/gitops/sre/.tekton/gateway-deployment.yaml new file mode 100644 index 000000000..099521901 --- /dev/null +++ b/gitops/sre/.tekton/gateway-deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: gateway-deployment + annotations: + pipelinesascode.tekton.dev/on-target-branch: "[main]" + pipelinesascode.tekton.dev/on-event: "[push]" + pipelinesascode.tekton.dev/task: "git-clone" +spec: + params: + - name: repo_url + value: "{{repo_url}}" + - name: revision + value: "{{revision}}" + pipelineSpec: + params: + - name: repo_url + - name: revision + workspaces: + - name: source + tasks: + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + params: + - name: depth + value: "500" + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: kcp-register + runAfter: + - fetch-repository + workspaces: + - name: source + workspace: source + taskSpec: + workspaces: + - name: source + steps: + - name: gateway-deployment + image: ghcr.io/openshift-pipelines/gateway-deployment:main + workingDir: $(workspaces.source.path) + env: + - name: DATA_DIR + value: $(workspaces.source.path)/gitops/sre + - name: KCP_ORG + value: "pipelines-service" + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi diff --git a/images/gateway/Dockerfile b/images/gateway/Dockerfile new file mode 100644 index 000000000..b2ff4187d --- /dev/null +++ b/images/gateway/Dockerfile @@ -0,0 +1,13 @@ +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6 +WORKDIR / +RUN mkdir /workspace && chmod 777 /workspace && chown 65532:65532 /workspace +ENV HOME /tmp/home +RUN mkdir $HOME && chmod 777 $HOME && chown 65532:65532 $HOME +COPY ./install.sh /usr/local/bin/install.sh +RUN KUBE_VERSION=v1.24.0 && \ + curl -L -o /usr/local/bin/kubectl "https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/amd64/kubectl" && \ + chmod 755 /usr/local/bin/kubectl +USER 65532:65532 +VOLUME /workspace +WORKDIR /workspace +ENTRYPOINT ["/usr/local/bin/install.sh"] diff --git a/images/gateway/install.sh b/images/gateway/install.sh new file mode 100755 index 000000000..536b13c80 --- /dev/null +++ b/images/gateway/install.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Copyright 2022 The pipelines-service Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +usage() { + + printf "Usage: KCP_ORG=root:pipelines-service KCP_WORKSPACE=infra DATA_DIR=/workspace ./install.sh\n\n" + + # Parameters + printf "The following parameters need to be passed to the script:\n" + printf "KCP_ORG: the organistation for which the workload clusters need to be registered, i.e.: root:pipelines-service\n" + printf "KCP_WORKSPACE: the name of the workspace where the gateway gets deployed (created if it does not exist), i.e: infra. If the workspace differs from the one where the WorkloadCluster has been created an APIBinding will need to be added\n" + printf "DATA_DIR: the location of the cluster files\n" +} + +prechecks () { + KCP_ORG="${KCP_ORG:-}" + if [[ -z "${KCP_ORG}" ]]; then + printf "KCP_ORG not set\n\n" + usage + exit 1 + fi + + KCP_WORKSPACE="${KCP_WORKSPACE:-}" + if [[ -z "${KCP_WORKSPACE}" ]]; then + printf "KCP_WORKSPACE not set\n\n" + usage + exit 1 + fi + + DATA_DIR="${DATA_DIR:-}" + if [[ -z "${DATA_DIR}" ]]; then + printf "DATA_DIR not set\n\n" + usage + exit 1 + fi +} + +# populate kcp_kcfg with the location of the kubeconfig for connecting to kcp +kcp_kubeconfig() { + if files=($(ls $DATA_DIR/credentials/kubeconfig/kcp/*.kubeconfig 2>/dev/null)); then + if [ ${#files[@]} -ne 1 ]; then + printf "A single kubeconfig file is expected at %s\n" "$DATA_DIR/credentials/kubeconfig/kcp" + usage + exit 1 + fi + kcp_kcfg="${files[0]}" + else + printf "A single kubeconfig file is expected at %s\n" "$DATA_DIR/credentials/kubeconfig/kcp" + usage + exit 1 + fi +} + +switch_org() { + KUBECONFIG=${kcp_kcfg} kubectl kcp workspace use ${KCP_ORG} + if ! (KUBECONFIG=${kcp_kcfg} kubectl api-resources >> /dev/null 2>&1); then + printf "%s is not a valid organization, wrong kubectl context in use or connectivity issue\n" ${KCP_ORG} + usage + exit 1 + fi +} + +switch_ws() { + if (KUBECONFIG=${kcp_kcfg} kubectl get workspaces -o name | grep "${KCP_WORKSPACE}"); then + printf "use existing workspace\n" + KUBECONFIG=${kcp_kcfg} kubectl kcp workspace use "${KCP_WORKSPACE}" + + else + printf "creating workspace %s\n" "${KCP_WORKSPACE}" + KUBECONFIG=${kcp_kcfg} kubectl kcp workspace create "${KCP_WORKSPACE}" --enter + fi +} + +install_gateway () { + CONFIG_DIR="${DATA_DIR}/environment/kcp/gateway" + KUBECONFIG=${kcp_kcfg} kubectl apply -k "${CONFIG_DIR}" +} + +main() { + prechecks + kcp_kubeconfig + printf "Switching to organization %s\n" "${KCP_ORG}" + switch_org + printf "Switching to workspace %s\n" "${KCP_WORKSPACE}" + switch_ws + printf "Installing gateway\n" + install_gateway +} + +if [ "${BASH_SOURCE[0]}" == "$0" ]; then + main "$@" +fi diff --git a/images/kcp-registrar/register.sh b/images/kcp-registrar/register.sh index 5823ee9c3..732008c02 100755 --- a/images/kcp-registrar/register.sh +++ b/images/kcp-registrar/register.sh @@ -39,7 +39,7 @@ prechecks () { exit 1 fi - KCP_ORG=${KCP_WORKSPACE:-} + KCP_WORKSPACE=${KCP_WORKSPACE:-} if [[ -z "${KCP_WORKSPACE}" ]]; then printf "KCP_WORKSPACE not set\n\n" usage