Merge pull request #227 from RADAR-base/release-1.1.0

Release 1.1.0

.github/ci_config/bin/install-dependencies

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+mkdir /tmp/dependencies
+cd /tmp/dependencies
+
+sudo apt update
+sudo apt install -y openjdk-11-jre-headless curl coreutils
+
+OS="$(uname | tr '[:upper:]' '[:lower:]')"
+ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')"
+
+if [ -n "${KUBECTL_VERSION}" ]; then
+  KUBECTL_VERSION=${KUBECTL_VERSION}
+else
+  KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
+fi
+curl -LO "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/$OS/$ARCH/kubectl"
+curl -LO "https://dl.k8s.io/$KUBECTL_VERSION/bin/$OS/$ARCH/kubectl.sha256"
+echo "$(cat kubectl.sha256) kubectl" | sha256sum --check
+sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+kubectl version --client --output=yaml
+
+# if [ -n "${KREW_VERSION}" ]; then
+#   KREW_VERSION=${KREW_VERSION}
+# else
+#   KREW_VERSION=$(curl -Ls "https://github.com/kubernetes-sigs/krew/releases" | grep 'href="/kubernetes-sigs/krew/releases/tag/v[0-9]*.[0-9]*.[0-9]\"' | sed -E 's/.*\/kubernetes-sigs\/krew\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1)
+# fi
+# KREW="krew-${OS}_${ARCH}"
+# curl -LO "https://github.com/kubernetes-sigs/krew/releases/download/$KREW_VERSION/$KREW.tar.gz"
+# curl -LO "https://github.com/kubernetes-sigs/krew/releases/download/$KREW_VERSION/$KREW.tar.gz.sha256"
+# echo "$(cat $KREW.tar.gz.sha256) $KREW.tar.gz" | sha256sum --check
+# tar zxvf "${KREW}.tar.gz"
+# ./"${KREW}" install krew
+# export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
+# kubectl krew version
+
+if [ -n "${HELM_VERSION}" ]; then
+  HELM_VERSION=${HELM_VERSION}
+else
+  HELM_VERSION=$(curl -Ls "https://github.com/helm/helm/releases" | grep 'href="/helm/helm/releases/tag/v3.[0-9]*.[0-9]*\"' | sed -E 's/.*\/helm\/helm\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1)
+fi
+curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+chmod 700 get_helm.sh
+./get_helm.sh --version $HELM_VERSION
+helm version
+
+if [ -n "${HELM_DIFF_VERSION}" ]; then
+  HELM_DIFF_VERSION="--version ${HELM_DIFF_VERSION}"
+else
+  HELM_DIFF_VERSION=""
+fi
+helm plugin install $HELM_DIFF_VERSION https://github.com/databus23/helm-diff
+helm diff version
+
+if [ -n "${HELMFILE_VERSION}" ]; then
+  HELMFILE_VERSION=${HELMFILE_VERSION}
+else
+  HELMFILE_VERSION=$(curl -Ls "https://github.com/helmfile/helmfile/releases" | \grep 'href="/helmfile/helmfile/releases/tag/v[0-9]*.[0-9]*.[0-9]*\"' | sed -E 's/.*\/helmfile\/helmfile\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1)
+fi
+curl -fsSL -o helmfile.tar.gz "https://github.com/helmfile/helmfile/releases/download/$HELMFILE_VERSION/helmfile_$(echo -n $HELMFILE_VERSION | cut -c 2-)_${OS}_${ARCH}.tar.gz"
+tar xzf helmfile.tar.gz
+sudo install -o root -g root -m 0755 helmfile /usr/local/bin/helmfile
+helmfile --version
+
+if [ -n "${YQ_VERSION}" ]; then
+  YQ_VERSION=${YQ_VERSION}
+else
+  YQ_VERSION=$(curl -Ls "https://github.com/mikefarah/yq/releases" | \grep 'href="/mikefarah/yq/releases/tag/v[0-9]*.[0-9]*.[0-9]*\"' | sed -E 's/.*\/mikefarah\/yq\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1)
+fi
+curl -fsSL -o yq "https://github.com/mikefarah/yq/releases/download/$YQ_VERSION/yq_${OS}_${ARCH}"
+sudo install -o root -g root -m 0755 yq /usr/local/bin/yq
+yq --version
+
+rm -rf /tmp/dependencies

.github/ci_config/install-all.yaml

@@ -0,0 +1,166 @@
+atomicInstall: false
+base_timeout: 180
+
+mongodb:
+  _install: true
+
+elasticsearch:
+  _install: true
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "500Mi"
+
+graylog:
+  _install: true
+
+fluent_bit:
+  _install: true
+
+cert_manager:
+  _install: true
+
+kube_prometheus_stack:
+  _install: true
+
+nginx_ingress:
+  _install: true
+
+kafka_manager:
+  _install: false
+
+cert_manager_letsencrypt:
+  _install: true
+
+confluent_cloud:
+  enabled: false
+
+cp_zookeeper:
+  _install: true
+
+cp_kafka:
+  _install: true
+  persistence:
+    size: 1Gi
+
+cp_schema_registry:
+  _install: true
+
+catalog_server:
+  _install: true
+
+postgresql:
+  _install: true
+  primary:
+    resources:
+      requests:
+        cpu: 50m
+    persistence:
+      size: 1Gi
+
+management_portal:
+  _install: true
+
+app_config:
+  _install: true
+
+app_config_frontend:
+  _install: true
+
+radar_appserver_postgresql:
+  _install: true
+  primary:
+    resources:
+      requests:
+        cpu: 50m  
+    persistence:
+      size: 1Gi
+
+radar_appserver:
+  _install: true
+
+radar_fitbit_connector:
+  _install: true
+
+radar_rest_sources_authorizer:
+  _install: true
+
+radar_rest_sources_backend:
+  _install: true
+
+timescaledb:
+  _install: true
+  primary:
+    resources:
+      requests:
+        cpu: 50m
+    persistence:
+      size: 1Gi
+
+radar_grafana:
+  _install: true
+
+radar_jdbc_connector:
+  _install: true
+
+radar_gateway:
+  _install: true
+
+radar_backend_monitor:
+  _install: true
+
+radar_backend_stream:
+  _install: true
+
+radar_integration:
+  _install: true
+
+redis:
+  _install: true
+
+minio:
+  _install: true
+  persistence:
+    size: 1Gi
+
+radar_s3_connector:
+  _install: true
+
+s3_proxy:
+  _install: true
+
+radar_output:
+  _install: true
+
+radar_upload_postgresql:
+  _install: true
+  primary:
+    resources:
+      requests:
+        cpu: 50m
+    persistence:
+      size: 1Gi
+
+radar_upload_connect_backend:
+  _install: true
+
+radar_upload_connect_frontend:
+  _install: true
+
+radar_upload_source_connector:
+  _install: true
+
+ccSchemaRegistryProxy:
+  _install: false
+
+radar_push_endpoint:
+  _install: true
+
+radar_jdbc_connector_agg:
+  _install: true
+
+ksql_server:
+  _install: true
+
+velero:
+  _install: true

.github/ci_config/secrets.yaml.gotmpl

@@ -0,0 +1,5 @@
+management_portal:
+  keystore: {{ readFile "../../etc/management-portal/keystore.p12" | b64enc | quote }}
+
+radar_appserver:
+  google_application_credentials: {{ requiredEnv "FIREBASE_ADMINSDK_JSON" | quote }}

.github/workflows/push.yaml

@@ -1,34 +1,113 @@
-name: CI
-on: [push]
+name: Install applications on Kubernets
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  install:
+    strategy:
+      matrix:
+        KUBECTL_VERSION: ["v1.23.0", "v1.24.0", "v1.25.0", "v1.26.0"]
+        include:
+          - KUBECTL_VERSION: "v1.23.0"
+            K3S_VERSION: "v1.23.15+k3s1"
+          - KUBECTL_VERSION: "v1.24.0"
+            K3S_VERSION: "v1.24.9+k3s2"
+          - KUBECTL_VERSION: "v1.25.0"
+            K3S_VERSION: "v1.25.5+k3s2"
+          - KUBECTL_VERSION: "v1.26.0"
+            K3S_VERSION: "v1.26.0+k3s2"
+    runs-on: self-hosted
     steps:
-    - uses: actions/[email protected]
+    - name: Cleanup
+      continue-on-error: true
+      run: |
+        ls -la ./
+        rm -rf ./* || true
+        rm -rf ./.??* || true
+        ls -la ./
+        /usr/local/bin/k3s-uninstall.sh
+
+    - uses: actions/checkout@v3
 
-    - name: Setup helmfile
-      uses: mamezou-tech/[email protected]
+    - name: Install dependencies
+      env:
+        KUBECTL_VERSION: ${{ matrix.KUBECTL_VERSION }}
+        HELM_VERSION: "v3.10.2"
+        HELM_DIFF_VERSION: "3.4.2"
+        HELMFILE_VERSION: "v0.148.1"
+        YQ_VERSION: "v4.30.1"
+      run: |
+        ./.github/ci_config/bin/install-dependencies
 
-    - name: Set up helmfile config
-      run: cp .github/environments.yaml environments.yaml
+    - name: Prepare the environment
+      run: |
+        DNAME='CN=CI,O=TheHyve,L=Utrecht,C=NL' ./bin/init
+        yq -i -f process '.environments.default.values += ["../.github/ci_config/install-all.yaml", "../.github/ci_config/secrets.yaml.gotmpl"]' environments.yaml
 
     - name: Run helmfile template
+      env:
+        FIREBASE_ADMINSDK_JSON: ${{ secrets.FIREBASE_ADMINSDK_JSON }}
       run: |
         helmfile template
 
-    # - name: Start a local k8s cluster
-    #   uses: jupyterhub/action-k3s-helm@v3
-    #   with:
-    #     # See available:
-    #     # - k3s release channels at https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml
-    #     # - k3s versions at https://github.com/k3s-io/k3s/tags
-    #     # - helm versions at https://github.com/helm/helm/tags
-    #     k3s-channel: latest
-    #
-    # - name: Verify function of k8s, kubectl, and helm
-    #   run: |
-    #     echo "kubeconfig: $KUBECONFIG"
-    #     kubectl version
-    #     kubectl get pods --all-namespaces
-    #
-    #     helmfile sync --concurrency 1
+    - name: Install K3S
+      env:
+        INSTALL_K3S_VERSION: ${{ matrix.K3S_VERSION }}
+        INSTALL_K3S_SYMLINK: "skip"
+        K3S_KUBECONFIG_MODE: "644"
+      run: |
+        curl -sfL https://get.k3s.io | sh -s - --disable traefik --disable-helm-controller
+        cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
+
+    - name: Install RADAR-Kubernetes
+      env:
+        FIREBASE_ADMINSDK_JSON: ${{ secrets.FIREBASE_ADMINSDK_JSON }}
+      run: |
+        echo "kubeconfig: $KUBECONFIG"
+        kubectl get pods --all-namespaces
+
+        helmfile sync --concurrency 1
+
+    # Enable tmate debugging of manually-triggered workflows if the input option was provided
+    - name: Manually triggered tmate session
+      uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
+
+    - name: Setup tmate session if jobs have failed
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 15
+      with:
+        limit-access-to-actor: true
+
+    - name: Report cluster state
+      if: always()
+      run: |
+        kubectl get pods --all-namespaces
+
+  check-gitignore:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Check gitignore
+      run: |
+        echo "Checking whether any files that should be ignored are uploaded"
+        echo
+        if find . -not -path './.git/*' -type f | git check-ignore --stdin --no-index; then
+          echo
+          echo "Remove the above files from this branch"
+          exit 1
+        else
+          echo "No ignored files have been added"
+        fi