Skip to content

System Tests Open Source #7157

System Tests Open Source

System Tests Open Source #7157

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: System Tests Open Source
on:
push:
branches:
- '.+-system-tests'
schedule:
# * is a special character in YAML so you have to quote this string
# Run the system tests every 3 hours
- cron: '0 */3 * * *'
workflow_dispatch:
inputs:
docker_registry:
description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
required: true
default: 'ghcr.io/'
docker_repo:
description: 'Docker repo to pull images from (default: mlrun)'
required: true
default: 'mlrun'
clean_resources_in_teardown:
description: 'Clean resources created by test (like project) in each test teardown (default: true - perform clean)'
required: true
default: 'true'
type: choice
options:
- 'true'
- 'false'
debug_enabled:
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: 'false'
type: choice
options:
- 'true'
- 'false'
env:
NAMESPACE: mlrun
MLRUN_API_NODE_PORT: 30070
jobs:
run-system-tests-opensource-ci:
name: Run System Tests Open Source
runs-on: ubuntu-latest
# let's not run this on every fork, change to your fork when developing
if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v4
# since github-actions gives us 14G only, and fills it up with some garbage
- name: Freeing up disk space
run: |
"${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
- name: Set up python
uses: actions/setup-python@v5
with:
python-version: 3.9
cache: pip
- name: Install automation scripts dependencies and add mlrun to dev packages
run: |
pip install \
-r automation/requirements.txt \
-r dockerfiles/mlrun-api/requirements.txt \
-r dev-requirements.txt \
-r extras-requirements.txt \
&& pip install -e .
sudo apt-get install curl jq
# TODO: How can we avoid these duplicate lines from the enterprise system tests
- name: Extract git branch
id: git_info
run: |
echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
- name: Extract git hashes from upstream and latest version
id: git_upstream_info
run: |
echo "mlrun_hash=$( \
cd /tmp && \
git clone --single-branch --branch development https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \
cd mlrun-upstream && \
git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
cd .. && \
rm -rf mlrun-upstream)" >> $GITHUB_OUTPUT
echo "ui_hash=$( \
cd /tmp && \
git clone --single-branch --branch development https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \
cd mlrun-ui && \
git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
cd .. && \
rm -rf mlrun-ui)" >> $GITHUB_OUTPUT
echo "unstable_version_prefix=$(cat automation/version/unstable_version_prefix)" >> $GITHUB_OUTPUT
- name: Set computed versions params
id: computed_params
run: |
action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \
upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \
export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`}
echo "mlrun_hash=$(echo $mlrun_hash)" >> $GITHUB_OUTPUT
action_mlrun_ui_hash=${{ steps.git_action_ui_info.outputs.ui_hash }} && \
upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`}
echo "ui_hash=$(echo $ui_hash)" >> $GITHUB_OUTPUT
echo "mlrun_version=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}+$mlrun_hash)" >> $GITHUB_OUTPUT
echo "mlrun_docker_tag=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$mlrun_hash)" >> $GITHUB_OUTPUT
echo "mlrun_ui_version=${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$ui_hash" >> $GITHUB_OUTPUT
echo "mlrun_docker_repo=$( \
input_docker_repo=$INPUT_DOCKER_REPO && \
echo ${input_docker_repo:-mlrun})" >> $GITHUB_OUTPUT
echo "mlrun_docker_registry=$( \
input_docker_registry=$INPUT_DOCKER_REGISTRY && \
echo ${input_docker_registry:-ghcr.io/})" >> $GITHUB_OUTPUT
echo "mlrun_system_tests_clean_resources=$( \
input_system_tests_clean_resources=$INPUT_CLEAN_RESOURCES_IN_TEARDOWN && \
echo ${input_system_tests_clean_resources:-true})" >> $GITHUB_OUTPUT
env:
INPUT_DOCKER_REPO: ${{ github.event.inputs.docker_repo }}
INPUT_DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }}
INPUT_CLEAN_RESOURCES_IN_TEARDOWN: ${{ github.event.inputs.clean_resources_in_teardown }}
- uses: azure/setup-helm@v4
with:
version: "v3.9.1"
- uses: manusa/[email protected]
with:
minikube version: "v1.32.0"
kubernetes version: "v1.28.5"
driver: docker
github token: ${{ github.token }}
# I couldn't find a way to configure the IP (https://github.com/kubernetes/minikube/issues/951)
# but this seems to work
start args: '--addons=registry --insecure-registry="192.168.49.2:5000"'
- name: Install MLRun CE helm chart
run: |
# TODO: There is a modification to the helm chart that we are doing right now:
# The mlrun DB is set as the old SQLite db. There is a bug in github workers when trying to run a mysql
# server pod in minikube installed on the worker, the mysql pod crashes. There isn't much information
# about this issue online as this isn't how github expect you to use mysql in workflows - the worker
# has a mysql server installed directly on it and should be enabled and used as the DB. So we might
# want in the future to use that instead, unless the mysql will be able to come up without crashing.
#
# TODO: Align the mlrun config env vars with the ones in the prepare.py script to avoid further inconsistencies.
python automation/deployment/ce.py deploy \
--verbose \
--minikube \
--namespace=${NAMESPACE} \
--registry-secret-name="" \
--sqlite /mlrun/db/mlrun.db \
--override-mlrun-api-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \
--override-mlrun-log-collector-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/log-collector:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \
--override-mlrun-ui-image="ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}" \
--set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER="mlrun[complete] @ git+https://github.com/mlrun/mlrun@${{ steps.computed_params.outputs.mlrun_hash }}"' \
--set mlrun.api.extraEnvKeyValue.MLRUN_IMAGES_REGISTRY="${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
--set mlrun.api.extraEnvKeyValue.MLRUN_LOG_LEVEL="DEBUG" \
--set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__SCHEDULING__MIN_ALLOWED_INTERVAL="0 seconds"' \
--set mlrun.api.extraEnvKeyValue.MLRUN_MODEL_ENDPOINT_MONITORING__PARQUET_BATCHING_MAX_EVENTS="100"
- name: Prepare system tests env
run: |
python automation/system_test/prepare.py env \
--mlrun-dbpath "http://$(minikube ip):${MLRUN_API_NODE_PORT}" \
--github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}"
# Enable tmate debugging of manually-triggered workflows if the input option was provided
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
with:
# run in detach mode to allow the workflow to continue running while session is active
# this will wait up to 10 minutes AFTER the entire job is done. Once user connects to the session,
# it will wait until the user disconnects before finishing up the job.
detached: true
- name: Run system tests
timeout-minutes: 180
run: |
MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}" \
MLRUN_VERSION="${{ steps.computed_params.outputs.mlrun_version }}" \
make test-system-open-source
- name: Output some logs in case of failure
if: ${{ failure() }}
# add set -x to print commands before executing to make logs reading easier
run: |
set -x
minikube ip
minikube logs
minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
minikube kubectl -- --namespace ${NAMESPACE} get all
minikube kubectl -- --namespace ${NAMESPACE} get all -o yaml
minikube kubectl -- --namespace ${NAMESPACE} describe pods
minikube kubectl -- --namespace ${NAMESPACE} get cm
minikube kubectl -- --namespace ${NAMESPACE} get cm -o yaml
minikube kubectl -- --namespace ${NAMESPACE} get secrets
minikube kubectl -- --namespace ${NAMESPACE} get secrets -o yaml
minikube kubectl -- --namespace ${NAMESPACE} get pvc
minikube kubectl -- --namespace ${NAMESPACE} get pv
set +x