System Tests Open Source #4838
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2023 Iguazio | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| name: System Tests Open Source | |
| on: | |
| push: | |
| branches: | |
| - '.+-system-tests' | |
| schedule: | |
| # * is a special character in YAML so you have to quote this string | |
| # Run the system tests every 3 hours | |
| - cron: '0 */3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| docker_registry: | |
| description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)' | |
| required: true | |
| default: 'ghcr.io/' | |
| docker_repo: | |
| description: 'Docker repo to pull images from (default: mlrun)' | |
| required: true | |
| default: 'mlrun' | |
| clean_resources_in_teardown: | |
| description: 'Clean resources created by test (like project) in each test teardown (default: true - perform clean)' | |
| required: true | |
| default: 'true' | |
| type: choice | |
| options: | |
| - 'true' | |
| - 'false' | |
| debug_enabled: | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: 'false' | |
| type: choice | |
| options: | |
| - 'true' | |
| - 'false' | |
| env: | |
| NAMESPACE: mlrun | |
| MLRUN_API_NODE_PORT: 30070 | |
| jobs: | |
| run-system-tests-opensource-ci: | |
| name: Run System Tests Open Source | |
| runs-on: ubuntu-latest | |
| # let's not run this on every fork, change to your fork when developing | |
| if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # since github-actions gives us 14G only, and fills it up with some garbage | |
| - name: Freeing up disk space | |
| run: | | |
| "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh" | |
| - name: Set up python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9 | |
| cache: pip | |
| - name: Install automation scripts dependencies and add mlrun to dev packages | |
| run: | | |
| pip install \ | |
| -r automation/requirements.txt \ | |
| -r dockerfiles/test-system/requirements.txt \ | |
| -r dockerfiles/mlrun-api/requirements.txt \ | |
| -r dev-requirements.txt \ | |
| -r extras-requirements.txt \ | |
| && pip install -e . | |
| sudo apt-get install curl jq | |
| # TODO: How can we avoid these duplicate lines from the enterprise system tests | |
| - name: Extract git branch | |
| id: git_info | |
| run: | | |
| echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT | |
| - name: Extract git hashes from upstream and latest version | |
| id: git_upstream_info | |
| run: | | |
| echo "mlrun_hash=$( \ | |
| cd /tmp && \ | |
| git clone --single-branch --branch development https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \ | |
| cd mlrun-upstream && \ | |
| git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \ | |
| cd .. && \ | |
| rm -rf mlrun-upstream)" >> $GITHUB_OUTPUT | |
| echo "ui_hash=$( \ | |
| cd /tmp && \ | |
| git clone --single-branch --branch development https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \ | |
| cd mlrun-ui && \ | |
| git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \ | |
| cd .. && \ | |
| rm -rf mlrun-ui)" >> $GITHUB_OUTPUT | |
| echo "unstable_version_prefix=$(cat automation/version/unstable_version_prefix)" >> $GITHUB_OUTPUT | |
| - name: Set computed versions params | |
| id: computed_params | |
| run: | | |
| action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \ | |
| upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \ | |
| export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`} | |
| echo "mlrun_hash=$(echo $mlrun_hash)" >> $GITHUB_OUTPUT | |
| action_mlrun_ui_hash=${{ steps.git_action_ui_info.outputs.ui_hash }} && \ | |
| upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \ | |
| export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`} | |
| echo "ui_hash=$(echo $ui_hash)" >> $GITHUB_OUTPUT | |
| echo "mlrun_version=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}+$mlrun_hash)" >> $GITHUB_OUTPUT | |
| echo "mlrun_docker_tag=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$mlrun_hash)" >> $GITHUB_OUTPUT | |
| echo "mlrun_ui_version=${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$ui_hash" >> $GITHUB_OUTPUT | |
| echo "mlrun_docker_repo=$( \ | |
| input_docker_repo=$INPUT_DOCKER_REPO && \ | |
| echo ${input_docker_repo:-mlrun})" >> $GITHUB_OUTPUT | |
| echo "mlrun_docker_registry=$( \ | |
| input_docker_registry=$INPUT_DOCKER_REGISTRY && \ | |
| echo ${input_docker_registry:-ghcr.io/})" >> $GITHUB_OUTPUT | |
| echo "mlrun_system_tests_clean_resources=$( \ | |
| input_system_tests_clean_resources=$INPUT_CLEAN_RESOURCES_IN_TEARDOWN && \ | |
| echo ${input_system_tests_clean_resources:-true})" >> $GITHUB_OUTPUT | |
| env: | |
| INPUT_DOCKER_REPO: ${{ github.event.inputs.docker_repo }} | |
| INPUT_DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }} | |
| INPUT_CLEAN_RESOURCES_IN_TEARDOWN: ${{ github.event.inputs.clean_resources_in_teardown }} | |
| - uses: azure/setup-helm@v3 | |
| with: | |
| version: "v3.9.1" | |
| - uses: manusa/[email protected] | |
| with: | |
| minikube version: "v1.28.0" | |
| kubernetes version: "v1.23.9" | |
| driver: docker | |
| github token: ${{ github.token }} | |
| # I couldn't find a way to configure the IP (https://github.com/kubernetes/minikube/issues/951) | |
| # but this seems to work | |
| start args: '--addons=registry --insecure-registry="192.168.49.2:5000"' | |
| - name: Install MLRun CE helm chart | |
| run: | | |
| # TODO: There are a couple of modifications to the helm chart that we are doing right now: | |
| # 1. The grafana prometheus stack is disabled as there are currently no system tests checking its | |
| # functionality. Once the model monitoring feature is complete and we have system tests for it, we | |
| # can enable it. | |
| # 2. The mlrun DB is set as the old SQLite db. There is a bug in github workers when trying to run a mysql | |
| # server pod in minikube installed on the worker, the mysql pod crashes. There isn't much information | |
| # about this issue online as this isn't how github expect you to use mysql in workflows - the worker | |
| # has a mysql server installed directly on it and should be enabled and used as the DB. So we might | |
| # want in the future to use that instead, unless the mysql will be able to come up without crashing. | |
| # | |
| # TODO: Align the mlrun config env vars with the ones in the prepare.py script to avoid further inconsistencies. | |
| python automation/deployment/ce.py deploy \ | |
| --verbose \ | |
| --minikube \ | |
| --namespace=${NAMESPACE} \ | |
| --registry-secret-name="" \ | |
| --disable-prometheus-stack \ | |
| --sqlite /mlrun/db/mlrun.db \ | |
| --override-mlrun-api-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \ | |
| --override-mlrun-ui-image="ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}" \ | |
| --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER="mlrun[complete] @ git+https://github.com/mlrun/mlrun@${{ steps.computed_params.outputs.mlrun_hash }}"' \ | |
| --set mlrun.api.extraEnvKeyValue.MLRUN_IMAGES_REGISTRY="${{ steps.computed_params.outputs.mlrun_docker_registry }}" \ | |
| --set mlrun.api.extraEnvKeyValue.MLRUN_LOG_LEVEL="DEBUG" \ | |
| --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__SCHEDULING__MIN_ALLOWED_INTERVAL="0 seconds"' \ | |
| --set mlrun.api.extraEnvKeyValue.MLRUN_MODEL_ENDPOINT_MONITORING__PARQUET_BATCHING_MAX_EVENTS="100" | |
| - name: Prepare system tests env | |
| run: | | |
| python automation/system_test/prepare.py env \ | |
| --mlrun-dbpath "http://$(minikube ip):${MLRUN_API_NODE_PORT}" \ | |
| --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}" | |
| # Enable tmate debugging of manually-triggered workflows if the input option was provided | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }} | |
| with: | |
| # run in detach mode to allow the workflow to continue running while session is active | |
| # this will wait up to 10 minutes AFTER the entire job is done. Once user connects to the session, | |
| # it will wait until the user disconnects before finishing up the job. | |
| detached: true | |
| - name: Run system tests | |
| timeout-minutes: 180 | |
| run: | | |
| MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}" \ | |
| MLRUN_VERSION="${{ steps.computed_params.outputs.mlrun_version }}" \ | |
| make test-system-open-source | |
| - name: Output some logs in case of failure | |
| if: ${{ failure() }} | |
| # add set -x to print commands before executing to make logs reading easier | |
| run: | | |
| set -x | |
| minikube ip | |
| minikube logs | |
| minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1 | |
| minikube kubectl -- --namespace ${NAMESPACE} get all | |
| minikube kubectl -- --namespace ${NAMESPACE} get all -o yaml | |
| minikube kubectl -- --namespace ${NAMESPACE} describe pods | |
| minikube kubectl -- --namespace ${NAMESPACE} get cm | |
| minikube kubectl -- --namespace ${NAMESPACE} get cm -o yaml | |
| minikube kubectl -- --namespace ${NAMESPACE} get secrets | |
| minikube kubectl -- --namespace ${NAMESPACE} get secrets -o yaml | |
| minikube kubectl -- --namespace ${NAMESPACE} get pvc | |
| minikube kubectl -- --namespace ${NAMESPACE} get pv | |
| set +x |