From dd8fa5dfe29758d588452c84634a4f1a60bef839 Mon Sep 17 00:00:00 2001 From: Abhishek Gaikwad Date: Mon, 11 Dec 2023 02:38:55 -0800 Subject: [PATCH] github-ci: update pipeline for transfomers Signed-off-by: Abhishek Gaikwad --- .github/workflows/transformer-docker.yml | 229 +++++++++++------- transformers/Makefile | 46 ---- transformers/tests/base.py | 4 +- transformers/tests/test_compress.py | 47 ---- transformers/tests/test_echo.py | 6 - transformers/tests/test_ffmpeg.py | 28 --- transformers/tests/test_go_echo.py | 6 - transformers/tests/test_keras_transformer.py | 22 -- transformers/tests/test_md5.py | 5 - transformers/tests/test_tar2tf.py | 7 - .../tests/test_torchvision_transformer.py | 14 -- 11 files changed, 150 insertions(+), 264 deletions(-) diff --git a/.github/workflows/transformer-docker.yml b/.github/workflows/transformer-docker.yml index b6c90e7..296c629 100644 --- a/.github/workflows/transformer-docker.yml +++ b/.github/workflows/transformer-docker.yml @@ -4,67 +4,62 @@ on: workflow_dispatch: inputs: build_echo_image: - description: 'Build TRANSFORMER_ECHO Image (aistorage/transformer_echo:latest)' + description: 'Build transformer_echo and transformer_echo_go image' required: true type: boolean - default: true - build_go_echo_image: - description: 'Build TRANSFORMER_GO_ECHO Image (aistorage/transformer_echo_go:latest)' - required: true - type: boolean - default: true + default: false build_hello_world_image: - description: 'Build TRANSFORMER_HELLO_WORLD Image (aistorage/transformer_hello_world:latest)' + description: 'Build hello_world image' required: true type: boolean - default: true + default: false build_md5_image: - description: 'Build TRANSFORMER_MD5 Image (aistorage/transformer_md5:latest)' + description: 'Build transformer_md5 image' required: true type: boolean - default: true + default: false build_tar2tf_image: - description: 'Build TRANSFORMER_TAR2TF Image (aistorage/transformer_tar2tf:latest)' + description: 'Build transformer_tar2tf image' required: true type: boolean - default: true + default: false build_compress_image: - description: 'Build TRANSFORMER_COMPRESS Image (aistorage/transformer_compress:latest)' - required: true - type: boolean - default: true - build_keras_preprocess_image: - description: 'Build TRANSFORMER_KERAS Image (aistorage/transformer_keras:latest)' + description: 'Build transformer_compress image' required: true type: boolean - default: true + default: false build_ffmpeg_image: - description: 'Build TRANSFORMER_FFMPEG Image (aistorage/transformer_ffmpeg:latest)' + description: 'Build transformer_ffmpeg image' required: true type: boolean - default: true + default: false build_torchvision_preprocess_image: - description: 'Build TRANSFORMER_TORCHVISION Image (aistorage/transformer_torchvision:latest)' + description: 'Build transformer_torchvision image' + required: true + type: boolean + default: false + build_keras_preprocess_image: + description: 'Build transformer_keras image' required: true type: boolean - default: true + default: false + push: paths: - 'transformers/**' env: ECHO_ENABLE: ${{ github.event.inputs.build_echo_image }} - GO_ECHO_ENABLE: ${{ github.event.inputs.build_go_echo_image }} HELLO_WORLD_ENABLE: ${{ github.event.inputs.build_hello_world_image }} MD5_ENABLE: ${{ github.event.inputs.build_md5_image }} TAR2TF_ENABLE: ${{ github.event.inputs.build_tar2tf_image }} COMPRESS_ENABLE: ${{ github.event.inputs.build_compress_image }} - KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }} FFMPEG_ENABLE: ${{ github.event.inputs.build_ffmpeg_image }} TORCHVISION_ENABLE: ${{ github.event.inputs.build_torchvision_preprocess_image }} + KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }} jobs: - build-test-tag: + build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -85,62 +80,121 @@ jobs: git clone https://github.com/NVIDIA/aistore.git cd aistore/deploy/dev/k8s ./minimal.sh + export AIS_ENDPOINT="http://$(minikube ip):8080" - name: Setup Python 3.11 uses: actions/setup-python@v2 with: python-version: 3.11 - - name: Install FFMPEG + - name: Check Path for Changes + id: paths_filter + uses: dorny/paths-filter@v2 + with: + filters: | + echo: + - transformers/echo/**/* + - transformers/go_echo/**/* + - transformers/tests/test_echo.py + - transformers/tests/test_go_echo.py + hello_world: + - transformers/hello_world/**/* + - transformers/go_hello_world/**/* + - transformers/tests/test_hello_world.py + md5: + - transformers/md5/**/* + - transformers/tests/test_md5.py + tar2tf: + - transformers/tar2tf/**/* + - transformers/tests/test_tar2tf.py + compress: + - transformers/compress/**/* + - transformers/tests/test_compress.py + ffmpeg: + - transformers/ffmpeg/**/* + - transformers/tests/test_ffmpeg.py + torchvision: + - transformers/torchvision_preprocess/**/* + - transformers/tests/test_torchvision_transformer.py + keras: + - transformers/keras_preprocess/**/* + - transformers/tests/test_keras_transformer.py + - name: Update var ECHO_ENABLE + if: steps.paths_filter.outputs.echo == 'true' + run: echo "ECHO_ENABLE=true" >> $GITHUB_ENV + - name: Update var HELLO_WORLD_ENABLE + if: steps.paths_filter.outputs.hello_world == 'true' + run: echo "HELLO_WORLD_ENABLE=true" >> $GITHUB_ENV + - name: Update var MD5_ENABLE + if: steps.paths_filter.outputs.md5 == 'true' + run: echo "MD5_ENABLE=true" >> $GITHUB_ENV + - name: Update var TAR2TF_ENABLE + if: steps.paths_filter.outputs.tar2tf == 'true' + run: echo "TAR2TF_ENABLE=true" >> $GITHUB_ENV + - name: Update var COMPRESS_ENABLE + if: steps.paths_filter.outputs.compress == 'true' + run: echo "COMPRESS_ENABLE=true" >> $GITHUB_ENV + - name: Update var FFMPEG_ENABLE + if: steps.paths_filter.outputs.ffmpeg == 'true' + run: echo "FFMPEG_ENABLE=true" >> $GITHUB_ENV + - name: Update var TORCHVISION_ENABLE + if: steps.paths_filter.outputs.torchvision == 'true' + run: echo "TORCHVISION_ENABLE=true" >> $GITHUB_ENV + - name: Update var KERAS_ENABLE + if: steps.paths_filter.outputs.keras == 'true' + run: echo "KERAS_ENABLE=true" >> $GITHUB_ENV + - name: Install Python Dependencies for Testing run: | - wget -nv http://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz - tar -xf ffmpeg-release-amd64-static.tar.xz - FFMPEG_DIR=$(ls | grep ffmpeg-*-static) - cp ${FFMPEG_DIR}/ffmpeg /usr/local/bin/ - cp ${FFMPEG_DIR}/ffprobe /usr/local/bin/ - rm -rf ffmpeg-*-static ffmpeg-release-amd64-static.tar.xz - ffmpeg -version - - name: Install Torchvision + make -B -C $GITHUB_WORKSPACE/transformers common_deps + - name: Build and Test ECHO + if: ${{ env.ECHO_ENABLE == 'true' }} run: | - pip install torchvision --index-url https://download.pytorch.org/whl/cpu - - name: Build & Push Transformer Docker Images w/ `test` Tag + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/hello_world all + make -B -C $GITHUB_WORKSPACE/transformers/go_hello_world all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_echo.py $GITHUB_WORKSPACE/transformers/tests/test_go_echo.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/echo all + make -B -C $GITHUB_WORKSPACE/transformers/go_echo all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_echo.py $GITHUB_WORKSPACE/transformers/tests/test_go_echo.py + - name: Build and Test HELLO_WORLD + if: ${{ env.HELLO_WORLD_ENABLE == 'true' }} run: | - pushd $GITHUB_WORKSPACE/transformers export GIT_TEST="true" - make -B all - popd - - name: Test Transformer Docker Images w/ `test` Tag + make -B -C $GITHUB_WORKSPACE/transformers/echo all + make -B -C $GITHUB_WORKSPACE/transformers/go_echo all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_hello_world.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/echo all + make -B -C $GITHUB_WORKSPACE/transformers/go_echo all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_hello_world.py + - name: Build and Test MD5 + if: ${{ env.MD5_ENABLE == 'true' }} run: | - cd $GITHUB_WORKSPACE/transformers/ - export AIS_ENDPOINT="http://$(minikube ip):8080" export GIT_TEST="true" - make test - - build-latest-tag: - needs: build-test-tag - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Docker Login - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Setup Minikube - uses: medyagh/setup-minikube@v0.0.13 - with: - driver: docker - container-runtime: docker - cpus: max - memory: max - - name: Run AIStore in Minikube + make -B -C $GITHUB_WORKSPACE/transformers/md5 all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_md5.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/md5 all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_md5.py + - name: Build and Test TAR2TF + if: ${{ env.TAR2TF_ENABLE == 'true' }} run: | - git clone https://github.com/NVIDIA/aistore.git - cd aistore/deploy/dev/k8s - ./minimal.sh - - name: Setup Python 3.11 - uses: actions/setup-python@v2 - with: - python-version: 3.11 - - name: Install FFMPEG + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_tar2tf.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_tar2tf.py + - name: Build and Test COMPRESS + if: ${{ env.COMPRESS_ENABLE == 'true' }} + run: | + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/compress all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_compress.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/compress all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_compress.py + - name: Build and Test FFMPEG + if: ${{ env.FFMPEG_ENABLE == 'true' }} run: | wget -nv http://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz tar -xf ffmpeg-release-amd64-static.tar.xz @@ -149,18 +203,29 @@ jobs: cp ${FFMPEG_DIR}/ffprobe /usr/local/bin/ rm -rf ffmpeg-*-static ffmpeg-release-amd64-static.tar.xz ffmpeg -version - - name: Install Torchvision + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/ffmpeg all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_ffmpeg.py + export GIT_TEST="false" + make -B -C $GITHUB_WORKSPACE/transformers/ffmpeg all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_ffmpeg.py + - name: Build and Test TORCHVISION + if: ${{ env.TORCHVISION_ENABLE == 'true' }} run: | pip install torchvision --index-url https://download.pytorch.org/whl/cpu - - name: Build & Push Transformer Docker Images w/ `latest` Tag - run: | - pushd $GITHUB_WORKSPACE/transformers + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_torchvision_transformer.py export GIT_TEST="false" - make -B all - popd - - name: Test Transformer Docker Images w/ `latest` Tag + make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_torchvision_transformer.py + - name: Build and Test KERAS + if: ${{ env.KERAS_ENABLE == 'true' }} run: | - cd $GITHUB_WORKSPACE/transformers/ - export AIS_ENDPOINT="http://$(minikube ip):8080" + export GIT_TEST="true" + make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_keras_transformer.py export GIT_TEST="false" - make test + make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all + pytest -v $GITHUB_WORKSPACE/transformers/tests/test_keras_transformer.py + diff --git a/transformers/Makefile b/transformers/Makefile index c53e45e..4d56d40 100644 --- a/transformers/Makefile +++ b/transformers/Makefile @@ -1,48 +1,2 @@ -SUBDIRS := compress echo go_echo hello_world md5 tar2tf keras_preprocess ffmpeg torchvision_preprocess - -ifeq ($(ECHO_ENABLE), false) -SUBDIRS := $(filter-out echo,$(SUBDIRS)) -endif - -ifeq ($(GO_ECHO_ENABLE), false) -SUBDIRS := $(filter-out go_echo,$(SUBDIRS)) -endif - -ifeq ($(HELLO_WORLD_ENABLE), false) -SUBDIRS := $(filter-out hello_world,$(SUBDIRS)) -endif - -ifeq ($(MD5_ENABLE), false) -SUBDIRS := $(filter-out md5,$(SUBDIRS)) -endif - -ifeq ($(TAR2TF_ENABLE), false) -SUBDIRS := $(filter-out tar2tf,$(SUBDIRS)) -endif - -ifeq ($(COMPRESS_ENABLE), false) -SUBDIRS := $(filter-out compress,$(SUBDIRS)) -endif - -ifeq ($(KERAS_ENABLE), false) -SUBDIRS := $(filter-out keras_preprocess,$(SUBDIRS)) -endif - -ifeq ($(FFMPEG_ENABLE), false) -SUBDIRS := $(filter-out ffmpeg,$(SUBDIRS)) -endif - -ifeq ($(TORCHVISION_ENABLE), false) -SUBDIRS := $(filter-out torchvision_preprocess,$(SUBDIRS)) -endif - -all: $(SUBDIRS) -$(SUBDIRS): - $(MAKE) -C $@ all - docker system prune -a -f --volumes - common_deps: pip install -r tests/requirements.txt - -test: common_deps - cd tests && for test_file in test_*.py; do pytest -v $$test_file; done diff --git a/transformers/tests/base.py b/transformers/tests/base.py index dff8bf8..fa9633a 100644 --- a/transformers/tests/base.py +++ b/transformers/tests/base.py @@ -15,7 +15,9 @@ def setUp(self): self.endpoint = os.environ.get("AIS_ENDPOINT", "http://192.168.49.2:8080") self.git_test_mode = os.getenv("GIT_TEST", "false") self.client = Client(self.endpoint) - self.test_bck = self.client.bucket("test-bucket" + generate_random_str()).create(exist_ok=True) + self.test_bck = self.client.bucket( + "test-bucket" + generate_random_str() + ).create(exist_ok=True) self.test_etl = self.client.etl("test-etl-" + generate_random_str()) def tearDown(self): diff --git a/transformers/tests/test_compress.py b/transformers/tests/test_compress.py index 2d6c679..bc9971e 100644 --- a/transformers/tests/test_compress.py +++ b/transformers/tests/test_compress.py @@ -7,8 +7,6 @@ import bz2 import gzip import json -import os -import unittest from aistore.sdk.etl_const import ETL_COMM_HPULL, ETL_COMM_HPUSH, ETL_COMM_HREV from aistore.sdk.etl_templates import COMPRESS @@ -158,103 +156,58 @@ def _decompress_test_helper(self, communication_type, compress_options): self.assertEqual(original_image_content, etl_decompressed_img) self.assertEqual(original_text_content, etl_decompressed_txt) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_default_compress_hpull(self): self._compress_test_helper(ETL_COMM_HPULL, {}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_default_compress_hpush(self): self._compress_test_helper(ETL_COMM_HPUSH, {}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_default_compress_hrev(self): self._compress_test_helper(ETL_COMM_HREV, {}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_compress_hpull(self): self._compress_test_helper(ETL_COMM_HPULL, {"compression": "gzip"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_compress_hpush(self): self._compress_test_helper(ETL_COMM_HPUSH, {"compression": "gzip"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_compress_hrev(self): self._compress_test_helper(ETL_COMM_HREV, {"compression": "gzip"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_compress_hpull(self): self._compress_test_helper(ETL_COMM_HPULL, {"compression": "bz2"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_compress_hpush(self): self._compress_test_helper(ETL_COMM_HPUSH, {"compression": "bz2"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_compress_hrev(self): self._compress_test_helper(ETL_COMM_HREV, {"compression": "bz2"}) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_decompress_hpull(self): self._decompress_test_helper( ETL_COMM_HPULL, {"mode": "decompress", "compression": "gzip"} ) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_decompress_hpush(self): self._decompress_test_helper( ETL_COMM_HPUSH, {"mode": "decompress", "compression": "gzip"} ) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_gzip_decompress_hrev(self): self._decompress_test_helper( ETL_COMM_HREV, {"mode": "decompress", "compression": "gzip"} ) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_decompress_hpull(self): self._decompress_test_helper( ETL_COMM_HPULL, {"mode": "decompress", "compression": "bz2"} ) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_decompress_hpush(self): self._decompress_test_helper( ETL_COMM_HPUSH, {"mode": "decompress", "compression": "bz2"} ) - @unittest.skipIf( - os.getenv("COMPRESS_ENABLE", "true") == "false", "COMPRESS is disabled" - ) def test_bz2_decompress_hrev(self): self._decompress_test_helper( ETL_COMM_HREV, {"mode": "decompress", "compression": "bz2"} diff --git a/transformers/tests/test_echo.py b/transformers/tests/test_echo.py index faeb998..b015fd6 100644 --- a/transformers/tests/test_echo.py +++ b/transformers/tests/test_echo.py @@ -4,9 +4,6 @@ # pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring -import os -import unittest - from aistore.sdk.etl_const import ETL_COMM_HPULL, ETL_COMM_HPUSH, ETL_COMM_HREV from aistore.sdk.etl_templates import ECHO @@ -44,19 +41,16 @@ def compare_transformed_data(self, filename: str, source: str): self.assertEqual(transformed_bytes, original_content) - @unittest.skipIf(os.getenv("ECHO_ENABLE", "true") == "false", "ECHO is disabled") def test_echo_hpull(self): self.initialize_template(ETL_COMM_HPULL) self.compare_transformed_data(self.test_image_filename, self.test_image_source) self.compare_transformed_data(self.test_text_filename, self.test_text_source) - @unittest.skipIf(os.getenv("ECHO_ENABLE", "true") == "false", "ECHO is disabled") def test_echo_hpush(self): self.initialize_template(ETL_COMM_HPUSH) self.compare_transformed_data(self.test_image_filename, self.test_image_source) self.compare_transformed_data(self.test_text_filename, self.test_text_source) - @unittest.skipIf(os.getenv("ECHO_ENABLE", "true") == "false", "ECHO is disabled") def test_echo_hrev(self): self.initialize_template(ETL_COMM_HREV) self.compare_transformed_data(self.test_image_filename, self.test_image_source) diff --git a/transformers/tests/test_ffmpeg.py b/transformers/tests/test_ffmpeg.py index 3ec746e..f7d411a 100644 --- a/transformers/tests/test_ffmpeg.py +++ b/transformers/tests/test_ffmpeg.py @@ -4,7 +4,6 @@ import json import os -import unittest import ffmpeg @@ -24,9 +23,6 @@ def decode_data(self, data, **kwargs): ) return out - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_wav_to_flac_hpull(self): self.run_ffmpeg_test( ETL_COMM_HPULL, @@ -35,9 +31,6 @@ def test_ffmpeg_from_wav_to_flac_hpull(self): {"format": "flac", "ar": 48000, "ac": 2}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_mp3_to_wav_hpull(self): self.run_ffmpeg_test( ETL_COMM_HPULL, @@ -46,9 +39,6 @@ def test_ffmpeg_from_mp3_to_wav_hpull(self): {"format": "wav", "ar": 44100, "ac": 2, "af": "loudnorm"}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_format_autodetection_hpull(self): test_filename = "test-audio-wav.wav" test_source = "./resources/test-audio-wav.wav" @@ -63,9 +53,6 @@ def test_ffmpeg_format_autodetection_hpull(self): autodetect_format=file_format, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_wav_to_flac_hpush(self): self.run_ffmpeg_test( ETL_COMM_HPUSH, @@ -74,9 +61,6 @@ def test_ffmpeg_from_wav_to_flac_hpush(self): {"format": "flac", "ar": 48000, "ac": 2}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_mp3_to_wav_hpush(self): self.run_ffmpeg_test( ETL_COMM_HPUSH, @@ -85,9 +69,6 @@ def test_ffmpeg_from_mp3_to_wav_hpush(self): {"format": "wav", "ar": 44100, "ac": 2, "af": "loudnorm"}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_format_autodetection_hpush(self): test_filename = "test-audio-wav.wav" test_source = "./resources/test-audio-wav.wav" @@ -102,9 +83,6 @@ def test_ffmpeg_format_autodetection_hpush(self): autodetect_format=file_format, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_wav_to_flac_hrev(self): self.run_ffmpeg_test( ETL_COMM_HREV, @@ -113,9 +91,6 @@ def test_ffmpeg_from_wav_to_flac_hrev(self): {"format": "flac", "ar": 48000, "ac": 2}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_from_mp3_to_wav_hrev(self): self.run_ffmpeg_test( ETL_COMM_HREV, @@ -124,9 +99,6 @@ def test_ffmpeg_from_mp3_to_wav_hrev(self): {"format": "wav", "ar": 44100, "ac": 2, "af": "loudnorm"}, ) - @unittest.skipIf( - os.getenv("FFMPEG_ENABLE", "true") == "false", "FFMPEG is disabled" - ) def test_ffmpeg_format_autodetection_hrev(self): test_filename = "test-audio-wav.wav" test_source = "./resources/test-audio-wav.wav" diff --git a/transformers/tests/test_go_echo.py b/transformers/tests/test_go_echo.py index e4f24dd..baabc87 100644 --- a/transformers/tests/test_go_echo.py +++ b/transformers/tests/test_go_echo.py @@ -3,9 +3,6 @@ # # pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring -import os -import unittest - from tests.base import TestBase from tests.utils import git_test_mode_format_image_tag_test @@ -23,9 +20,6 @@ def setUp(self): self.test_bck.object(self.test_image_filename).put_file(self.test_image_source) self.test_bck.object(self.test_text_filename).put_file(self.test_text_source) - @unittest.skipIf( - os.getenv("GO_ECHO_ENABLE", "true") == "false", "GO_ECHO is disabled" - ) def test_go_echo(self): template = GO_ECHO.format(communication_type=ETL_COMM_HPULL) diff --git a/transformers/tests/test_keras_transformer.py b/transformers/tests/test_keras_transformer.py index 9bb2c5c..22072fd 100644 --- a/transformers/tests/test_keras_transformer.py +++ b/transformers/tests/test_keras_transformer.py @@ -4,9 +4,7 @@ # pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring import logging -import unittest import io -import os from keras.preprocessing.image import ( ImageDataGenerator, @@ -74,37 +72,17 @@ def run_keras_test(self, communication_type: str, fqn_flag: bool = False): ) self.assertEqual(self.get_transformed_image_local(), transformed_image_etl) - @unittest.skipIf( - os.getenv("KERAS_ENABLE", "true") == "false", - "Keras image was not built, skipping keras test", - ) def test_keras_transformer_hpull(self): self.run_keras_test(communication_type=ETL_COMM_HPULL, fqn_flag=False) - @unittest.skipIf( - os.getenv("KERAS_ENABLE", "true") == "false", - "Keras image was not built, skipping keras test", - ) def test_keras_transformer_hrev(self): self.run_keras_test(communication_type=ETL_COMM_HREV, fqn_flag=False) - @unittest.skipIf( - os.getenv("KERAS_ENABLE", "true") == "false", - "Keras image was not built, skipping keras test", - ) def test_keras_transformer_hpush(self): self.run_keras_test(communication_type=ETL_COMM_HPUSH, fqn_flag=False) - @unittest.skipIf( - os.getenv("KERAS_ENABLE", "true") == "false", - "Keras image was not built, skipping keras test", - ) def test_keras_transformer_hpush_fqn(self): self.run_keras_test(communication_type=ETL_COMM_HPUSH, fqn_flag=True) - @unittest.skipIf( - os.getenv("KERAS_ENABLE", "true") == "false", - "Keras image was not built, skipping keras test", - ) def test_keras_transformer_hpull_fqn(self): self.run_keras_test(communication_type=ETL_COMM_HPULL, fqn_flag=True) diff --git a/transformers/tests/test_md5.py b/transformers/tests/test_md5.py index 47e3987..ef133c3 100644 --- a/transformers/tests/test_md5.py +++ b/transformers/tests/test_md5.py @@ -4,8 +4,6 @@ # pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring import hashlib -import os -import unittest from aistore.sdk.etl_const import ETL_COMM_HPULL, ETL_COMM_HPUSH, ETL_COMM_HREV from aistore.sdk.etl_templates import MD5 @@ -53,14 +51,11 @@ def run_md5_test(self, communication_type): self.test_text_filename, self.test_text_source ) - @unittest.skipIf(os.getenv("MD5_ENABLE", "true") == "false", "MD5 is disabled") def test_md5_hpull(self): self.run_md5_test(ETL_COMM_HPULL) - @unittest.skipIf(os.getenv("MD5_ENABLE", "true") == "false", "MD5 is disabled") def test_md5_hpush(self): self.run_md5_test(ETL_COMM_HPUSH) - @unittest.skipIf(os.getenv("MD5_ENABLE", "true") == "false", "MD5 is disabled") def test_md5_hrev(self): self.run_md5_test(ETL_COMM_HREV) diff --git a/transformers/tests/test_tar2tf.py b/transformers/tests/test_tar2tf.py index 2464786..f296fba 100644 --- a/transformers/tests/test_tar2tf.py +++ b/transformers/tests/test_tar2tf.py @@ -7,7 +7,6 @@ import os import shutil import tarfile -import unittest import numpy as np import tensorflow as tf @@ -37,9 +36,6 @@ def tearDown(self): shutil.rmtree(dir_path) super().tearDown() - @unittest.skipIf( - os.getenv("TAR2TF_ENABLE", "true") == "false", "TAR2TF is disabled" - ) def test_tar2tf_simple(self): template = TAR2TF.format(communication_type=ETL_COMM_HREV, arg="", val="") @@ -81,9 +77,6 @@ def test_tar2tf_simple(self): ) self.assertEqual(cls, original_cls) - @unittest.skipIf( - os.getenv("TAR2TF_ENABLE", "true") == "false", "TAR2TF is disabled" - ) def test_tar2tf_rotation(self): spec = { "conversions": [ diff --git a/transformers/tests/test_torchvision_transformer.py b/transformers/tests/test_torchvision_transformer.py index d55369c..babe062 100644 --- a/transformers/tests/test_torchvision_transformer.py +++ b/transformers/tests/test_torchvision_transformer.py @@ -5,8 +5,6 @@ # pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring import io -import os -import unittest from PIL import Image from torchvision import transforms @@ -64,23 +62,11 @@ def simple_torchvision_test(self, communication_type): # Compare Results of Separate Transforms self.assertEqual(transformed_image_bytes, etl_transformed_image_bytes) - @unittest.skipIf( - os.getenv("TORCHVISION_ENABLE", "true") == "false", - "TORCHVISION_PREPROCESS is diabled", - ) def test_torch_transformer_simple_hpull(self): self.simple_torchvision_test(ETL_COMM_HPULL) - @unittest.skipIf( - os.getenv("TORCHVISION_ENABLE", "true") == "false", - "TORCHVISION_PREPROCESS is diabled", - ) def test_torch_transformer_simple_hpush(self): self.simple_torchvision_test(ETL_COMM_HPUSH) - @unittest.skipIf( - os.getenv("TORCHVISION_ENABLE", "true") == "false", - "TORCHVISION_PREPROCESS is diabled", - ) def test_torch_transformer_simple_hrev(self): self.simple_torchvision_test(ETL_COMM_HREV)