diff --git a/.github/workflows/ansible-test-plugins.yml b/.github/workflows/ansible-test-plugins.yml index 5aeee56e..b961550d 100644 --- a/.github/workflows/ansible-test-plugins.yml +++ b/.github/workflows/ansible-test-plugins.yml @@ -35,12 +35,35 @@ jobs: pull-request-change-detection: true integration: - name: "Integration (Python: ${{ matrix.python }}, Ansible: ${{ matrix.ansible }}, MySQL: ${{ matrix.db_engine_version }}, Connector: ${{ matrix.connector }})" + name: "Integration (Python: ${{ matrix.python }}, Ansible: ${{ matrix.ansible }}, DB: ${{ matrix.db_engine_name }} ${{ matrix.db_engine_version }}, connector: ${{ matrix.connector_name }} ${{ matrix.connector_version }})" runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: - include: + ansible: + - stable-2.12 + - stable-2.13 + - stable-2.14 + - devel + db_engine_name: + - mysql + - mariadb + db_engine_version: + - 5.7.40 + - 8.0.31 + - 10.4.27 + - 10.5.18 + - 10.6.11 + python: + - '3.8' + - '3.9' + - '3.10' + connector_name: + - pymysql + - mysqlclient + connector_version: + - 0.7.11 + - 0.9.3 # Before we can activate test with pymysql 1.0.2 we should debug the # following plugins: # @@ -49,244 +72,128 @@ jobs: # # mysql_replication: # test "Assert that startreplica is not changed" failed + # - 1.0.2 + - 2.0.1 + - 2.0.3 + - 2.1.1 + exclude: + - db_engine_name: mysql + db_engine_version: 10.4.27 - # ================================================================== - # mysql-client 5.7 + Python 3.8 - # ================================================================== - - ansible: stable-2.12 - db_engine_version: mysql:5.7.40 - python: '3.8' - connector: pymysql==0.7.11 - docker_image: ghcr.io/laurent-indermuehle/test-container-my57-py38-pymysql0711:latest - - ansible: stable-2.12 - db_engine_version: mysql:5.7.40 - python: '3.8' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my57-py38-pymysql093:latest - - ansible: stable-2.12 - db_engine_version: mysql:5.7.40 - python: '3.8' - connector: mysqlclient==2.0.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-my57-py38-mysqlclient201:latest + - db_engine_name: mysql + db_engine_version: 10.5.18 + - db_engine_name: mysql + db_engine_version: 10.6.11 - # ================================================================== - # mysql-client 8 + Python 3.8 - # ================================================================== - - ansible: stable-2.12 - db_engine_version: mysql:8.0.31 - python: '3.8' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py38-pymysql093:latest - - ansible: stable-2.12 - db_engine_version: mysql:8.0.31 - python: '3.8' - connector: mysqlclient==2.0.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py38-mysqlclient201:latest + - db_engine_name: mariadb + db_engine_version: 5.7.40 + - db_engine_name: mariadb + db_engine_version: 8.0.31 - # ================================================================== - # mysql-client 8 + Python 3.9 - # ================================================================== - - ansible: stable-2.13 - db_engine_version: mysql:8.0.31 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py39-pymysql093:latest - - ansible: stable-2.13 - db_engine_version: mysql:8.0.31 - python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py39-mysqlclient203:latest + - connector_name: pymysql + connector_version: 2.0.1 - - ansible: stable-2.14 - db_engine_version: mysql:8.0.31 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py39-pymysql093:latest - - ansible: stable-2.14 - db_engine_version: mysql:8.0.31 + - connector_name: pymysql + connector_version: 2.0.3 + + - connector_name: pymysql + connector_version: 2.1.1 + + - connector_name: mysqlclient + connector_version: 0.7.11 + + - connector_name: mysqlclient + connector_version: 0.9.3 + + - connector_name: mysqlclient + connector_version: 1.0.2 + + - db_engine_name: mariadb + connector_version: 0.7.11 + + - db_engine_version: 5.7.40 python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py39-mysqlclient203:latest - - - # ================================================================== - # mysql-client 8 + Python 3.10 - # ================================================================== - # - ansible: stable-2.13 - # db_engine_version: mysql:8.0.31 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-pymysql102:latest - - ansible: stable-2.13 - db_engine_version: mysql:8.0.31 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-mysqlclient211:latest - - # - ansible: stable-2.14 - # db_engine_version: mysql:8.0.31 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-pymysql102:latest - - ansible: stable-2.14 - db_engine_version: mysql:8.0.31 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-mysqlclient211:latest - - # - ansible: devel - # db_engine_version: mysql:8.0.31 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-pymysql102:latest - - ansible: devel - db_engine_version: mysql:8.0.31 + + - db_engine_version: 5.7.40 python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-my80-py310-mysqlclient211:latest - - # ================================================================== - # mariadb-client 10.3 + Python 3.8 - # ================================================================== - - ansible: stable-2.12 - db_engine_version: mariadb:10.4.27 - python: '3.8' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py38-pymysql093:latest - - ansible: stable-2.12 - db_engine_version: mariadb:10.4.27 - python: '3.8' - connector: mysqlclient==2.0.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py38-mysqlclient201:latest - - ansible: stable-2.12 - db_engine_version: mariadb:10.5.18 + + - db_engine_version: 5.7.40 + ansible: stable-2.13 + + - db_engine_version: 5.7.40 + ansible: stable-2.14 + + - db_engine_version: 5.7.40 + ansible: devel + + - db_engine_version: 8.0.31 python: '3.8' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py38-pymysql093:latest - - ansible: stable-2.12 - db_engine_version: mariadb:10.5.18 + + - db_engine_version: 8.0.31 python: '3.8' - connector: mysqlclient==2.0.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py38-mysqlclient201:latest + - db_engine_version: 10.4.27 + python: '3.10' - # ================================================================== - # mariadb-client 10.3 + Python 3.9 - # ================================================================== - - ansible: stable-2.13 - db_engine_version: mariadb:10.4.27 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-pymysql093:latest - - ansible: stable-2.13 - db_engine_version: mariadb:10.4.27 - python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-mysqlclient203:latest - - ansible: stable-2.13 - db_engine_version: mariadb:10.5.18 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-pymysql093:latest - - ansible: stable-2.13 - db_engine_version: mariadb:10.5.18 - python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-mysqlclient203:latest + - db_engine_version: 10.4.27 + ansible: devel - - ansible: stable-2.14 - db_engine_version: mariadb:10.4.27 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-pymysql093:latest - - ansible: stable-2.14 - db_engine_version: mariadb:10.4.27 - python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-mysqlclient203:latest - - ansible: stable-2.14 - db_engine_version: mariadb:10.5.18 - python: '3.9' - connector: pymysql==0.9.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-pymysql093:latest - - ansible: stable-2.14 - db_engine_version: mariadb:10.5.18 + - db_engine_version: 10.6.11 + python: '3.8' + + - db_engine_version: 10.6.11 python: '3.9' - connector: mysqlclient==2.0.3 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb103-py39-mysqlclient203:latest - - - # ================================================================== - # mariadb-client 10.6 + Python 3.10 - # ================================================================== - # - ansible: stable-2.13 - # db_engine_version: mariadb:10.5.18 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: stable-2.13 - db_engine_version: mariadb:10.5.18 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest - # - ansible: stable-2.13 - # db_engine_version: mariadb:10.6.11 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: stable-2.13 - db_engine_version: mariadb:10.6.11 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest - - # - ansible: stable-2.14 - # db_engine_version: mariadb:10.5.18 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: stable-2.14 - db_engine_version: mariadb:10.5.18 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest - # - ansible: stable-2.14 - # db_engine_version: mariadb:10.6.11 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: stable-2.14 - db_engine_version: mariadb:10.6.11 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest - - # - ansible: devel - # db_engine_version: mariadb:10.5.18 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: devel - db_engine_version: mariadb:10.5.18 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest - # - ansible: devel - # db_engine_version: mariadb:10.6.11 - # python: '3.10' - # connector: pymysql==1.0.2 - # docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-pymysql102:latest - - ansible: devel - db_engine_version: mariadb:10.6.11 - python: '3.10' - connector: mysqlclient==2.1.1 - docker_image: ghcr.io/laurent-indermuehle/test-container-mariadb106-py310-mysqlclient211:latest + + - python: '3.8' + connector_version: 2.0.3 + + - python: '3.8' + connector_version: 2.1.1 + + - python: '3.9' + connector_version: 0.7.11 + + - python: '3.9' + connector_version: 2.0.1 + + - python: '3.9' + connector_version: 2.1.1 + + - python: '3.10' + connector_version: 0.7.11 + + - python: '3.10' + connector_version: 0.9.3 + + - python: '3.10' + connector_version: 2.0.1 + + - python: '3.10' + connector_version: 2.0.3 + + - python: '3.8' + ansible: stable-2.13 + + - python: '3.8' + ansible: stable-2.14 + + - python: '3.8' + ansible: devel + + - python: '3.9' + ansible: stable-2.12 + + - python: '3.9' + ansible: devel + + - python: '3.10' + ansible: stable-2.12 services: db_primary: - image: docker.io/library/${{ matrix.db_engine_version }} + image: docker.io/library/${{ matrix.db_engine_name }}:${{ matrix.db_engine_version }} env: MARIADB_ROOT_PASSWORD: msandbox MYSQL_ROOT_PASSWORD: msandbox @@ -302,7 +209,7 @@ jobs: --health-retries 6 db_replica1: - image: docker.io/library/${{ matrix.db_engine_version }} + image: docker.io/library/${{ matrix.db_engine_name }}:${{ matrix.db_engine_version }} env: MARIADB_ROOT_PASSWORD: msandbox MYSQL_ROOT_PASSWORD: msandbox @@ -316,7 +223,7 @@ jobs: --health-retries 6 db_replica2: - image: docker.io/library/${{ matrix.db_engine_version }} + image: docker.io/library/${{ matrix.db_engine_name }}:${{ matrix.db_engine_version }} env: MARIADB_ROOT_PASSWORD: msandbox MYSQL_ROOT_PASSWORD: msandbox @@ -343,8 +250,46 @@ jobs: docker restart -t 30 ${{ job.services.db_replica2.id }} - name: Wait for the primary to be healthy - run: | - while ! /usr/bin/docker inspect --format="{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}" ${{ job.services.db_primary.id }} | grep healthy && [[ "$SECONDS" -lt 120 ]]; do sleep 1; done + run: > + while ! /usr/bin/docker inspect + --format="{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}" + ${{ job.services.db_primary.id }} + | grep healthy && [[ "$SECONDS" -lt 120 ]]; do sleep 1; done + + - name: Compute docker_image - Set python_version_flat + run: > + echo "python_version_flat=$(echo ${{ matrix.python }} + | tr -d '.')" >> $GITHUB_ENV + + - name: Compute docker_image - Set connector_version_flat + run: > + echo "connector_version_flat=$(echo ${{ matrix.connector_version }} + |tr -d .)" >> $GITHUB_ENV + + - name: Compute docker_image - Set db_engine_version_flat + run: > + echo "db_engine_version_flat=$(echo ${{ matrix.db_engine_version }} + | awk -F '.' '{print $1 $2}')" >> $GITHUB_ENV + + - name: Compute docker_image - Set db_client + run: > + if [[ ${{ env.db_engine_version_flat }} == 57 ]]; then + echo "db_client=my57" >> $GITHUB_ENV; + else + echo "db_client=$(echo ${{ matrix.db_engine_name }})" >> $GITHUB_ENV; + fi + + - name: Set docker_image + run: > + docker_image_multiline=(" + ghcr.io/ansible-collections/community.mysql\ + /test-container-${{ env.db_client }}\ + -py${{ env.python_version_flat }}\ + -${{ matrix.connector_name }}${{ env.connector_version_flat }}\ + :latest") + + echo "docker_image=$(printf '%s' $docker_image_multiline)" + >> $GITHUB_ENV - name: >- Perform integration testing against @@ -354,15 +299,31 @@ jobs: with: ansible-core-version: ${{ matrix.ansible }} pre-test-cmd: >- - echo Setting db_engine_version to "${{ matrix.db_engine_version }}"...; - echo -n "${{ matrix.db_engine_version }}" > tests/integration/db_engine_version; - echo Setting Connector version to "${{ matrix.connector }}"...; - echo -n "${{ matrix.connector }}" > tests/integration/connector; - echo Setting Python version to "${{ matrix.python }}"...; - echo -n "${{ matrix.python }}" > tests/integration/python; - echo Setting Ansible version to "${{ matrix.ansible }}"...; - echo -n "${{ matrix.ansible }}" > tests/integration/ansible - docker-image: ${{ matrix.docker_image }} + echo Setting db_engine_name to "${{ matrix.db_engine_name }}"...; + echo -n "${{ matrix.db_engine_name }}" + > tests/integration/db_engine_name; + + echo Setting db_engine_version to \ + "${{ matrix.db_engine_version }}"...; + echo -n "${{ matrix.db_engine_version }}" + > tests/integration/db_engine_version; + + echo Setting Connector name to "${{ matrix.connector_name }}"...; + echo -n "${{ matrix.connector_name }}" + > tests/integration/connector_name; + + echo Setting Connector name to "${{ matrix.connector_version }}"...; + echo -n "${{ matrix.connector_version }}" + > tests/integration/connector_version; + + echo Setting Python version to "${{ matrix.python }}"...; + echo -n "${{ matrix.python }}" + > tests/integration/python; + + echo Setting Ansible version to "${{ matrix.ansible }}"...; + echo -n "${{ matrix.ansible }}" + > tests/integration/ansible + docker-image: ${{ env.docker_image }} target-python-version: ${{ matrix.python }} testing-type: integration diff --git a/.github/workflows/docker-image-mariadb-py310-mysqlclient211.yml b/.github/workflows/docker-image-mariadb-py310-mysqlclient211.yml new file mode 100644 index 00000000..a893d3b4 --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py310-mysqlclient211.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py310-mysqlclient211 + +on: + push: + paths: + - 'test-containers/mariadb-py310-mysqlclient211/**' + - '.github/workflows/docker-image-mariadb-py310-mysqlclient211.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py310-mysqlclient211 + context: test-containers/mariadb-py310-mysqlclient211 diff --git a/.github/workflows/docker-image-mariadb-py310-pymysql102.yml b/.github/workflows/docker-image-mariadb-py310-pymysql102.yml new file mode 100644 index 00000000..dbd8a9d1 --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py310-pymysql102.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py310-pymysql102 + +on: + push: + paths: + - 'test-containers/mariadb-py310-pymysql102/**' + - '.github/workflows/docker-image-mariadb-py310-pymysql102.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py310-pymysql102 + context: test-containers/mariadb-py310-pymysql102 diff --git a/.github/workflows/docker-image-mariadb-py38-mysqlclient201.yml b/.github/workflows/docker-image-mariadb-py38-mysqlclient201.yml new file mode 100644 index 00000000..59fcf00e --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py38-mysqlclient201.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py38-mysqlclient201 + +on: + push: + paths: + - 'test-containers/mariadb-py38-mysqlclient201/**' + - '.github/workflows/docker-image-mariadb-py38-mysqlclient201.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py38-mysqlclient201 + context: test-containers/mariadb-py38-mysqlclient201 diff --git a/.github/workflows/docker-image-mariadb-py38-pymysql093.yml b/.github/workflows/docker-image-mariadb-py38-pymysql093.yml new file mode 100644 index 00000000..2602ebe5 --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py38-pymysql093.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py38-pymysql093 + +on: + push: + paths: + - 'test-containers/mariadb-py38-pymysql093/**' + - '.github/workflows/docker-image-mariadb-py38-pymysql093.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py38-pymysql093 + context: test-containers/mariadb-py38-pymysql093 diff --git a/.github/workflows/docker-image-mariadb-py39-mysqlclient203.yml b/.github/workflows/docker-image-mariadb-py39-mysqlclient203.yml new file mode 100644 index 00000000..3f260e58 --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py39-mysqlclient203.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py39-mysqlclient203 + +on: + push: + paths: + - 'test-containers/mariadb-py39-mysqlclient203/**' + - '.github/workflows/docker-image-mariadb-py39-mysqlclient203.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py39-mysqlclient203 + context: test-containers/mariadb-py39-mysqlclient203 diff --git a/.github/workflows/docker-image-mariadb-py39-pymysql093.yml b/.github/workflows/docker-image-mariadb-py39-pymysql093.yml new file mode 100644 index 00000000..77bb664f --- /dev/null +++ b/.github/workflows/docker-image-mariadb-py39-pymysql093.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mariadb-py39-pymysql093 + +on: + push: + paths: + - 'test-containers/mariadb-py39-pymysql093/**' + - '.github/workflows/docker-image-mariadb-py39-pymysql093.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mariadb-py39-pymysql093 + context: test-containers/mariadb-py39-pymysql093 diff --git a/.github/workflows/docker-image-mariadb103-py38-mysqlclient201.yml b/.github/workflows/docker-image-mariadb103-py38-mysqlclient201.yml deleted file mode 100644 index 3d902700..00000000 --- a/.github/workflows/docker-image-mariadb103-py38-mysqlclient201.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb103-py38-mysqlclient201 - -on: - push: - paths: - - 'test-containers/mariadb103-py38-mysqlclient201/**' - - '.github/workflows/docker-image-mariadb103-py38-mysqlclient201.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb103-py38-mysqlclient201 - context: test-containers/mariadb103-py38-mysqlclient201 diff --git a/.github/workflows/docker-image-mariadb103-py38-pymysql093.yml b/.github/workflows/docker-image-mariadb103-py38-pymysql093.yml deleted file mode 100644 index 1ca4600d..00000000 --- a/.github/workflows/docker-image-mariadb103-py38-pymysql093.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb103-py38-pymysql093 - -on: - push: - paths: - - 'test-containers/mariadb103-py38-pymysql093/**' - - '.github/workflows/docker-image-mariadb103-py38-pymysql093.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb103-py38-pymysql093 - context: test-containers/mariadb103-py38-pymysql093 diff --git a/.github/workflows/docker-image-mariadb103-py39-mysqlclient203.yml b/.github/workflows/docker-image-mariadb103-py39-mysqlclient203.yml deleted file mode 100644 index 37e91ee1..00000000 --- a/.github/workflows/docker-image-mariadb103-py39-mysqlclient203.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb103-py39-mysqlclient203 - -on: - push: - paths: - - 'test-containers/mariadb103-py39-mysqlclient203/**' - - '.github/workflows/docker-image-mariadb103-py39-mysqlclient203.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb103-py39-mysqlclient203 - context: test-containers/mariadb103-py39-mysqlclient203 diff --git a/.github/workflows/docker-image-mariadb103-py39-pymysql093.yml b/.github/workflows/docker-image-mariadb103-py39-pymysql093.yml deleted file mode 100644 index 30acfc12..00000000 --- a/.github/workflows/docker-image-mariadb103-py39-pymysql093.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb103-py39-pymysql093 - -on: - push: - paths: - - 'test-containers/mariadb103-py39-pymysql093/**' - - '.github/workflows/docker-image-mariadb103-py39-pymysql093.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb103-py39-pymysql093 - context: test-containers/mariadb103-py39-pymysql093 diff --git a/.github/workflows/docker-image-mariadb106-py310-mysqlclient211.yml b/.github/workflows/docker-image-mariadb106-py310-mysqlclient211.yml deleted file mode 100644 index 0fa7403f..00000000 --- a/.github/workflows/docker-image-mariadb106-py310-mysqlclient211.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb106-py310-mysqlclient211 - -on: - push: - paths: - - 'test-containers/mariadb106-py310-mysqlclient211/**' - - '.github/workflows/docker-image-mariadb106-py310-mysqlclient211.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb106-py310-mysqlclient211 - context: test-containers/mariadb106-py310-mysqlclient211 diff --git a/.github/workflows/docker-image-mariadb106-py310-pymysql102.yml b/.github/workflows/docker-image-mariadb106-py310-pymysql102.yml deleted file mode 100644 index adfe9e37..00000000 --- a/.github/workflows/docker-image-mariadb106-py310-pymysql102.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI mariadb106-py310-pymysql102 - -on: - push: - paths: - - 'test-containers/mariadb106-py310-pymysql102/**' - - '.github/workflows/docker-image-mariadb106-py310-pymysql102.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-mariadb106-py310-pymysql102 - context: test-containers/mariadb106-py310-pymysql102 diff --git a/.github/workflows/docker-image-my80-py310-mysqlclient211.yml b/.github/workflows/docker-image-my80-py310-mysqlclient211.yml deleted file mode 100644 index 824f77c4..00000000 --- a/.github/workflows/docker-image-my80-py310-mysqlclient211.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py310-mysqlclient211 - -on: - push: - paths: - - 'test-containers/my80-py310-mysqlclient211/**' - - '.github/workflows/docker-image-my80-py310-mysqlclient211.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py310-mysqlclient211 - context: test-containers/my80-py310-mysqlclient211 diff --git a/.github/workflows/docker-image-my80-py310-pymysql102.yml b/.github/workflows/docker-image-my80-py310-pymysql102.yml deleted file mode 100644 index 0c54e126..00000000 --- a/.github/workflows/docker-image-my80-py310-pymysql102.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py310-pymysql102 - -on: - push: - paths: - - 'test-containers/my80-py310-pymysql102/**' - - '.github/workflows/docker-image-my80-py310-pymysql102.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py310-pymysql102 - context: test-containers/my80-py310-pymysql102 diff --git a/.github/workflows/docker-image-my80-py38-mysqlclient201.yml b/.github/workflows/docker-image-my80-py38-mysqlclient201.yml deleted file mode 100644 index 0ac76b2c..00000000 --- a/.github/workflows/docker-image-my80-py38-mysqlclient201.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py38-mysqlclient201 - -on: - push: - paths: - - 'test-containers/my80-py38-mysqlclient201/**' - - '.github/workflows/docker-image-my80-py38-mysqlclient201.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py38-mysqlclient201 - context: test-containers/my80-py38-mysqlclient201 diff --git a/.github/workflows/docker-image-my80-py38-pymysql093.yml b/.github/workflows/docker-image-my80-py38-pymysql093.yml deleted file mode 100644 index 1677be69..00000000 --- a/.github/workflows/docker-image-my80-py38-pymysql093.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py38-pymysql093 - -on: - push: - paths: - - 'test-containers/my80-py38-pymysql093/**' - - '.github/workflows/docker-image-my80-py38-pymysql093.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py38-pymysql093 - context: test-containers/my80-py38-pymysql093 diff --git a/.github/workflows/docker-image-my80-py39-mysqlclient203.yml b/.github/workflows/docker-image-my80-py39-mysqlclient203.yml deleted file mode 100644 index e6b41db1..00000000 --- a/.github/workflows/docker-image-my80-py39-mysqlclient203.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py39-mysqlclient203 - -on: - push: - paths: - - 'test-containers/my80-py39-mysqlclient203/**' - - '.github/workflows/docker-image-my80-py39-mysqlclient203.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py39-mysqlclient203 - context: test-containers/my80-py39-mysqlclient203 diff --git a/.github/workflows/docker-image-my80-py39-pymysql093.yml b/.github/workflows/docker-image-my80-py39-pymysql093.yml deleted file mode 100644 index 72ffd602..00000000 --- a/.github/workflows/docker-image-my80-py39-pymysql093.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Docker Image CI my80-py39-pymysql093 - -on: - push: - paths: - - 'test-containers/my80-py39-pymysql093/*' - - '.github/workflows/docker-image-my80-py39-pymysql093.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - - call-workflow-passing-data: - uses: ./.github/workflows/build-docker-image.yml - secrets: inherit - with: - registry: ghcr.io - image_name: test-container-my80-py39-pymysql093 - context: test-containers/my80-py39-pymysql093 diff --git a/.github/workflows/docker-image-mysql-py310-mysqlclient211.yml b/.github/workflows/docker-image-mysql-py310-mysqlclient211.yml new file mode 100644 index 00000000..70eea1ca --- /dev/null +++ b/.github/workflows/docker-image-mysql-py310-mysqlclient211.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py310-mysqlclient211 + +on: + push: + paths: + - 'test-containers/mysql-py310-mysqlclient211/**' + - '.github/workflows/docker-image-mysql-py310-mysqlclient211.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py310-mysqlclient211 + context: test-containers/mysql-py310-mysqlclient211 diff --git a/.github/workflows/docker-image-mysql-py310-pymysql102.yml b/.github/workflows/docker-image-mysql-py310-pymysql102.yml new file mode 100644 index 00000000..bcf88fa2 --- /dev/null +++ b/.github/workflows/docker-image-mysql-py310-pymysql102.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py310-pymysql102 + +on: + push: + paths: + - 'test-containers/mysql-py310-pymysql102/**' + - '.github/workflows/docker-image-mysql-py310-pymysql102.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py310-pymysql102 + context: test-containers/mysql-py310-pymysql102 diff --git a/.github/workflows/docker-image-mysql-py38-mysqlclient201.yml b/.github/workflows/docker-image-mysql-py38-mysqlclient201.yml new file mode 100644 index 00000000..7fcfb606 --- /dev/null +++ b/.github/workflows/docker-image-mysql-py38-mysqlclient201.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py38-mysqlclient201 + +on: + push: + paths: + - 'test-containers/mysql-py38-mysqlclient201/**' + - '.github/workflows/docker-image-mysql-py38-mysqlclient201.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py38-mysqlclient201 + context: test-containers/mysql-py38-mysqlclient201 diff --git a/.github/workflows/docker-image-mysql-py38-pymysql093.yml b/.github/workflows/docker-image-mysql-py38-pymysql093.yml new file mode 100644 index 00000000..5a43ab84 --- /dev/null +++ b/.github/workflows/docker-image-mysql-py38-pymysql093.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py38-pymysql093 + +on: + push: + paths: + - 'test-containers/mysql-py38-pymysql093/**' + - '.github/workflows/docker-image-mysql-py38-pymysql093.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py38-pymysql093 + context: test-containers/mysql-py38-pymysql093 diff --git a/.github/workflows/docker-image-mysql-py39-mysqlclient203.yml b/.github/workflows/docker-image-mysql-py39-mysqlclient203.yml new file mode 100644 index 00000000..4486a6ee --- /dev/null +++ b/.github/workflows/docker-image-mysql-py39-mysqlclient203.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py39-mysqlclient203 + +on: + push: + paths: + - 'test-containers/mysql-py39-mysqlclient203/**' + - '.github/workflows/docker-image-mysql-py39-mysqlclient203.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py39-mysqlclient203 + context: test-containers/mysql-py39-mysqlclient203 diff --git a/.github/workflows/docker-image-mysql-py39-pymysql093.yml b/.github/workflows/docker-image-mysql-py39-pymysql093.yml new file mode 100644 index 00000000..d06dc740 --- /dev/null +++ b/.github/workflows/docker-image-mysql-py39-pymysql093.yml @@ -0,0 +1,19 @@ +--- +name: Docker Image CI mysql-py39-pymysql093 + +on: + push: + paths: + - 'test-containers/mysql-py39-pymysql093/*' + - '.github/workflows/docker-image-mysql-py39-pymysql093.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + + call-workflow-passing-data: + uses: ./.github/workflows/build-docker-image.yml + secrets: inherit + with: + registry: ghcr.io + image_name: test-container-mysql-py39-pymysql093 + context: test-containers/mysql-py39-pymysql093 diff --git a/Makefile b/Makefile index a94ffd8e..dc6d6d09 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,32 @@ ifdef continue_on_errors _continue_on_errors = --retry-on-error --continue-on-error endif + +db_ver_tuple := $(subst ., , $(db_engine_version)) +db_engine_version_flat := $(word 1, $(db_ver_tuple))$(word 2, $(db_ver_tuple)) + +con_ver_tuple := $(subst ., , $(connector_version)) +connector_version_flat := $(word 1, $(con_ver_tuple))$(word 2, $(con_ver_tuple))$(word 3, $(con_ver_tuple)) + +py_ver_tuple := $(subst ., , $(python)) +python_version_flat := $(word 1, $(py_ver_tuple))$(word 2, $(py_ver_tuple)) + +ifeq ($(db_engine_version_flat), 57) + db_client := my57 +else + db_client := $(db_engine_name) +endif + + .PHONY: test-integration test-integration: - echo -n $(db_engine_version) > tests/integration/db_engine_version - echo -n $(connector) > tests/integration/connector - echo -n $(python) > tests/integration/python - echo -n $(ansible) > tests/integration/ansible + @echo -n $(db_engine_name) > tests/integration/db_engine_name + @echo -n $(db_engine_version) > tests/integration/db_engine_version + @echo -n $(connector_name) > tests/integration/connector_name + @echo -n $(connector_version) > tests/integration/connector_version + @echo -n $(python) > tests/integration/python + @echo -n $(ansible) > tests/integration/ansible + # Create podman network for systems missing it. Error can be ignored podman network create podman || true podman run \ @@ -28,7 +48,7 @@ test-integration: --network podman \ --publish 3307:3306 \ --health-cmd 'mysqladmin ping -P 3306 -pmsandbox | grep alive || exit 1' \ - docker.io/library/$(db_engine_version) \ + docker.io/library/$(db_engine_name):$(db_engine_version) \ mysqld podman run \ --detach \ @@ -39,7 +59,7 @@ test-integration: --network podman \ --publish 3308:3306 \ --health-cmd 'mysqladmin ping -P 3306 -pmsandbox | grep alive || exit 1' \ - docker.io/library/$(db_engine_version) \ + docker.io/library/$(db_engine_name):$(db_engine_version) \ mysqld podman run \ --detach \ @@ -50,7 +70,7 @@ test-integration: --network podman \ --publish 3309:3306 \ --health-cmd 'mysqladmin ping -P 3306 -pmsandbox | grep alive || exit 1' \ - docker.io/library/$(db_engine_version) \ + docker.io/library/$(db_engine_name):$(db_engine_version) \ mysqld # Setup replication and restart containers podman exec primary bash -c 'echo -e [mysqld]\\nserver-id=1\\nlog-bin=/var/lib/mysql/primary-bin > /etc/mysql/conf.d/replication.cnf' @@ -69,9 +89,13 @@ test-integration: source .venv/$(ansible)/bin/activate python$(local_python_version) -m ensurepip python$(local_python_version) -m pip install --disable-pip-version-check https://github.com/ansible/ansible/archive/$(ansible).tar.gz - -set -x; ansible-test integration $(target) -v --color --coverage --diff --docker $(docker_image) --docker-network podman $(_continue_on_errors) $(_keep_containers_alive) --python $(python); set +x + -set -x; ansible-test integration $(target) -v --color --coverage --diff \ + --docker ghcr.io/ansible-collections/community.mysql/test-container-$(db_client)-py$(python_version_flat)-$(connector_name)$(connector_version_flat):latest \ + --docker-network podman $(_continue_on_errors) $(_keep_containers_alive) --python $(python); set +x + rm tests/integration/db_engine_name rm tests/integration/db_engine_version - rm tests/integration/connector + rm tests/integration/connector_name + rm tests/integration/connector_version rm tests/integration/python rm tests/integration/ansible ifndef keep_containers_alive diff --git a/TESTING.md b/TESTING.md index 9aad0f56..a24193a4 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,7 +2,7 @@ This collection uses GitHub Actions to run ansible-test to validate its content. Three type of tests are used: Sanity, Integration and Units. -The tests covers the code for plugins and roles (no role available yet, but tests are ready) and can be found here: +The tests covers plugins and roles (no role available yet, but tests are ready) and can be found here: - Plugins: *.github/workflows/ansible-test-plugins.yml* - Roles: *.github/workflows/ansible-test-roles.yml* (unused yet) @@ -16,48 +16,125 @@ You can use GitHub to run ansible-test either on the community repo or your fork For now, the makefile only supports Podman. + ### Requirements - python >= 3.8 and <= 3.10 - make +- podman - Minimum 15GB of free space on the device storing containers images and volumes. You can use this command to check: `podman system info --format='{{.Store.GraphRoot}}'|xargs findmnt --noheadings --nofsroot --output SOURCE --target|xargs df -h --output=size,used,avail,pcent,target` - Minimum 2GB of RAM -### Makefile options +### Custom ansible-test containers -The Makefile accept the following options: - -- **local_python_version**: This option can be omitted if your system has a version supported by Ansible. You can check with `python -V`. -- **ansible**: Mandatory version of ansible to install in a venv to run ansible-test. -- **docker_image**: - The container image to use to run our tests. Those images Dockerfile are in https://github.com/community.mysql/test-containers and then pushed to quay.io: E.G.: - `quay.io/mws/community-mysql-test-containers-my57-py38-mysqlclient201-pymysql0711:latest`. Look in the link above for a complete list of available containers. You can also look into `.github/workflows/ansible-test-plugins.yml` - Unfortunatly you must provide the right container_image yourself. And you still need to provides db_engine_version, python, etc... because ansible-test won't do black magic to try to detect what we expect. Explicit is better than implicit anyway. - To minimise the amount of images, pymysql 0.7.11 and mysqlclient are shipped together. -- **db_engine_version**: The name of the container to use for the service containers that will host a primary database and two replicas. Either MYSQL or MariaDB. Use ':' as a separator. Do not use short version, like mysql:8 for instance. Our tests expect a full version to filter tests precisely. For instance: `when: db_version is version ('8.0.22', '>')`. -- **connector**: The name of the python package of the connector along with its version number. Use '==' as a separator. -- **python**: The python version to use in the controller. -- **target** : If omitted, all test targets will run. But you can limit the tests to a single target to speed up your tests. -- **keep_containers_alive**: This option keeps all tree databases containers and the ansible-test container alive at the end of tests or in case of failure. This is useful to enter one of the containers with `podman exec -it bash` for debugging. Rerunning the -test will recreate those containers. -- **continue_on_errors**: Tells ansible-test to retry on errors and also continue on errors. This is the way the GitHub Action's workflow runs the tests. If you develop a new target, this option can be used to validate that your tests cleanup everything so a new run can restart without errors like "Failed to create database x because it already exists". +Our integrations tests use custom containers for ansible-test. Those images have their definition file stored in the directory [test-containers](test-containers/). We build and publish the images on ghcr.io under the ansible-collection namespace: E.G.: +`ghcr.io/ansible-collections/community.mysql/test-container-mariadb106-py310-mysqlclient211:latest`. -Examples: +Availables images are listed [here](https://github.com/orgs/ansible-collections/packages). + + +### Makefile options + +The Makefile accept the following options + +- `local_python_version` + - Mandatory: false + - Choices: + - "3.8" + - "3.9" + - "3.10" + - Description: If `Python -V` shows an unsupported version, use this option and choose one of the version available on your system. Use `ls /usr/bin/python3*|grep -v config` to list them. + +- `ansible` + - Mandatory: true + - Choices: + - "stable-2.12" + - "stable-2.13" + - "stable-2.14" + - "devel" + - Description: Version of ansible to install in a venv to run ansible-test + +- `db_engine_name` + - Mandatory: true + - Choices: + - "mysql" + - "mariadb" + - Description: The name of the database engine to use for the service containers that will host a primary database and two replicas. + +- `db_engine_version` + - Mandatory: true + - Choices: + - "5.7.40" <- mysql + - "8.0.31" <- mysql + - "10.4.24" <- mariadb + - "10.5.18" <- mariadb + - "10.6.11" <- mariadb + - Description: The tag of the container to use for the service containers that will host a primary database and two replicas. Do not use short version, like `mysql:8` (don't do that) because our tests expect a full version to filter tests precisely. For instance: `when: db_version is version ('8.0.22', '>')`. You can use any tag available on [hub.docker.com/_/mysql](https://hub.docker.com/_/mysql) and [hub.docker.com/_/mariadb](https://hub.docker.com/_/mariadb) but GitHub Action will only use the versions listed above. + +- `connector_name` + - Mandatory: true + - Choices: + - "pymysql + - "mysqlclient" + - Description: The python package of the connector to use. This value is used to filter tests meant for other connectors. + +- `connector_version` + - Mandatory: true + - Choices: + - "0.7.11" <- Only for MySQL 5.7 + - "0.9.3" + - "1.0.2" <- Not working, need fix + - "2.0.1" + - "2.0.3" + - "2.1.1" + - Description: The version of the python package of the connector to use. This value is used to filter tests meant for other connectors. + +- `python` + - Mandatory: true + - Choices: + - "3.8" + - "3.9" + - "3.10" + - Description: The python version to use in the controller (ansible-test container). + +- `target` + - Mandatory: false + - Choices: + - "test_mysql_db" + - "test_mysql_info" + - "test_mysql_query" + - "test_mysql_replication" + - "test_mysql_role" + - "test_mysql_user" + - "test_mysql_variables" + - Description: If omitted, all test targets will run. But you can limit the tests to a single target to speed up your tests. + +- `keep_containers_alive` + - Mandatory: false + - Description: This option keeps all tree databases containers and the ansible-test container alive at the end of tests or in case of failure. This is useful to enter one of the containers with `podman exec -it bash` for debugging. Rerunning the +test will recreate those containers so no need to kill it. Add any value to activate this option: `keep_containers_alive=1` + +- `continue_on_errors` + - Mandatory: false + - Description: Tells ansible-test to retry on errors and also continue on errors. This is the way the GitHub Action's workflow runs the tests. This can be use to catch all errors in a single run, but you'll need to scroll up to find them. Add any value to activate this option: `continue_on_errors=1` + + +#### Makefile usage examples: ```sh # Run all targets -make ansible="stable-2.12" db_engine_version="mysql:5.7.40" python="3.8" connector="pymysql==0.7.11" docker_image="ghcr.io/community.mysql/test-container-my57-py38-pymysql0711:latest" +make ansible="stable-2.12" db_engine_name="mysql" db_engine_version="5.7.40" python="3.8" connector_name="pymysql" connector_version="0.7.11" # A single target -make ansible="stable-2.14" db_engine_version="mysql:5.7.40" python="3.8" connector="pymysql==0.7.11" docker_image="ghcr.io/community.mysql/test-container-my57-py38-pymysql0711:latest" target="test_mysql_db" +make ansible="stable-2.14" db_engine_name="mysql" db_engine_version="5.7.40" python="3.8" connector_name="pymysql" connector_version="0.7.11" # Keep databases and ansible tests containers alives # A single target and continue on errors -make ansible="stable-2.14" db_engine_version="mysql:8.0.31" python="3.9" connector="mysqlclient==2.0.3" docker_image="ghcr.io/community.mysql/test-container-my80-py39-mysqlclient203:latest" target="test_mysql_db" keep_containers_alive=1 continue_on_errors=1 +make ansible="stable-2.14" db_engine_name="mysql" db_engine_version="8.0.31" python="3.9" connector_name="mysqlclient" connector_version="2.0.3" # If your system has an usupported version of Python: -make local_python_version="3.8" ansible="stable-2.14" db_engine_version="mariadb:10.6.11" python="3.9" connector="pymysql==0.9.3" docker_image="ghcr.io/community.mysql/test-container-mariadb103-py39-pymysql093:latest" +make local_python_version="3.8" ansible="stable-2.14" db_engine_name="mariadb" db_engine_version="10.6.11" python="3.9" connector_name="pymysql" connector_version="0.9.3" ``` @@ -74,14 +151,16 @@ python run_all_tests.py ### Add a new Python, Connector or Database version +You can look into `[.github/workflows/ansible-test-plugins.yml](https://github.com/ansible-collections/community.mysql/tree/main/.github/workflows)` to see how those containers are built using [build-docker-image.yml](https://github.com/ansible-collections/community.mysql/blob/main/.github/workflows/build-docker-image.yml) and all [docker-image-xxx.yml](https://github.com/ansible-collections/community.mysql/blob/main/.github/workflows/docker-image-mariadb103-py38-mysqlclient201.yml) files. + 1. Add a workflow in [.github/workflows/](.github/workflows) 1. Add a new folder in [test-containers](test-containers) containing a new Dockerfile. Your container must contains 3 things: - - The python interpreter - - The python package to connect to the database (pymysql, mysqlclient, ...) - - A mysql client to query the database before to prepare tests before our tests starts. This client must provide both `mysql` and `mysqldump` commands. -1. Add your version in *.github/workflows/ansible-test-plugins.yml* - -After pushing the commit to the remote, the container will be build and published on ghcr.io. Have a look in the "Action" tab to see if it worked. In case of error `failed to copy: io: read/write on closed pipe` re-run the workflow, this append unfortunately a lot. + - Python + - A connector: The python package to connect to the database (pymysql, mysqlclient, ...) + - A mysql client to prepare databases before our tests starts. This client must provide both `mysql` and `mysqldump` commands. +1. Add your version in the matrix of *.github/workflows/ansible-test-plugins.yml*. You can use [run_all_tests.py](run_all_tests.py) to help you see what the matrix will be. Simply comment out the line `os.system(make_cmd)` before runing the script. You can also add `print(len(matrix))` to display how many tests there will be on GitHub Action. +1. Ask the lead maintainer to mark your new image(s) as `public` under [https://github.com/orgs/ansible-collections/packages](https://github.com/orgs/ansible-collections/packages) -To see the docker image produced, go to the main GitHub page of your fork or community.mysql (depending were you pushed) and look for the link "Packages" on the right hand side of the page. This page indicate a "Published x days ago" that is updated infrequently. To see the last time the container has been updated you must click on its title and look in the right hands side bellow the title "Last published". +After pushing your commit to the remote, the container will be built and published on ghcr.io. Have a look in the "Action" tab to see if it worked. In case of error `failed to copy: io: read/write on closed pipe` re-run the workflow, this append unfortunately a lot. +To see the docker image produced, go to the package page in the ansible-collection namespace [https://github.com/orgs/ansible-collections/packages](https://github.com/orgs/ansible-collections/packages). This page indicate a "Published x days ago" that is updated infrequently. To see the last time the container has been updated you must click on its title and look in the right hands side bellow the title "Last published". diff --git a/run_all_tests.py b/run_all_tests.py index b7779a5c..94cf799b 100755 --- a/run_all_tests.py +++ b/run_all_tests.py @@ -28,54 +28,83 @@ def extract_matrix(workflow_yaml): return matrix -# def is_exclude(exclude_list, test_suite): -# test_is_excluded = False -# for excl in exclude_list: -# match = 0 +def is_exclude(exclude_list, test_suite): + test_is_excluded = False + for excl in exclude_list: + match = 0 -# if 'ansible' in excl: -# if excl.get('ansible') == test_suite[0]: -# match += 1 + if 'ansible' in excl: + if excl.get('ansible') == test_suite.get('ansible'): + match += 1 -# if 'db_engine_version' in excl: -# if excl.get('db_engine_version') == test_suite[1]: -# match += 1 + if 'db_engine_name' in excl: + if excl.get('db_engine_name') == test_suite.get('db_engine_name'): + match += 1 -# if 'python' in excl: -# if excl.get('python') == test_suite[2]: -# match += 1 + if 'db_engine_version' in excl: + if excl.get('db_engine_version') == test_suite.get('db_engine_version'): + match += 1 -# if 'connector' in excl: -# if excl.get('connector') == test_suite[3]: -# match += 1 + if 'python' in excl: + if excl.get('python') == test_suite.get('python'): + match += 1 -# if match > 1: -# test_is_excluded = True + if 'connector_name' in excl: + if excl.get('connector_name') == test_suite.get('connector_name'): + match += 1 -# return test_is_excluded + if 'connector_version' in excl: + if excl.get('connector_version') == test_suite.get('connector_version'): + match += 1 + + if match > 1: + test_is_excluded = True + return test_is_excluded + + return test_is_excluded def main(): workflow_yaml = read_github_workflow_file() tests_matrix_yaml = extract_matrix(workflow_yaml) - # matrix = [] - # exclude_list = tests_matrix_yaml.get('exclude') - # for ansible in tests_matrix_yaml.get('ansible'): - # for db_engine in tests_matrix_yaml.get('db_engine_version'): - # for python in tests_matrix_yaml.get('python'): - # for connector in tests_matrix_yaml.get('connector'): - # if not is_exclude(exclude_list, (ansible, db_engine, python, connector)): - # matrix.append((ansible, db_engine, python, connector)) - - for tests in tests_matrix_yaml.get('include'): + matrix = [] + exclude_list = tests_matrix_yaml.get('exclude') + for ansible in tests_matrix_yaml.get('ansible'): + for db_engine_name in tests_matrix_yaml.get('db_engine_name'): + for db_engine_version in tests_matrix_yaml.get('db_engine_version'): + for python in tests_matrix_yaml.get('python'): + for connector_name in tests_matrix_yaml.get('connector_name'): + for connector_version in tests_matrix_yaml.get('connector_version'): + test_suite = { + 'ansible': ansible, + 'db_engine_name': db_engine_name, + 'db_engine_version': db_engine_version, + 'python': python, + 'connector_name': connector_name, + 'connector_version': connector_version + } + if not is_exclude(exclude_list, test_suite): + matrix.append(test_suite) + + for tests in matrix: a = tests.get('ansible') - d = tests.get('db_engine_version') + dn = tests.get('db_engine_name') + dv = tests.get('db_engine_version') p = tests.get('python') - c = tests.get('connector') - i = tests.get('docker_image') - make_cmd = f'make ansible="{a}" db_engine_version="{d}" python="{p}" connector="{c}" docker_image="{i}" test-integration' - print(f'Run tests for: Ansible: {a}, DB: {d}, Python: {p}, Connector: {c}, Docker image: {i}') + cn = tests.get('connector_name') + cv = tests.get('connector_version') + make_cmd = ( + f'make ' + f'ansible="{a}" ' + f'db_engine_name="{dn}" ' + f'db_engine_version="{dv}" ' + f'python="{p}" ' + f'connector_name="{cn}" ' + f'connector_version="{cv}" ' + f'test-integration' + ) + print(f'Run tests for: Ansible: {a}, DB: {dn} {dv}, Python: {p}, Connector: {cn} {cv}') os.system(make_cmd) # TODO, allow for CTRL+C to break the loop more easily # TODO, store the failures from this iteration diff --git a/test-containers/mariadb106-py310-mysqlclient211/Dockerfile b/test-containers/mariadb-py310-mysqlclient211/Dockerfile similarity index 100% rename from test-containers/mariadb106-py310-mysqlclient211/Dockerfile rename to test-containers/mariadb-py310-mysqlclient211/Dockerfile diff --git a/test-containers/mariadb106-py310-pymysql102/Dockerfile b/test-containers/mariadb-py310-pymysql102/Dockerfile similarity index 100% rename from test-containers/mariadb106-py310-pymysql102/Dockerfile rename to test-containers/mariadb-py310-pymysql102/Dockerfile diff --git a/test-containers/mariadb103-py38-mysqlclient201/Dockerfile b/test-containers/mariadb-py38-mysqlclient201/Dockerfile similarity index 100% rename from test-containers/mariadb103-py38-mysqlclient201/Dockerfile rename to test-containers/mariadb-py38-mysqlclient201/Dockerfile diff --git a/test-containers/mariadb103-py38-pymysql093/Dockerfile b/test-containers/mariadb-py38-pymysql093/Dockerfile similarity index 100% rename from test-containers/mariadb103-py38-pymysql093/Dockerfile rename to test-containers/mariadb-py38-pymysql093/Dockerfile diff --git a/test-containers/mariadb103-py39-mysqlclient203/Dockerfile b/test-containers/mariadb-py39-mysqlclient203/Dockerfile similarity index 100% rename from test-containers/mariadb103-py39-mysqlclient203/Dockerfile rename to test-containers/mariadb-py39-mysqlclient203/Dockerfile diff --git a/test-containers/mariadb103-py39-pymysql093/Dockerfile b/test-containers/mariadb-py39-pymysql093/Dockerfile similarity index 100% rename from test-containers/mariadb103-py39-pymysql093/Dockerfile rename to test-containers/mariadb-py39-pymysql093/Dockerfile diff --git a/test-containers/my80-py310-mysqlclient211/Dockerfile b/test-containers/mysql-py310-mysqlclient211/Dockerfile similarity index 100% rename from test-containers/my80-py310-mysqlclient211/Dockerfile rename to test-containers/mysql-py310-mysqlclient211/Dockerfile diff --git a/test-containers/my80-py310-pymysql102/Dockerfile b/test-containers/mysql-py310-pymysql102/Dockerfile similarity index 100% rename from test-containers/my80-py310-pymysql102/Dockerfile rename to test-containers/mysql-py310-pymysql102/Dockerfile diff --git a/test-containers/my80-py38-mysqlclient201/Dockerfile b/test-containers/mysql-py38-mysqlclient201/Dockerfile similarity index 100% rename from test-containers/my80-py38-mysqlclient201/Dockerfile rename to test-containers/mysql-py38-mysqlclient201/Dockerfile diff --git a/test-containers/my80-py38-pymysql093/Dockerfile b/test-containers/mysql-py38-pymysql093/Dockerfile similarity index 100% rename from test-containers/my80-py38-pymysql093/Dockerfile rename to test-containers/mysql-py38-pymysql093/Dockerfile diff --git a/test-containers/my80-py39-mysqlclient203/Dockerfile b/test-containers/mysql-py39-mysqlclient203/Dockerfile similarity index 100% rename from test-containers/my80-py39-mysqlclient203/Dockerfile rename to test-containers/mysql-py39-mysqlclient203/Dockerfile diff --git a/test-containers/my80-py39-pymysql093/Dockerfile b/test-containers/mysql-py39-pymysql093/Dockerfile similarity index 100% rename from test-containers/my80-py39-pymysql093/Dockerfile rename to test-containers/mysql-py39-pymysql093/Dockerfile diff --git a/tests/integration/targets/setup_controller/tasks/setvars.yml b/tests/integration/targets/setup_controller/tasks/setvars.yml index d74136da..3e070a93 100644 --- a/tests/integration/targets/setup_controller/tasks/setvars.yml +++ b/tests/integration/targets/setup_controller/tasks/setvars.yml @@ -8,12 +8,22 @@ - name: "{{ role_name }} | Setvars | Set Fact" ansible.builtin.set_fact: gateway_addr: "{{ ip_route_output.stdout }}" - connector_name_version: >- + connector_name_lookup: >- {{ lookup( 'file', - '/root/ansible_collections/community/mysql/tests/integration/connector' + '/root/ansible_collections/community/mysql/tests/integration/connector_name' ) }} - db_engine_version: >- + connector_version_lookup: >- + {{ lookup( + 'file', + '/root/ansible_collections/community/mysql/tests/integration/connector_version' + ) }} + db_engine_name_lookup: >- + {{ lookup( + 'file', + '/root/ansible_collections/community/mysql/tests/integration/db_engine_name' + ) }} + db_engine_version_lookup: >- {{ lookup( 'file', '/root/ansible_collections/community/mysql/tests/integration/db_engine_version' @@ -31,10 +41,10 @@ - name: "{{ role_name }} | Setvars | Set Fact using above facts" ansible.builtin.set_fact: - connector_name: "{{ connector_name_version.split('=')[0].strip() }}" - connector_version: "{{ connector_name_version.split('=')[2].strip() }}" - db_engine: "{{ db_engine_version.split(':')[0].strip() }}" - db_version: "{{ db_engine_version.split(':')[1].strip() }}" + connector_name: "{{ connector_name_lookup.strip() }}" + connector_version: "{{ connector_version_lookup.strip() }}" + db_engine: "{{ db_engine_name_lookup.strip() }}" + db_version: "{{ db_engine_version_lookup.strip() }}" python_version: "{{ python_version_lookup.strip() }}" test_ansible_version: >- {%- if ansible_version_lookup == 'devel' -%}