fix: refactor workflows

openedx · Oct 29, 2024 · 46fd01f · 46fd01f
1 parent 2c71edb
commit 46fd01f
Show file tree

Hide file tree

Showing 27 changed files with 545 additions and 532 deletions.
diff --git a/.ci/docker-compose-ci.yml b/.ci/docker-compose-ci.yml
diff --git a/.ci/docker.mk b/.ci/docker.mk
diff --git a/.ci/run_check_keywords.sh b/.ci/run_check_keywords.sh
diff --git a/.ci/run_pii_checker.sh b/.ci/run_pii_checker.sh
diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh
diff --git a/.dockerignore b/.dockerignore
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,47 +2,65 @@ name: Django CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches:
+      - "**"
 
 jobs:
-  build:
-
+  run_tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
-        python-version: ['py311', 'py312']
-        django-version: ['django42']
-        db-version: ['mysql80']
+        python-version: ["3.11", "3.12"]
+        toxenv: ["django42"] # "quality", "pii_check", "check_keywords"
+
+    services:
+      mysql:
+        image: mysql:8.0
+        options: '--health-cmd="mysqladmin ping -h localhost" --health-interval=10s --health-timeout=5s --health-retries=3'
+        env:
+          MYSQL_ROOT_PASSWORD:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+          MYSQL_DATABASE: "edx_notes_api"
+        ports:
+          - 3306:3306
+
+      elasticsearch:
+        image: elasticsearch:7.13.4
+        options: '--health-cmd="curl -f http://localhost:9200 || exit 1" --health-interval=10s --health-timeout=5s --health-retries=3'
+        env:
+          discovery.type: single-node
+          bootstrap.memory_lock: "true"
+          ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+        ports:
+          - 9200:9200
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Start container
-      run: |
-        docker compose -f .ci/docker-compose-ci.yml up -d
-    - name: Install Dependencies
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "apt-get update && apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config"
-    - name: setup python 311
-      if: ${{ matrix.python-version == 'py311' }}
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.11 python3.11-dev python3.11-distutils -y"    
-    - name: setup python 312
-      if: ${{ matrix.python-version == 'py312' }}
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.12 python3.12-dev python3.12-distutils -y"
-    - name: Run Tests
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_tests.sh
-    - name: Run PII Check
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_pii_checker.sh
-    - name: Run Reserved Keywords Check
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_check_keywords.sh
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev
+
+      - name: Install pip and Tox
+        run: pip install --upgrade pip tox
+
+      - name: Run Tox tests
+        env:
+          CONN_MAX_AGE: 60
+          DB_ENGINE: django.db.backends.mysql
+          DB_HOST: 127.0.0.1
+          DB_NAME: edx_notes_api
+          DB_PASSWORD:
+          DB_PORT: 3306
+          DB_USER: root
+          ENABLE_DJANGO_TOOLBAR: 1
+          ELASTICSEARCH_URL: http://127.0.0.1:9200
+        run: tox -e ${{ matrix.toxenv }}
diff --git a/.github/workflows/migrations-mysql8-check.yml b/.github/workflows/migrations-mysql8-check.yml
@@ -1,4 +1,4 @@
-name: Migrations check on mysql8
+name: Migrations check on MySQL 8
 
 on:
   workflow_dispatch:
@@ -9,74 +9,53 @@ on:
 
 jobs:
   check_migrations:
-    name: check migrations
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ '3.11', '3.12' ]
+        python-version: ["3.11", "3.12"]
 
     steps:
-    - name: Checkout repo
-      uses: actions/checkout@v4
-
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install system Packages
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libxmlsec1-dev
-    - name: Get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: Cache pip dependencies
-      id: cache-dependencies
-      uses: actions/cache@v4
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip-tools.txt') }}
-        restore-keys: ${{ runner.os }}-pip-
-
-    - name: Ubuntu and sql Versions
-      run: |
-        lsb_release -a
-        mysql -V
-
-    # pinning xmlsec to version 1.3.13 to avoid the CI error, migration checks are failing due to an issue in the latest release of python-xmlsec
-    # https://github.com/xmlsec/python-xmlsec/issues/314
-    - name: Install Python dependencies
-      run: |
-        pip install -r requirements/pip-tools.txt
-        pip install -r requirements/test.txt
-        pip install -r requirements/base.txt
-        pip uninstall -y mysqlclient
-        pip install --no-binary mysqlclient mysqlclient
-        pip uninstall -y xmlsec
-        pip install --no-binary xmlsec xmlsec==1.3.13
-
-    - name: Initiate Services
-      run: |
-        sudo /etc/init.d/mysql start
-    - name: Reset mysql password
-      run: |
-        cat <<EOF | mysql -h 127.0.0.1 -u root --password=root
-          UPDATE mysql.user SET authentication_string = null WHERE user = 'root';
-          FLUSH PRIVILEGES;
-        EOF
-    - name: Run Tests
-      env:
-        DB_ENGINE: django.db.backends.mysql
-        DB_NAME: edx_notes_api
-        DB_USER: root
-        DB_PASSWORD:
-        DB_HOST: localhost
-        DB_PORT: 3306
-      run: |
-        echo "CREATE DATABASE IF NOT EXISTS edx_notes_api;" | sudo mysql -u root
-        echo "Running the migrations."
-        python manage.py migrate --settings=notesserver.settings.test
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }} with cache
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: "**/pip-tools.txt"
+
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev
+
+      # pinning xmlsec to version 1.3.13 to avoid the CI error, migration checks are failing due to an issue in the latest release of python-xmlsec
+      # https://github.com/xmlsec/python-xmlsec/issues/314
+      - name: Install Python dependencies
+        run: |
+          pip install -r requirements/pip-tools.txt
+          pip install -r requirements/test.txt
+          pip install -r requirements/base.txt
+          pip uninstall -y mysqlclient
+          pip install --no-binary mysqlclient mysqlclient
+          pip uninstall -y xmlsec
+          pip install --no-binary xmlsec xmlsec==1.3.13
+
+      - name: Start MySQL service
+        run: sudo service mysql start
+
+      - name: Reset MySQL root password
+        run: |
+          mysql -h 127.0.0.1 -u root -proot -e "UPDATE mysql.user SET authentication_string = null WHERE user = 'root'; FLUSH PRIVILEGES;"
+
+      - name: Run migrations
+        env:
+          DB_ENGINE: django.db.backends.mysql
+          DB_NAME: edx_notes_api
+          DB_USER: root
+          DB_PASSWORD:
+          DB_HOST: localhost
+          DB_PORT: 3306
+        run: |
+          echo "CREATE DATABASE IF NOT EXISTS edx_notes_api;" | sudo mysql -u root
+          python manage.py migrate --settings=notesserver.settings.test
diff --git a/.github/workflows/push-docker-image.yml b/.github/workflows/push-docker-image.yml