Set up Db Seeding Fixture & Increase Test Coverage (#16)

.github/workflows/a11y.yml

@@ -26,11 +26,6 @@ jobs:
         uses: oNaiPs/secrets-to-env-action@v1
         with:
           secrets: ${{ toJSON(vars) }}
-      - name: Spin up Server and Wait
-        run: |
-          echo 'y' | flask resetdb
-          flask run --debugger &
-          sleep 5
       - name: Run A11y Audit
         run: |
           pytest -s tests/a11y

.github/workflows/test.yml

@@ -40,12 +40,6 @@ jobs:
           mkdir -p reports
           touch reports/pytest-${{ matrix.test-type }}.txt
           touch reports/pytest-coverage-${{ matrix.test-type }}.txt
-      - name: Start Flask server for e2e tests
-        if: matrix.test-type == 'e2e'
-        run: |
-          echo 'y' | flask resetdb
-          flask run --debugger &
-          sleep 5
       - name: Run tests with pytest and generate reports
         run: |
           # make sure pipe propogates exit code to fail CI on test failure

README.md

@@ -258,10 +258,8 @@ flask unit
 flask e2e
 # run a11y tests
 flask a11y
-# run all tests (unit, e2e)
+# run all tests (unit, e2e, a11y)
 flask test
-# run coverage
-flask cov
 # run security audit
 flask audit
 ```
@@ -270,14 +268,31 @@ flask audit
 
 #### Testing
 
-(under construction)
-our framework is pytest, using selenium for e2e testing
-should mention deets on oauth/api stubbing, email testing
+This repo is equipped with some typical testing scaffoldings you might expect to find in a full-stack application:
+
+- Test runner: `pytest`
+- Coverage: `pytest-cov`, which uses `coverage.py` under the hood
+- HTTP request stubbing: `responses`
+- Seeding database from fixture files: `flask-fixtures`
+- Webdriver for UI/e2e tests: `selenium`
+
+There are a few other aspects worth noting:
+
+- Canvas OAuth
+
+We did not use `responses` library to stub out Auth API Call. Instead, when `MOCK_CANVAS` env var is set to `True`, our app would connect to a fake local OAuth2 server instead of the actual Canvas OAuth2 server. It still completes the entire OAuth2 flow but draws user information from `server/services/canvas/fake_data/fake_users.json`.
+
+- Canvas API
+
+We did not use `responses` library to stub out Canvas API as we are using the `canvasapi` library instead of calling HTTP endpoints directly. What we did is to use a fake `CanvasClient` when `MOCK_CANVAS` env var is set to `True`, which draws information from `server/services/canvas/fake_data`.
+
+- Email
+
+We provide three ways to test the email feature.
 
 #### CI/CD
 
-(under construction)
-github actions for now
+We are using
 
 #### Deployment
 

cli/db.py

@@ -2,6 +2,7 @@
 
 from server.models import db
 from server import app
+from tests.fixtures import seed_db as _seed_db
 
 
 @app.cli.command('initdb')
@@ -32,7 +33,8 @@ def seed_db():
     There is no need to seed even in development, since the database is
     dynamically populated when app launches, see stub.py
     """
-    pass
+    click.echo('Seeding database...')
+    _seed_db()
 
 
 @app.cli.command('resetdb')

cli/qa.py

@@ -1,38 +1,37 @@
 import click
 import os
-import pytest
 
 from server import app
 
 
-@app.cli.command('e2e')
-def run_e2e():
-    click.echo('Running end-to-end tests...')
-    pytest.main(['-s', 'tests/e2e'])
+@app.cli.command('test')
+def run_all():
+    click.echo('Running all tests...')
+    os.system("""pytest tests --cov=server --cov-report=term-missing --junitxml=reports/pytest.xml --cov-report=term-missing:skip-covered --cov-report=html:reports/coverage --cov-config=tests/.coveragerc""")  # noqa: E501
+    # the above uses command line to run tests (instead of pytest module) so python import lines are counted as covered
+    # see https://stackoverflow.com/questions/70674067
+    # pytest.main(['tests', '--cov=server', '--cov-report=term-missing', '--junitxml=reports/pytest.xml',
+    #             '--cov-report=term-missing:skip-covered', '--cov-report=html:reports/coverage',
+    #              ])
 
 
 @app.cli.command('unit')
 def run_unit():
     click.echo('Running unit tests...')
-    pytest.main(['-s', 'tests/unit'])
-
+    os.system("""pytest tests/unit --cov=server --cov-report=term-missing --junitxml=reports/pytest-unit.xml --cov-report=term-missing:skip-covered --cov-report=html:reports/coverage-unit --cov-config=tests/.coveragerc""")  # noqa: E501
 
-@app.cli.command('a11y')
-def run_a11y():
-    click.echo('Running accessibility tests...')
-    pytest.main(['-s', 'tests/a11y'])
 
+@app.cli.command('e2e')
+def run_e2e():
+    click.echo('Running end-to-end tests...')
 
-@app.cli.command('test')
-def run_all():
-    click.echo('Running all tests...')
-    pytest.main(['-s', 'tests'])
+    os.system("""pytest tests/e2e --cov=server --cov-report=term-missing --junitxml=reports/pytest-e2e.xml --cov-report=term-missing:skip-covered --cov-report=html:reports/coverage-e2e --cov-config=tests/.coveragerc""")  # noqa: E501
 
 
-@app.cli.command('cov')
-def run_cov():
-    click.echo('Running all tests with coverage...')
-    pytest.main(['-s', '--cov=server', 'tests'])
+@app.cli.command('a11y')
+def run_a11y():
+    click.echo('Running accessibility tests...')
+    os.system("""pytest tests/a11y --cov=server --cov-report=term-missing --junitxml=reports/pytest-a11y.xml --cov-report=term-missing:skip-covered --cov-report=html:reports/coverage-a11y --cov-config=tests/.coveragerc""")  # noqa: E501
 
 
 @app.cli.command('audit')

config.py

@@ -16,6 +16,7 @@ def getenv(key, default: Optional[str] = None):
 
     FLASK_APP = "server"
     BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+    FIXTURES_DIRS = [os.path.join('tests', 'fixtures')]
     SERVER_BASE_URL = getenv('SERVER_BASE_URL', "http://localhost:5000/")
 
     SQLALCHEMY_TRACK_MODIFICATIONS = False

requirements-dev.txt

@@ -6,3 +6,4 @@ safety==1.10.3
 pip-audit==2.6.1
 selenium==4.14.0
 responses==0.23.3
+flask-fixtures==0.3.8

server/controllers/dev_login_controllers.py

@@ -15,7 +15,8 @@ def dev_login_page():
         if form.validate_on_submit():
             if form.user_id.data:
                 return oauth_provider.authorize(
-                    callback=url_for('auth.authorized'), state=None,
+                    callback=url_for('auth.authorized'),
+                    state=None,
                     user_id=form.user_id.data,
                     _external=True, _scheme="http")
             else:

server/services/canvas/fake_data/fake_courses.json

@@ -3,7 +3,7 @@
     "id": 1234567,
     "name": "Introduction to Software Engineering (Fall 2023)",
     "sis_course_id": "CRS:COMPSCI-169A-2023-D",
-    "course_code": "COMPSCI 169A-LEC-001"
+    "course_code": "COMPSCI 169A"
   },
   "2345678": {
     "id": 2345678,

tests/.coveragerc

@@ -0,0 +1,10 @@
+[run]
+branch = True
+source = server
+parallel = True
+concurrency = 
+    multiprocessing
+    thread
+
+[data]
+parallel = True

tests/a11y/test_a11y_student.py

@@ -1,4 +1,5 @@
 from tests.a11y.utils import run_axe, assert_no_violations, save_report, print_violations
+from selenium.webdriver.common.by import By
 
 
 def test_a11y_sanity(driver):
@@ -11,19 +12,18 @@ def test_a11y_sanity(driver):
 
 
 # Only three pages are visible to students: select offering, select exam, and seating chart
-def test_a11y_select_offering_page(get_authed_driver):
+def test_a11y_select_offering_page(seeded_db, get_authed_driver):
     """
     Checks a11y for select offering page
     """
     report = run_axe(get_authed_driver("123456"))
     assert_no_violations(report)
 
 
-def test_a11y_select_exam_page(get_authed_driver):
+def test_a11y_select_exam_page(seeded_db, get_authed_driver):
     """
     Checks a11y for select exam page
     """
-    from selenium.webdriver.common.by import By
     driver = get_authed_driver("123456")
     first_offering_btn = driver.find_element(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
     assert first_offering_btn is not None

tests/conftest.py

@@ -1,12 +1,16 @@
 import os
+import subprocess
+import time
 
 import pytest
 import responses
+
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 
 from server import app as flask_app
 from server.models import db as sqlalchemy_db
+from tests.fixtures import seed_db
 
 
 @pytest.fixture()
@@ -33,7 +37,6 @@ def db(app):
         yield sqlalchemy_db
 
         sqlalchemy_db.session.remove()
-        sqlalchemy_db.drop_all()
 
 
 @pytest.fixture()
@@ -43,7 +46,28 @@ def mocker():
 
 
 @pytest.fixture()
-def driver():
+def start_flask_app():
+    import requests
+    TIMEOUT = 5
+    server = subprocess.Popen(['coverage', 'run', '--source', 'server', '-m', 'flask', 'run'])
+    start_time = time.time()
+    while True:
+        try:
+            response = requests.get(flask_app.config.get("SERVER_BASE_URL"))
+            if response.status_code < 400:
+                break
+        except requests.ConnectionError:
+            if time.time() - start_time > TIMEOUT:
+                raise TimeoutError("Flask server did not start within 5 seconds")
+            time.sleep(0.5)
+
+    yield
+
+    server.terminate()
+
+
+@pytest.fixture()
+def driver(start_flask_app):
     options = Options()
     options.add_argument('--headless')
     options.add_argument('--disable-gpu')
@@ -68,3 +92,16 @@ def _get_authed_driver(some_user_id):
         return driver
 
     yield _get_authed_driver
+
+
+@pytest.fixture()
+def seeded_db(app):
+    with app.app_context():
+        sqlalchemy_db.drop_all()
+        sqlalchemy_db.create_all()
+
+        seed_db()
+
+        yield sqlalchemy_db
+
+        sqlalchemy_db.session.expunge_all()

tests/e2e/test_student.py

@@ -0,0 +1,24 @@
+from selenium.webdriver.common.by import By
+
+
+def test_student_can_see_offerings(get_authed_driver, seeded_db):
+    driver = get_authed_driver("234567")
+    offering_btns = driver.find_elements(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    for btn in offering_btns:
+        if "Introduction to Software Engineering" in btn.text:
+            break
+    else:
+        assert False
+
+
+def test_student_can_see_exam(get_authed_driver, seeded_db):
+    driver = get_authed_driver("234567")
+    offering_btns = driver.find_elements(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    for btn in offering_btns:
+        if "Introduction to Software Engineering" in btn.text:
+            btn.click()
+            break
+    else:
+        assert False
+    exam_btn = driver.find_element(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    assert "Midterm 1" in exam_btn.text

tests/e2e/test_ta.py

@@ -0,0 +1,15 @@
+from selenium.webdriver.common.by import By
+
+
+def test_ta_can_see_offerings(get_authed_driver, seeded_db):
+    driver = get_authed_driver("123456")
+    first_offering_btn = driver.find_element(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    assert "Introduction to Software Engineering" in first_offering_btn.text
+
+
+def test_ta_can_see_exam(get_authed_driver, seeded_db):
+    driver = get_authed_driver("123456")
+    first_offering_btn = driver.find_element(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    first_offering_btn.click()
+    exam_btn = driver.find_element(By.CSS_SELECTOR, ".mdl-list__item-primary-content")
+    assert "Midterm 1" in exam_btn.text

tests/e2e/test_web.py

@@ -1,6 +1,3 @@
-from selenium.webdriver.common.by import By
-
-
 def test_webdriver_health(driver):
     """
     Test that webdriver works
@@ -14,3 +11,19 @@ def test_local_home_page(driver):
     Test that home page works
     """
     driver.get('http://localhost:5000/')
+
+
+def test_health_page(driver):
+    """
+    Test that health page works
+    """
+    driver.get('http://localhost:5000/health')
+    assert 'UP' in driver.page_source
+
+
+def test_db_health_page(driver):
+    """
+    Test that db health page works
+    """
+    driver.get('http://localhost:5000/health/db')
+    assert 'UP' in driver.page_source

tests/fixtures/__init__.py

@@ -0,0 +1,20 @@
+from server import app
+from server.models import db
+
+
+def seed_db():
+    import flask_fixtures as ff
+    import os
+    seed_dir_paths = [os.path.join(app.config.get('BASE_DIR'), d)
+                      for d in app.config.get('FIXTURES_DIRS')]
+    seed_files_names = []
+    seed_file_formats = set(['.yaml', '.yml', '.json'])
+
+    for d in seed_dir_paths:
+        for file in os.listdir(d):
+            if not any([file.endswith(form) for form in seed_file_formats]):
+                continue
+            seed_files_names.append(file)
+
+    for filename in seed_files_names:
+        ff.load_fixtures_from_file(db, filename, seed_dir_paths)

tests/fixtures/exam.json

@@ -0,0 +1,14 @@
+[
+  {
+    "model": "server.models.Exam",
+    "records": [
+      {
+        "id": 1,
+        "offering_canvas_id": "1234567",
+        "name": "midterm1",
+        "display_name": "Midterm 1",
+        "is_active": true
+      }
+    ]
+  }
+]