Merge pull request #91 from Wenzel/e2e_testing

E2e testing

.github/workflows/ci.yml

@@ -99,10 +99,15 @@ jobs:
 
   test:
     needs: build
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Set up Python 3.7 🐍
         uses: actions/setup-python@v1
@@ -113,13 +118,7 @@ jobs:
         run: python -m pip install nox==2020.8.22
 
       - name: Run tests
-        run: nox -s test
-
-      - name: Install package 📦
-        run: python -m pip install .
-
-      - name: Run smoke test
-        run: checksec /usr/lib
+        run: nox -s test_e2e -- -s
 
   release:
     needs: test

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/binaries"]
+	path = tests/binaries
+	url = [email protected]:Wenzel/checksec.py-test-binaries.git

noxfile.py

@@ -1,6 +1,6 @@
 import nox
 
-nox.options.sessions = ["fmt", "lint", "test"]
+nox.options.sessions = ["fmt", "lint", "test_e2e"]
 
 
 @nox.session
@@ -31,7 +31,28 @@ def run(session):
 
 
 @nox.session
-def test(session):
+def test_unit(session):
+    # run unit tests
+    args = session.posargs
     session.install("-r", "requirements.txt")
     session.install("pytest==6.0.2", "coverage==5.3")
-    session.run("coverage", "run", "-m", "pytest", "-v")
+    session.run("coverage", "run", "--concurrency=multiprocessing", "-m", "pytest", "-v", "-k", "unit", *args)
+    session.run("coverage", "combine")
+    session.run("coverage", "report")
+
+
+@nox.session
+def test_e2e(session):
+    args = session.posargs
+    session.install(".")
+    session.install("pytest==6.0.2", "coverage==5.3")
+    session.run("coverage", "run", "--concurrency=multiprocessing", "-m", "pytest", "-v", "-k", "e2e", *args)
+    session.run("coverage", "combine")
+    session.run("coverage", "report")
+
+
+@nox.session
+def coverage(session):
+    args = session.posargs if session.posargs else ["report"]
+    session.install("coverage==5.3")
+    session.run("coverage", *args)

setup.cfg

@@ -3,3 +3,10 @@ max-line-length=120
 exclude = .git, .nox, venv, build, dist
 # black compatibility
 extend-ignore = E203
+
+[coverage:run]
+# measure branch coverage in addition to statement coverage
+branch = True
+# restrict measurement to avoid .nox/*
+source =
+    checksec

setup.py

@@ -1,14 +1,14 @@
 import setuptools
 
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 with open("requirements.txt") as f:
     requirements = f.read().splitlines()
 
 setuptools.setup(
     name="checksec.py",
-    version="0.5.0",
+    version="0.5.1",
     author="Mathieu Tarral",
     author_email="[email protected]",
     description="Checksec tool implemented in Python",

tests/conftest.py

@@ -0,0 +1,13 @@
+import json
+import subprocess
+from pathlib import Path
+from typing import Dict
+
+
+def run_checksec(bin_path: Path, libc_path: Path = None) -> Dict:
+    """Runs checksec from command line, returns json output as dict"""
+    cmd = ["checksec", str(bin_path), "-j"]
+    if libc_path:
+        cmd.extend(["-s", str(libc_path)])
+    output = subprocess.check_output(cmd)
+    return json.loads(output.decode())

tests/e2e/test_e2e_elf.py

@@ -0,0 +1,66 @@
+"""E2E tests for ELF"""
+
+from pathlib import Path
+
+import pytest
+
+from checksec.elf import PIEType, RelroType
+from tests.conftest import run_checksec
+
+ELF_BINARIES = Path(__file__).parent.parent / "binaries" / "elf"
+
+
+@pytest.mark.parametrize("is_enabled", [False, True])
+@pytest.mark.parametrize("prop", ["nx", "canary", "rpath", "runpath", "symbols", "fortify_source"])
+def test_bool_prop(prop: str, is_enabled: bool):
+    """Test that boolean prop is disabled/enabled"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / f"{prop}_{'enabled' if is_enabled else 'disabled'}"
+    chk_data = run_checksec(bin_path, libc_path)
+    assert chk_data[str(bin_path)][prop] == is_enabled
+
+
+@pytest.mark.parametrize("relro_type", list(RelroType))
+def test_relro(relro_type: RelroType):
+    """Test that relro type is No/Partial/Full"""
+    bin_path = ELF_BINARIES / f"relro_{relro_type.name.lower()}"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)]["relro"] == relro_type.name
+
+
+@pytest.mark.parametrize("pie_type", list(PIEType))
+def test_pie(pie_type):
+    """Test that PIE is No/Partial/Full"""
+    bin_path = ELF_BINARIES / f"pie_{pie_type.name.lower()}"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)]["pie"] == pie_type.name
+
+
+def test_fortified():
+    """Test the fortified functions"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    assert chk_data[str(bin_path)]["fortified"] == len(fortified_funcs)
+
+
+def test_fortifiable():
+    """Test the fortifiable functions"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    non_fortified_funcs = ["fgets"]
+    assert chk_data[str(bin_path)]["fortify-able"] == len(fortified_funcs) + len(non_fortified_funcs)
+
+
+def test_fortify_score():
+    """Test the fortify score"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    non_fortified_funcs = ["fgets"]
+    total = len(fortified_funcs) + len(non_fortified_funcs)
+    assert chk_data[str(bin_path)]["fortify_score"] == round((2 * 100) / total, 0)

tests/e2e/test_e2e_pe.py

@@ -0,0 +1,31 @@
+"""E2E tests for PE"""
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import run_checksec
+
+PE_BINARIES = Path(__file__).parent.parent / "binaries" / "pe"
+
+
+@pytest.mark.parametrize("is_enabled", [False, True])
+@pytest.mark.parametrize(
+    "prop",
+    [
+        "nx",
+        "canary",
+        "dynamic_base",
+        "aslr",
+        "high_entropy_va",
+        "safe_seh",
+        "force_integrity",
+        "guard_cf",
+        "isolation",
+    ],
+)
+def test_bool_prop(prop: str, is_enabled: bool):
+    """Test that boolean prop is disabled/enabled"""
+    bin_path = PE_BINARIES / f"{prop}_{'enabled' if is_enabled else 'disabled'}.exe"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)][prop] == is_enabled