Merge pull request #5 from nlsandler/complete-test-suite

Complete test suite
nlsandler · Jun 27, 2024 · 7d4cba6 · 7d4cba6
2 parents d0e9a78 + 09f707c
commit 7d4cba6
Show file tree

Hide file tree

Showing 1,944 changed files with 47,630 additions and 1,588 deletions.
diff --git a/.github/workflows/test_tests.yaml b/.github/workflows/test_tests.yaml
@@ -0,0 +1,110 @@
+# This workflow will test out our test suite; yes, it is a test for our tests
+# It runs the unit tests in tests/test_tests, and runs the whole test script to validate its results
+
+name: Compiler Test Suite CI
+
+on:
+  pull_request:
+    branches: [ "main", "complete-test-suite" ]
+
+  workflow_dispatch:
+
+jobs:
+
+  build-final-nqcc:
+
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    uses: nlsandler/nqcc2/.github/workflows/build.yaml@main
+    with:
+      chapter: 20
+      os: ${{ matrix.os }}
+
+  build-partial-nqcc:
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    uses: nlsandler/nqcc2/.github/workflows/build.yaml@main
+    with:
+      chapter: 19
+      os: ${{ matrix.os }}
+
+  test:
+
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.12"] #, "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    needs: [build-final-nqcc, build-partial-nqcc]
+    steps:
+
+    # now checkout test suite
+    - uses: actions/checkout@v4
+      with:
+        path: tests
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+   # next, download both versions of NQCC
+
+    - name: Get final NQCC
+      uses: actions/download-artifact@v4
+      with:
+        name: nqcc-${{ matrix.os }}-20
+        path: nqcc
+
+    - name: Get partial NQCC
+      uses: actions/download-artifact@v4
+      with:
+        name: nqcc-${{ matrix.os }}-19
+        path: nqcc-partial
+
+    - name: Make NQCC executable
+      run: |
+        chmod u+x "$NQCC"
+        chmod u+x "$NQCC_PARTIAL"
+      working-directory: tests
+      env:
+        NQCC: ${{ github.workspace }}/nqcc/main.exe
+        NQCC_PARTIAL: ${{ github.workspace }}/nqcc-partial/main.exe
+
+    # run the test suite
+    # use Rosetta on macOS (instead of just using an x86_64 Github runner)
+    # because that's probably what most readers are doing
+    - name: Test the test suite (macOS M1)
+      if: runner.arch == 'ARM64'
+      run: arch -x86_64 python -m unittest
+      working-directory: tests
+      env:
+        NQCC: ${{ github.workspace }}/nqcc/main.exe
+        NQCC_PARTIAL: ${{ github.workspace }}/nqcc-partial/main.exe
+
+    - name: Test the test suite (x86-64)
+      if: runner.arch != 'ARM64'
+      run: python -m unittest
+      working-directory: tests
+      env:
+        NQCC: ${{ github.workspace }}/nqcc/main.exe
+        NQCC_PARTIAL: ${{ github.workspace }}/nqcc-partial/main.exe
+
+    # Linting
+    - name: Install linter dependencies
+      # note: we do this _after_ running tests, b/c tests should work without
+      # these linters or their dependencies (e.g. typing_extensions)
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install mypy pylint
+    - name: Lint with mypy and pylint
+      run: |
+        # stop the build if there are errors
+        pylint -E tests/test_framework
+        # stop the build if there are type errors
+        mypy tests/test_framework
diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
-*.out
+**/__pycache__/*
+mycc
+*.s
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+gMIT License
+
+Copyright (c) 2023 Nora Sandler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -1,3 +1,64 @@
 # Writing a C Compiler Test Suite
 
-The current version of this test suite is in the `complete-test-suite` branch.
+The test suite for the upcoming book [Writing a C Compiler](https://nostarch.com/writing-c-compiler), a hands-on guide to writing your own compiler for a big chunk of C. These tests are still a work in progress!
+
+Each test case is a C program. Some (in the `valid/` subdirectories) are valid, and others (in the `invalid_*/` subdirectories) have compile-time errors.
+
+The test runner compiles each test program with the compiler under test. An invalid test case passes if the compiler rejects it (i.e. terminates with a non-zero exit code and does not produce any output files). A valid test case passes if it's compiled correctly (i.e. the compiler produces an executable which, when run, produces the expected output and terminates with the expected exit code).
+
+Some invalid test cases include errors that most compilers don't warn about by default. If GCC or Clang isn't complaining about an invalid test case, try compiling it again with the `-pedantic` flag.
+
+## Prerequisites
+You need the `gcc` command on your path. (On macOS this is an alias for Clang; this is fine.) You also need Python 3.8 or later.
+## Quickstart
+
+```
+git clone https://github.com/nlsandler/writing-a-c-compiler-tests.git
+cd writing-a-c-compiler-tests
+git checkout complete-test-suite # until it's merged into main
+./test_compiler --check-setup # make sure you meet all the system requirements
+```
+
+## Usage Examples
+
+
+1. Run the tests for chapters 1-4
+```
+./test_compiler ~/mycc --chapter 4
+```
+
+2. Run the tests for chapter 4 but not chapters 1-3
+
+```
+./test_compiler ~/mycc --chapter 4 --latest-only
+```
+
+3. Run the valid test cases for chapters 1-4, but skip the invalid ones (useful when your frontend passess are working but the backend is buggy).
+
+```
+./test_compiler ~/mycc --chapter 4 --skip-invalid
+```
+
+4. Run the tests for chapters 1-4, stopping after the first test failure:
+
+```
+./test_compiler ~/mycc --chapter 4 -f
+```
+
+5. Run the tests for chapters 1-9; include tests for bitwise operations and switch statements (extra credit features) but not for other extra credit features.
+
+```
+./test_compiler ~/mycc --chapter 9 --bitwise --compound
+```
+
+6. Run test cases for chapter 1; specify that the compiler exits with code `1` or `2` if it hits a lexer or parser error. When specified, an invalid test case passes only if the compiler exits with one of these exit codes, and fails otherwise. Useful for distinguishing expected failures (i.e. the compiler detected an error) from unexpected failures (e.g. internal errors, segfaults).
+
+```
+./test_compiler ~/my_cc --chapter 1 --expected-error-codes 1 2
+```
+
+# Note for Early Access Readers
+
+Two things have changed since the initial early access version of the book:
+1. The chapter numbers have decreased by 1 (e.g. Chapter 2 in the EA version is now Chapter 1).
+2. We now use `int main(void)` instead of `int main()` to declare a function with no parameters. You'll need to define a `void` token in the lexer and include it in the grammar rule for function definitions.
diff --git a/chapter10/invalid_parse/fun_decl_for_loop.c b/chapter10/invalid_parse/fun_decl_for_loop.c
diff --git a/chapter10/invalid_parse/nested_function_definition.c b/chapter10/invalid_parse/nested_function_definition.c
diff --git a/chapter10/invalid_parse/trailing_comma.c b/chapter10/invalid_parse/trailing_comma.c
diff --git a/chapter10/invalid_types/assign_value_to_function.c b/chapter10/invalid_types/assign_value_to_function.c
diff --git a/chapter10/invalid_types/call_variable_as_function.c b/chapter10/invalid_types/call_variable_as_function.c
diff --git a/chapter10/invalid_types/conflicting_function_declarations.c b/chapter10/invalid_types/conflicting_function_declarations.c
diff --git a/chapter10/invalid_types/multiple_function_definitions.c b/chapter10/invalid_types/multiple_function_definitions.c
diff --git a/chapter10/invalid_types_extra_credit/compound_assignment_function_call.c b/chapter10/invalid_types_extra_credit/compound_assignment_function_call.c
diff --git a/chapter10/invalid_types_extra_credit/switch_on_function.c b/chapter10/invalid_types_extra_credit/switch_on_function.c
diff --git a/chapter10/valid/arguments_in_parameters/parameter_shadows_function.c b/chapter10/valid/arguments_in_parameters/parameter_shadows_function.c
diff --git a/chapter10/valid/libraries/addition_client.c b/chapter10/valid/libraries/addition_client.c
diff --git a/chapter10/valid/libraries/many_args.c b/chapter10/valid/libraries/many_args.c
diff --git a/chapter10/valid/libraries/many_args_client.c b/chapter10/valid/libraries/many_args_client.c
diff --git a/chapter10/valid/no_arguments/forward_decl.c b/chapter10/valid/no_arguments/forward_decl.c
diff --git a/chapter10/valid/no_arguments/multiple_declarations.c b/chapter10/valid/no_arguments/multiple_declarations.c
diff --git a/chapter10/valid/no_arguments/no_return_value.c b/chapter10/valid/no_arguments/no_return_value.c
diff --git a/chapter10/valid/no_arguments/precedence.c b/chapter10/valid/no_arguments/precedence.c