reviewing and fixing chapter part i no-coalescing tests

Use templates to generate some common code

generate_expected_results.py

@@ -7,8 +7,9 @@
 import itertools
 import json
 import subprocess
+import sys
 from pathlib import Path
-from typing import Any, Iterable, List
+from typing import Any, Iterable
 
 # NOTE: basic loads EXPECTED_RESULTS from a file so this whole script will fail
 # if expected_results.json doesn't already exist
@@ -18,21 +19,9 @@
 results: dict[str, dict[str, Any]] = {}
 
 
-def lookup_regalloc_libs(prog: Path) -> List[Path]:
-    """Look up extra library we need to link against for regalloc tests"""
-    # TODO fix copypasta b/t here and test_programs.py
-    test_info = regalloc.REGALLOC_TESTS.get(prog.name)
-    if test_info is None:
-        return []
-    if test_info.extra_lib is None:
-        # this uses the wrapper script b/c test inspects assembly
-        # but doesn't use other library
-        return [regalloc.WRAPPER_SCRIPT]
-    # uses wrapper script and other library
-    return [
-        regalloc.WRAPPER_SCRIPT,
-        TEST_DIR / "chapter_20/libraries" / test_info.extra_lib,
-    ]
+def needs_wrapper(prog: Path) -> bool:
+    """Check whether we need to link against wrapper script"""
+    return prog.name in regalloc.REGALLOC_TESTS
 
 
 def cleanup_keys() -> None:
@@ -111,7 +100,7 @@ def main() -> None:
                 str(rel_path) in changed_files
                 or str(rel_path).replace(".c", "_client.c") in changed_files
                 or str(rel_path).replace("_client.c", ".c") in changed_files
-                or any(lib in changed_files for lib in lookup_regalloc_libs(p))
+                or (needs_wrapper(p) and regalloc.WRAPPER_SCRIPT in changed_files)
                 or any(lib in changed_files for lib in basic.get_libs(p))
                 or any(
                     h
@@ -151,9 +140,9 @@ def main() -> None:
         source_files.extend(basic.get_libs(prog))
 
         if "chapter_20" in prog.parts:
-            # we may need to include wrapper script and other library files
-            extra_libs = lookup_regalloc_libs(prog)
-            source_files.extend(extra_libs)
+            # we may need to include wrapper script too
+            if needs_wrapper(prog):
+                source_files.append(regalloc.WRAPPER_SCRIPT)
 
         opts = []
         if any(basic.needs_mathlib(p) for p in source_files):
@@ -170,6 +159,8 @@ def main() -> None:
 
             key = str(prog.relative_to(TEST_DIR))
             results[key] = result_dict
+            if result.returncode:
+                print(f"Return code for {key} is {result.returncode}", file=sys.stderr)
         finally:
             # delete executable
             exe = source_files[0].with_suffix("")

regalloc_templates.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+"""Autogenerate several very similar test cases where we create specific interference graphs"""
+from pathlib import Path
+from string import ascii_lowercase
+
+from jinja2 import Environment, FileSystemLoader, pass_environment
+from jinja2.filters import do_wordwrap
+
+
+@pass_environment
+def comment_wrap(e: Environment, value: str, width: int = 73) -> str:
+    # default width is short b/c we usually call this in a context w/ indent of 4
+    # and there's no good way to directly track current indent
+    lines = [l.strip().removeprefix("//") for l in value.splitlines()]
+    oneline = "//" + "".join(lines)
+    return (
+        do_wordwrap(
+            e,
+            oneline,
+            width=width,
+            break_long_words=False,
+            wrapstring="\n// ",
+        )
+        + "\n"
+    )
+
+
+test_cases = {
+    "tests/chapter_11/valid/long_expressions/rewrite_large_multiply_regression.c": {
+        "comment": {
+            "instr": "imul",
+            "extra_desc": " and source operands are immediates value larger than INT32_MAX",
+            "other_test": "tests/chapter_11/valid/long_expressions/large_constants.c",
+            "operation_desc": "a multiply by a large immediate value",
+            "operation_name": "multiply",
+        },
+        "glob": {"type": "long", "init": "5l"},
+        "should_spill": {
+            "type": "long",
+            "expr": "glob * 4294967307l",
+            "val": "21474836535l",
+        },
+        "one_expr": "glob - 4",
+        "thirteen_expr": "glob + 8",
+    },
+    "tests/chapter_12/valid/explicit_casts/rewrite_movz_regression.c": {
+        "comment": {
+            "instr": "MovZeroExtend",
+            "operation_desc": "a zero extension",
+            "operation_name": "zero extend",
+        },
+        "glob": {"type": "unsigned", "init": "5000u"},
+        "should_spill": {"type": "long", "expr": "(long)glob", "val": "5000l"},
+        "one_expr": "glob - 4999",
+        "thirteen_expr": "glob - 4987u",
+    },
+    "tests/chapter_16/valid/chars/rewrite_movz_regression.c": {
+        "comment": {
+            "instr": "movz",
+            "operation_desc": "a zero extension",
+            "operation_name": "zero extend",
+        },
+        "glob": {"type": "unsigned char", "init": "5"},
+        "should_spill": {"type": "int", "expr": "(int)glob", "val": "5"},
+        "one_expr": "glob - 4",
+        "thirteen_expr": "8 + glob",
+    },
+    "tests/chapter_13/valid/explicit_casts/rewrite_cvttsd2si_regression.c": {
+        "comment": {
+            "instr": "cvttsd2si",
+            "operation_desc": "a cvttsd2si",
+            "operation_name": "cvttsd2sdi",
+        },
+        "glob": {"type": "double", "init": "5000."},
+        "should_spill": {"type": "long", "expr": "(long)glob", "val": "5000"},
+        "one_expr": "glob - 4999",
+        "thirteen_expr": "glob - 4987",
+    },
+}
+
+env = Environment(
+    loader=FileSystemLoader("templates"), trim_blocks=True, lstrip_blocks=True
+)
+env.globals["letters"] = list(ascii_lowercase[0:12])
+env.filters["comment_wrap"] = comment_wrap
+
+# pre-chapter 20 tests
+for k, v in test_cases.items():
+    templ = env.get_template("pre_ch20_spill_var.c.jinja")
+    src = templ.render(v)
+    with open(k, "w", encoding="utf-8") as f:
+        f.write(src)
+
+# chapter 20 tests
+
+# for templates we use to generate multiple test cases,
+# specify each test's destination and variables
+configurable_templates = {
+    # none yet!
+}
+
+
+template_files = Path("templates/chapter_20_templates").iterdir()
+for t in template_files:
+    if t.suffix != ".jinja":
+        exit(f"Found non-template {f} in templates directory")
+
+    templ = env.get_template(str(t.relative_to("templates")))
+    if t.name in configurable_templates:
+        for dest, templ_vars in configurable_templates[t.name].items():
+            src = templ.render(templ_vars)
+            output_path = Path("tests/chapter_20/int_only/no_coalescing") / dest
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(src)
+    elif str(t).endswith(".s.jinja"):
+        # generate once per platform
+        basename = t.name.removesuffix(".s.jinja")
+
+        for platform in ["linux", "osx"]:
+            src = templ.render(platform=platform)
+            new_name = f"{basename}_{platform}.s"
+            output_path = Path("tests/chapter_20/libraries") / new_name
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(src)
+
+    else:
+        src = templ.render()
+        output_path = Path("tests/chapter_20/int_only/no_coalescing") / t.stem
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(src)

templates/chapter_20_templates/alignment_check_wrapper.s.jinja

@@ -0,0 +1,30 @@
+{% extends "includes/wrapper_base.s.jinja" %}
+{% block call %}
+	# call test functions, all of which exit early on failure
+	callq	test1
+	callq	test2
+	callq	test3
+{% endblock %}
+{% block more_functions %}
+    .text
+    .globl {{id_prefix}}check_alignment
+{{id_prefix}}check_alignment:
+    pushq   %rbp
+    movq    %rsp, %rbp
+    # calculate rsp % 16
+    movq    %rsp, %rax
+    movq    $0, %rdx
+    movq    $16, %rcx
+    div     %rcx
+    # compare result (in rdx) to 0
+    cmpq    $0, %rdx
+    je      {{local_prefix}}_OK
+    # it's not zero; exit
+    # using exit code already in EDI
+    call    exit@PLT
+{{local_prefix}}_OK:
+    # success; rsp is aligned correctly
+    movl    $0, %eax
+    popq    %rbp
+    retq
+{% endblock %}

templates/chapter_20_templates/force_spill.c.jinja

@@ -0,0 +1,33 @@
+{%- import 'includes/regalloc_macros.c.jinja' as helpers -%}
+{% set one_expr = "glob_three - 2" %}
+{% set thirteen_expr = "10 + glob_three" %}
+{% set spill_thing="should_spill" %}
+/* Test that we can handle spilling correctly.
+ * We have to spill one pseudo. The test script will validate that
+ * we spill only one and it's the cheapest one.
+ * Note that this isn't a foolproof test of spill cost calculation;
+ * because of optimistic coloring, we might end up spilling should_spill
+ * even if it's not the first spill candidate we select.
+ * This test program is generated from templates/{{ self._TemplateReference__context.name }}
+ * */
+
+#include "util.h"
+
+int glob_three = 3;
+
+int target(void) {
+    // This is our spill candidate: it has the highest degree and is
+    // used only once.
+    int should_spill = glob_three + 3;
+
+    {% filter indent(width=4, first=true) %}
+    {% include 'includes/spill_var.c.jinja' %}
+    {% endfilter %}
+
+
+    if (should_spill != 6) {
+        return -1;  // fail
+    }
+
+    return 0;  // success
+}

templates/chapter_20_templates/rewrite_regression_test.c.jinja

@@ -0,0 +1,47 @@
+{%- import 'includes/regalloc_macros.c.jinja' as helpers -%}
+{% set one_expr = "glob_three - 2" %}
+{% set thirteen_expr = "10 + glob_three" %}
+{% set spill_thing="imul, add, and sub results" %}
+/* This isn't really a test of the register allocator.
+ * Verify that we correctly rewrite add/sub/imul instructions with operands in
+ * memory. Test programs for earlier chapters exercise these rewrite rules only
+ * when register allocation and optimizations are disabled. But once we complete
+ * Part III, these are either optimized away entirely in earlier chapters' test
+ * programs, or their operands are all hard registers.
+ *
+ * This test program is generated from templates/{{ self._TemplateReference__context.name }}
+ * */
+
+#include "util.h"
+
+int glob_three = 3;
+int glob_four = 4;
+
+int target(void) {
+    // We'll force the results of imul, add, and sub instructions to spill
+    // to memory (by making them conflict with more registers and have fewer
+    // uses then any other pseudo) to verify that we rewrite them correctly
+
+    // These results will conflict with all other pseudos and only be used once
+    // each, so they'll all spill
+    int imul_result = glob_three * glob_four;
+    int add_result = glob_three + glob_four;
+    int sub_result = glob_four - glob_three;
+
+    {% filter indent(width=4, first=true) %}
+    {% include 'includes/spill_var.c.jinja' %}
+    {% endfilter %}
+
+
+    if (imul_result != 12) {
+        return 100;
+    }
+    if (add_result != 7) {
+        return 101;
+    }
+    if (sub_result != 1) {
+        return 102;
+    }
+
+    return 0;  // success
+}

templates/chapter_20_templates/use_all_hardregs.c.jinja

@@ -0,0 +1,20 @@
+/* Make sure we use all hardregs rather than spill;
+ * Create 12 pseudos that all interfere with each other
+ * and make sure we assign all of them to hardregs
+ * This test program is generated from templates/{{ self._TemplateReference__context.name }}
+ * */
+#include "util.h"
+
+int global_one = 1;  // to prevent constant-folding
+
+int target(void) {
+    // create a clique of 12 pseudos that interfere
+    // we can color all of them w/out spilling anything
+    {% set one_expr="2 - global_one" %}
+
+    {% filter indent(width=4, first=true) %}
+    {% include 'includes/twelve_regs_conflict.c.jinja' %}
+    {% endfilter %}
+
+    return 0;  // success
+}

templates/chapter_20_templates/wrapper.s.jinja

@@ -0,0 +1,18 @@
+{% extends "includes/wrapper_base.s.jinja" %}
+{% block call %}
+	# call target
+	movl	$1, %edi
+	movl	$2, %esi
+	movl	$3, %edx
+	movl	$4, %ecx
+	movl	$5, %r8d
+	movl	$6, %r9d
+	movsd	{{local_prefix}}one(%rip), %xmm0
+	movsd	{{local_prefix}}two(%rip), %xmm1
+	movsd	{{local_prefix}}three(%rip), %xmm2
+	movsd	{{local_prefix}}four(%rip), %xmm3
+	movsd	{{local_prefix}}five(%rip), %xmm4
+	movsd	{{local_prefix}}six(%rip), %xmm5
+	movsd	{{local_prefix}}seven(%rip), %xmm7
+	callq	{{id_prefix}}target
+{% endblock %}

templates/includes/regalloc_macros.c.jinja

@@ -0,0 +1,26 @@
+{% set check_12_ints_decl -%}
+// for validation
+int check_12_ints(int start, int a, int b, int c, int d, int e, int f, int g,
+                  int h, int i, int j, int k, int l);
+{%- endset %}
+
+{% set check_12_ints %}
+// validate that a == start, b == start + 1, ...l == start + 11
+// NOTE: 'start' is the last param because if it were first, every
+// arg in the caller would interfere with EDI and we'd have to spill more than
+// one pseudo
+int check_12_ints(int a, int b, int c, int d, int e, int f, int g, int h, int i,
+                  int j, int k, int l, int start) {
+    int expected = 0;
+    {% set args = letters %}
+    {% for arg in args %}
+
+    expected = start + {{ loop.index0 }};
+    if ({{ arg }} != expected) {
+        return expected;
+    }
+    {% endfor %}
+
+    return 0;  // success
+}
+{% endset %}