Assimilate rspec questionwriter into fppgen

fppgen/__main__.py

@@ -52,13 +52,49 @@ def dump(self) -> str:
 
 def generate_question_html(
     prompt_code: str, *,
+    prefix_code: str = '',
+    suffix_code: str = '',
     question_text: str = None,
     tab: str = '  ',
+    display_format: str = '',
     setup_names: List[AnnotatedName] = None,
     answer_names: List[AnnotatedName] = None
 ) -> str:
     """Turn an extracted prompt string into a question html file body"""
     indented = prompt_code.replace('\n', '\n' + tab)
+    if prefix_code != '' or suffix_code != '':
+        xml_items = [
+            "<code-lines>",
+                tab + indented,
+            "</code-lines>"
+        ]
+
+        if prefix_code != '':
+            xml_items = [
+                "<pre-text>",
+                    tab + prefix_code.replace('\n', '\n' + tab),
+                "</pre-text>"
+            ] + xml_items
+
+        if suffix_code != '':
+            xml_items += [
+                "<post-text>",
+                    tab + suffix_code.replace('\n', '\n' + tab),
+                "</post-text>"
+            ]
+
+        indented = f"\n{tab}".join(xml_items)
+
+    xml_tag_dict = { }
+    if prefix_code != '' or suffix_code != '':
+        xml_tag_dict['format'] = "bottom"
+    if display_format != '':
+        xml_tag_dict['format'] = display_format
+
+    xml_tags = " ".join([ 
+        f'{key}="{value}"'
+        for key, value in xml_tag_dict.items() 
+    ])
 
     if question_text is None:
         question_text = tab + '<!-- Write the question prompt here -->'
@@ -95,9 +131,9 @@ def format_annotated_name(name: AnnotatedName) -> str:
 </pl-question-panel>
 
 <!-- see README for where the various parts of question live -->
-<pl-faded-parsons>
+<pl-faded-parsons {xml_tags}>
 {tab}{indented}
-</pl-faded-parsons>""".format(question_text=question_text, tab=tab, indented=indented)
+</pl-faded-parsons>""".format(question_text=question_text, tab=tab, indented=indented, xml_tags=xml_tags)
 
 
 def generate_info_json(question_name: str, autograder: AutograderConfig, *, indent=4) -> str:
@@ -178,7 +214,9 @@ def remove_region(key, default=''):
     setup_code = remove_region('setup_code', SETUP_CODE_DEFAULT)
     answer_code = remove_region('answer_code')
     server_code = remove_region('server')
+    prefix_code = remove_region('prefix_code')
     prompt_code = remove_region('prompt_code')
+    suffix_code = remove_region('suffix_code')
     question_text = remove_region('question_text')
 
     if options.verbosity > 0: print('- Populating {} ...'.format(question_dir))
@@ -192,7 +230,10 @@ def remove_region(key, default=''):
 
     question_html = generate_question_html(
         prompt_code,
+        prefix_code=prefix_code,
+        suffix_code=suffix_code,
         question_text=question_text,
+        # display_format=..., # TODO: get this working
         setup_names=setup_names,
         # show_required removed:
         # answer_names=answer_names if show_required else None
@@ -224,7 +265,10 @@ def remove_region(key, default=''):
         answer_code,
         setup_code,
         test_region,
-        log_details= options.verbosity > 0
+        source_dir= path.dirname(source_path),
+        pre_code  = prefix_code,
+        post_code = suffix_code,
+        log_details = options.verbosity > 0
     )
 
     if metadata:
@@ -263,6 +307,7 @@ def generate_one(source_path, force_json=False):
         options = Options(args)
         options.force_generate_json = force_json
 
+        source_path = path.abspath(source_path)
         try:
             generate_fpp_question(
                 source_path,

fppgen/lib/autograde.py

@@ -1,7 +1,17 @@
 from abc import ABC, abstractmethod
 from typing import Dict, Union
-from json import loads, dumps
+from subprocess import run, PIPE
 from os import makedirs, path, popen
+from shutil import copytree
+from json import (
+    loads as json_loads, 
+    dumps as json_dumps
+)
+from yaml import (
+    load as yaml_load,
+    Loader as Yaml_Loader,
+    YAMLError,
+)
 
 from lib.name_visitor import AnnotatedName, generate_server, SERVER_DEFAULT
 from lib.consts import TEST_DEFAULT
@@ -22,7 +32,8 @@ def info_json_update(self) -> Dict[str, Union[str, Dict[str, Union[bool, str, in
         pass
 
     @abstractmethod
-    def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, test_region: str, pre_code: str='', post_code: str='', log_details: bool= True) -> None:
+    def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, test_region: str,
+                    source_dir: str, pre_code: str = '', post_code: str = '', log_details: bool = True) -> None:
         pass
 
     def clean_tests_dir(self, test_dir: str) -> None:
@@ -71,7 +82,7 @@ def info_json_update(self) -> Dict[str, Union[str, Dict[str, Union[bool, str, in
         }
 
     def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, test_region: str,
-                    pre_code: str='', post_code: str='', log_details: bool= True) -> None:
+                    source_dir: str, pre_code: str = '', post_code: str = '', log_details: bool = True) -> None:
         test_region = test_region if test_region != "" else TEST_DEFAULT
         try:
             try:
@@ -90,7 +101,7 @@ def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, t
             test_file = test_region
 
         write_to(test_dir, 'test.py', test_file)
-        write_to(test_dir, 'ans.py', answer_code)
+        write_to(test_dir, 'ans.py', "\n".join([pre_code, answer_code, post_code]))
         write_to(test_dir, 'setup_code.py', setup_code)
 
     def generate_server(self, setup_code: str, answer_code: str, *,
@@ -113,7 +124,7 @@ def info_json_update(self) -> Dict[str, Union[str, Dict[str, Union[bool, str, in
         }
 
     def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, test_region: str,
-                    pre_code: str='', post_code: str='', log_details: bool= True) -> None:
+                    source_dir: str, pre_code: str = '', post_code: str = '', log_details: bool = True) -> None:
         app_dir = path.join(path.dirname(f"{test_dir}/"), "app")
         spec_dir = path.join(path.dirname(f"{app_dir}/"), "spec")
         makedirs(spec_dir, exist_ok=True)
@@ -137,8 +148,8 @@ def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, t
         write_to(test_dir, 'meta.json', metadata)
         write_to(test_dir, 'solution', answer_code)
 
-    def clean_tests_dir(self, test_dir: str) -> None:
-        app_dir = path.join(path.dirname(f"{test_dir}/"), "app")
+    def clean_tests_dir(self, test_dir: str, app_dirname: str = "app") -> None:
+        app_dir = path.join(path.dirname(f"{test_dir}/"), app_dirname)
         print(f"Installing gems locally in `{app_dir}` with `{RUBY_SETUP_CMD}` ... ", end="")
         with popen(f"cd {app_dir} && " + RUBY_SETUP_CMD) as out:
             out.read()
@@ -150,3 +161,89 @@ def generate_server(self, setup_code: str, answer_code: str, *,
                     no_ast: bool = False, tab: str = '    ') -> tuple[str, list[AnnotatedName], list[AnnotatedName]]:
         return super().generate_server(setup_code, answer_code, no_ast=no_ast, tab=tab)
 
+@register_autograder(extension='.rspec')
+class RSpecAutograder(RubyAutograder):
+
+    def info_json_update(self) -> Dict[str, str | Dict[str, bool | str | int]]:
+        return {
+            'gradingMethod': 'External',
+            'externalGradingOptions': {
+                'enabled': True,
+                'image': 'saasbook/pl-rspec-autograder',
+                'entrypoint': '/grader/run.py',
+                'timeout': 60
+            }
+        }
+
+    def _apply_mutation(self, app_dir: str, variant_dir: str, filename: str, patch_str: str) -> int:
+        """runs `patch` on the filename with patchfile input `mutations` and returns the error code"""
+        in_file = path.join(app_dir, filename)
+        out_file = path.join(variant_dir, filename)
+
+        command = [
+            "patch",
+            "--normal",
+            "-o", out_file,
+            in_file,
+        ]
+
+        return_code = run(command, input=f"{patch_str}\n".encode(), stderr=PIPE).returncode
+        return return_code
+
+    def populate_tests_dir(self, test_dir: str, answer_code: str, setup_code: str, test_region: str,
+                    source_dir: str, pre_code: str = '', post_code: str = '', log_details: bool = True) -> None:
+        # 1) put solution in tests/solution/_submission_file
+        # 2) put AG metadata in tests/meta.json
+        # 3) put application in tests/common/ (e.g. tests/common/file_at_root.rb)
+        # 4) put mutations in tests/var_{variant_name}/file_at_root.rb
+
+        test_dir = path.dirname(f"{test_dir}/")
+
+        ## 1) put solution in tests/solution/_submission_file
+        # solution_code = "\n".join([pre_code, answer_code, post_code])
+        solution_dir = path.join(test_dir, "solution")
+        makedirs(solution_dir, exist_ok=True)
+        if log_details:
+            print(f"  - Copying solution to {solution_dir} ...")
+        write_to(solution_dir, "_submission_file", answer_code)
+
+        ## 2) put AG metadata in tests/meta.json
+        try:
+            testing_data = yaml_load(test_region, Loader=Yaml_Loader)
+        except YAMLError as e:
+            raise GenerationError(f"Could not parse YAML in `test` region!") from e
+        testing_data["pre-text"] = pre_code
+        testing_data["post-text"] = post_code
+        if log_details:
+            print(f"  - Writing autograder metadata to {test_dir}/meta.json ...")
+        write_to(test_dir, "meta.json", json_dumps(testing_data))
+
+        ## 3) put application in tests/common/ (e.g. tests/common/file_at_root.rb)
+        common_dir = path.join(test_dir, "common/")
+        app_dir = path.join(path.dirname(f"{source_dir}/"), setup_code.strip() + "/")
+        if not path.isdir(app_dir):
+            raise FileNotFoundError(f"System Under Test was not found at {app_dir}! You may need to modify the `setup_code` region")
+        if log_details:
+            print(f"  - Copying SUT from {app_dir} to {common_dir} ...")
+        copytree(app_dir, common_dir, dirs_exist_ok=True)
+
+        ## 4) put mutations in tests/var_{variant_name}/file_at_root.rb
+        # each suite has a set of mutations
+        for variant, data in testing_data["mutations"].items():
+            variant_dir = path.join(test_dir, f"var_{variant}")
+            makedirs(variant_dir, exist_ok=True)
+            files = data["files"]
+
+            for file, patch_str in files.items():
+                file_path = path.join(variant_dir, file)
+                file_dir = path.dirname(file_path)
+                makedirs(file_dir, exist_ok=True)
+
+                if log_details:
+                    print(f"  - Applying mutation to {file} ...")
+                err: int = self._apply_mutation(app_dir, variant_dir, file, patch_str)
+                if err: # make sure to swap file and mutations args
+                    raise ChildProcessError(f"Unexpected error when applying mutation to {file} in variant {variant}: Exited with code {err}")
+
+    def clean_tests_dir(self, test_dir: str) -> None:
+        return super().clean_tests_dir(test_dir, app_dirname = "common")

pl-faded-parsons-question.mustache

@@ -23,7 +23,7 @@
         </div>
         {{/scrambled}}
         {{#pre_text}}
-        <pre class="prettyprint" language="{{{language}}}"> {{{text}}} </pre>
+        <pre class="prettyprint" {{#language}}language="{{{language}}}"{{/language}}> {{{text}}} </pre>
         {{/pre_text}}
         {{#given}}
         <div id="solution-{{uuid}}" class="solution-code-tray codeline-tray soft-border col-sm-{{#narrow}}6{{/narrow}}{{#wide}}12{{/wide}} {{^wide}}px-1{{/wide}}">
@@ -35,7 +35,7 @@
         </div>
         {{/given}}
         {{#post_text}}
-        <pre class="prettyprint" language="{{{language}}}"> {{{text}}} </pre>
+        <pre class="prettyprint" {{#language}}language="{{{language}}}"{{/language}}> {{{text}}} </pre>
         {{/post_text}}
     </div>