Multithreaded Compile All

examtool/examtool/api/gen_latex.py

@@ -1,7 +1,10 @@
 import os
+import sys
+import subprocess
 import re
 from collections import defaultdict
 from contextlib import contextmanager
+from pathlib import Path
 
 from examtool.api.scramble import latex_escape
 
@@ -109,7 +112,16 @@ def write_question(question):
 
 
 @contextmanager
-def render_latex(exam, subs=None, *, do_twice=False):
+def render_latex(
+    exam,
+    subs=None,
+    *,
+    do_twice=False,
+    path="temp",
+    outname="out",
+    supress_output=False,
+    return_out_path=False,
+):
     latex = generate(exam)
     latex = re.sub(
         r"\\includegraphics(\[.*\])?{(http.*/(.+))}",
@@ -119,15 +131,33 @@ def render_latex(exam, subs=None, *, do_twice=False):
     if subs:
         for k, v in subs.items():
             latex = latex.replace(f"<{k.upper()}>", v)
-    if not os.path.exists("temp"):
-        os.mkdir("temp")
-    with open("temp/out.tex", "w+") as f:
+    if not os.path.exists(path):
+        Path(path).mkdir(parents=True, exist_ok=True)
+    with open(os.path.join(path, outname + ".tex"), "w+") as f:
         f.write(latex)
-    old = os.getcwd()
-    os.system("cd temp && pdflatex --shell-escape -interaction=nonstopmode out.tex")
+    # old = os.getcwd()
+
+    def compile():
+        subprocess.run(
+            [
+                "pdflatex",
+                "--shell-escape",
+                "-interaction=nonstopmode",
+                f"{outname}.tex",
+            ],
+            stdout=subprocess.DEVNULL if supress_output else sys.stdout,
+            stderr=subprocess.DEVNULL if supress_output else sys.stderr,
+            cwd=path,
+        )
+
+    compile()
     if do_twice:
-        os.system("cd temp && pdflatex --shell-escape -interaction=nonstopmode out.tex")
-    with open("temp/out.pdf", "rb") as f:
-        os.chdir(old)
-        yield f.read()
-    # shutil.rmtree("temp")
+        compile()
+
+    out_path = os.path.join(path, outname + ".pdf")
+    if return_out_path:
+        yield out_path
+    else:
+        with open(out_path, "rb") as f:
+            # os.chdir(old)
+            yield f.read()

examtool/examtool/cli/compile_all.py

@@ -3,7 +3,10 @@
 import pathlib
 from datetime import datetime
 from io import BytesIO
+import multiprocessing
 
+from multiprocessing.pool import ThreadPool, Pool, queue
+from tqdm import tqdm
 from pikepdf import Pdf, Encryption
 import click
 import pytz
@@ -49,7 +52,30 @@
     default=None,
     help="Generates exam regardless of if student is in roster with the set deadline.",
 )
-def compile_all(exam, subtitle, out, do_twice, email, exam_type, semester, deadline):
+@click.option(
+    "--num-threads",
+    default=16,
+    type=int,
+    help="The number of simultaneous exams to process.",
+)
+@click.option(
+    "--same-folder",
+    default=False,
+    is_flag=True,
+    help="This flag will cause the compilation to all occur in the same folder. This improves speed as the tool no longer has to redownload images though can cause images to get overwritten if they share the same name.",
+)
+def compile_all(
+    exam,
+    subtitle,
+    out,
+    do_twice,
+    email,
+    exam_type,
+    semester,
+    deadline,
+    num_threads,
+    same_folder,
+):
     """
     Compile individualized PDFs for the specified exam.
     Exam must have been deployed first.
@@ -74,30 +100,88 @@ def compile_all(exam, subtitle, out, do_twice, email, exam_type, semester, deadl
             else:
                 raise ValueError("Email does not exist in the roster!")
 
-    for email, deadline in roster:
-        if not deadline:
-            continue
-        exam_data = json.loads(exam_str)
-        scramble(email, exam_data)
-        deadline_utc = datetime.utcfromtimestamp(int(deadline))
-        deadline_pst = pytz.utc.localize(deadline_utc).astimezone(
-            pytz.timezone("America/Los_Angeles")
+    rosterlist = list(roster)
+
+    other_data = [
+        exam_str,
+        exam,
+        subtitle,
+        exam_type,
+        semester,
+        do_twice,
+        password,
+        out,
+        same_folder,
+    ]
+
+    for item in rosterlist:
+        item.append(other_data)
+
+    with Pool(num_threads) as p:
+        list(
+            tqdm(
+                p.imap_unordered(render_student_pdf, rosterlist),
+                total=len(rosterlist),
+                desc="Exams Generated",
+                unit="Exam",
+            )
         )
-        deadline_string = deadline_pst.strftime("%I:%M%p")
-
-        with render_latex(
-            exam_data,
-            {
-                "emailaddress": sanitize_email(email),
-                "deadline": deadline_string,
-                "coursecode": prettify(exam.split("-")[0]),
-                "description": subtitle,
-                "examtype": exam_type,
-                "semester": semester,
-            },
-            do_twice=do_twice,
-        ) as pdf:
-            pdf = Pdf.open(BytesIO(pdf))
+
+
+def render_student_pdf(data):
+    (
+        email,
+        deadline,
+        other_data,
+    ) = data
+    (
+        exam_str,
+        exam,
+        subtitle,
+        exam_type,
+        semester,
+        do_twice,
+        password,
+        out,
+        same_folder,
+    ) = other_data
+    if not deadline:
+        return
+    exam_data = json.loads(exam_str)
+    scramble(email, exam_data)
+    deadline_utc = datetime.utcfromtimestamp(int(deadline))
+    deadline_pst = pytz.utc.localize(deadline_utc).astimezone(
+        pytz.timezone("America/Los_Angeles")
+    )
+    deadline_string = deadline_pst.strftime("%I:%M%p")
+
+    uid = multiprocessing.current_process().name
+
+    if same_folder:
+        outname = f"out{uid}"
+        path = "temp"
+    else:
+        outname = "out"
+        path = f"temp/{uid}"
+
+    with render_latex(
+        exam_data,
+        {
+            "emailaddress": sanitize_email(email),
+            "deadline": deadline_string,
+            "coursecode": prettify(exam.split("-")[0]),
+            "description": subtitle,
+            "examtype": exam_type,
+            "semester": semester,
+        },
+        do_twice=do_twice,
+        path=path,
+        outname=outname,
+        supress_output=True,
+        return_out_path=True,
+    ) as out_path:
+        with open(out_path, "rb") as pdf:
+            pdf = Pdf.open(BytesIO(pdf.read()))
             pdf.save(
                 os.path.join(
                     out, "exam_" + email.replace("@", "_").replace(".", "_") + ".pdf"