Merge pull request #17 from dinkelk/persistant-model-cache-do

Persistant Model Caching

default.do

@@ -16,75 +16,77 @@ from util import redo_arg
 # as those built by a generator.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  directory, base = redo_arg.split_redo_arg(sys.argv[2])
-  rule_cls = None
-  # Is the file a product of the metric generator:
-  if redo_arg.in_build_metric_dir(sys.argv[2]):
-    from rules.build_metric import build_metric as rule_cls
-  # Is the file a type range yaml file?
-  elif base.endswith(".type_ranges.yaml"):
-    from rules.build_type_ranges_yaml import build_type_ranges_yaml as rule_cls
-  elif base.endswith("_h.ads") or base.endswith("_hpp.ads"):
-    from rules.build_bindings import build_bindings as rule_cls
-  # Is the file a product of a generator?
-  elif redo_arg.in_build_dir(sys.argv[2]):
-    from rules.build_via_generator import build_via_generator as rule_cls
-  # Special redo directives:
-  elif base == "clean":
-    from rules.build_clean import build_clean as rule_cls
-  elif base == "clean_all":
-    from rules.build_clean_all import build_clean_all as rule_cls
-  elif base == "what":
-    from rules.build_what import build_what as rule_cls
-  elif base == "prove":
-    from rules.build_prove import build_prove as rule_cls
-  elif base == "analyze":
-    from rules.build_analyze import build_analyze as rule_cls
-  elif base == "style":
-    from rules.build_style import build_style as rule_cls
-  elif base == "style_all":
-    from rules.build_style_all import build_style_all as rule_cls
-  elif base == "pretty":
-    from rules.build_pretty import build_pretty as rule_cls
-  elif base == "targets":
-    from rules.build_targets import build_targets as rule_cls
-  elif base == "templates":
-    from rules.build_templates import build_templates as rule_cls
-  elif base == "all":
-    from rules.build_all import build_all as rule_cls
-  elif base == "recursive":
-    from rules.build_recursive import build_recursive as rule_cls
-  elif base == "path":
-    from rules.build_path import build_path as rule_cls
-  elif base == "print_path":
-    from rules.build_print_path import build_print_path as rule_cls
-  elif base == "test":
-    from rules.build_test import build_test as rule_cls
-  elif base == "coverage":
-    from rules.build_coverage import build_coverage as rule_cls
-    from util import target as tgt
-    tgt.set_default_coverage_target()
-  elif base == "run":
-    from rules.build_run import build_run as rule_cls
-  elif base == "test_all":
-    from rules.build_test_all import build_test_all as rule_cls
-  elif base == "coverage_all":
-    from rules.build_coverage_all import build_coverage_all as rule_cls
-  elif base == "publish":
-    from rules.build_publish import build_publish as rule_cls
-  elif base == "codepeer_server":
-    from rules.build_codepeer_server import build_codepeer_server as rule_cls
-  elif base == "yaml_sloc":
-    from rules.build_yaml_sloc import build_yaml_sloc as rule_cls
+    assert len(sys.argv) == 4
+    directory, base = redo_arg.split_redo_arg(sys.argv[2])
+    rule_cls = None
+    # Is the file a product of the metric generator:
+    if redo_arg.in_build_metric_dir(sys.argv[2]):
+        from rules.build_metric import build_metric as rule_cls
+    # Is the file a type range yaml file?
+    elif base.endswith(".type_ranges.yaml"):
+        from rules.build_type_ranges_yaml import build_type_ranges_yaml as rule_cls
+    elif base.endswith("_h.ads") or base.endswith("_hpp.ads"):
+        from rules.build_bindings import build_bindings as rule_cls
+    # Is the file a product of a generator?
+    elif redo_arg.in_build_dir(sys.argv[2]):
+        from rules.build_via_generator import build_via_generator as rule_cls
+    # Special redo directives:
+    elif base == "clean":
+        from rules.build_clean import build_clean as rule_cls
+    elif base == "clean_all":
+        from rules.build_clean_all import build_clean_all as rule_cls
+    elif base == "clear_cache":
+        from rules.build_clear_cache import build_clear_cache as rule_cls
+    elif base == "what":
+        from rules.build_what import build_what as rule_cls
+    elif base == "prove":
+        from rules.build_prove import build_prove as rule_cls
+    elif base == "analyze":
+        from rules.build_analyze import build_analyze as rule_cls
+    elif base == "style":
+        from rules.build_style import build_style as rule_cls
+    elif base == "style_all":
+        from rules.build_style_all import build_style_all as rule_cls
+    elif base == "pretty":
+        from rules.build_pretty import build_pretty as rule_cls
+    elif base == "targets":
+        from rules.build_targets import build_targets as rule_cls
+    elif base == "templates":
+        from rules.build_templates import build_templates as rule_cls
+    elif base == "all":
+        from rules.build_all import build_all as rule_cls
+    elif base == "recursive":
+        from rules.build_recursive import build_recursive as rule_cls
+    elif base == "path":
+        from rules.build_path import build_path as rule_cls
+    elif base == "print_path":
+        from rules.build_print_path import build_print_path as rule_cls
+    elif base == "test":
+        from rules.build_test import build_test as rule_cls
+    elif base == "coverage":
+        from rules.build_coverage import build_coverage as rule_cls
+        from util import target as tgt
+        tgt.set_default_coverage_target()
+    elif base == "run":
+        from rules.build_run import build_run as rule_cls
+    elif base == "test_all":
+        from rules.build_test_all import build_test_all as rule_cls
+    elif base == "coverage_all":
+        from rules.build_coverage_all import build_coverage_all as rule_cls
+    elif base == "publish":
+        from rules.build_publish import build_publish as rule_cls
+    elif base == "codepeer_server":
+        from rules.build_codepeer_server import build_codepeer_server as rule_cls
+    elif base == "yaml_sloc":
+        from rules.build_yaml_sloc import build_yaml_sloc as rule_cls
 
-  # Run the rule
-  if rule_cls:
-    rule = rule_cls()
-    rule.build(*sys.argv[1:])
-  else:
-    from util import error
-    error.error_abort("default.do: No rule to build '" + sys.argv[1] + "'.")
+    # Run the rule
+    if rule_cls:
+        rule = rule_cls()
+        rule.build(*sys.argv[1:])
+    else:
+        from util import error
+        error.error_abort("default.do: No rule to build '" + sys.argv[1] + "'.")
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.elf.do

@@ -11,9 +11,9 @@ from rules.build_executable import build_executable
 # This .do file builds .elf (executable binary) files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_executable()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_executable()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.eps.do

@@ -11,9 +11,9 @@ from rules.build_eps import build_eps
 # This .do file builds .eps (PostScript) files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_eps()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_eps()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.gpr.do

@@ -11,9 +11,9 @@ from rules.build_gpr import build_gpr
 # This .do file builds .gpr files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_gpr()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_gpr()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.o.do

@@ -11,9 +11,9 @@ from rules.build_object import build_object
 # This .do file builds .o (object) files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_object()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_object()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.pdf.do

@@ -11,9 +11,9 @@ from rules.build_pdf import build_pdf
 # This .do file builds .pdf files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_pdf()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_pdf()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.png.do

@@ -11,9 +11,9 @@ from rules.build_png import build_png
 # This .do file builds .png files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_png()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_png()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

default.svg.do

@@ -11,9 +11,9 @@ from rules.build_svg import build_svg
 # This .do file builds .svg (vector graphics) files.
 
 if __name__ == "__main__":
-  assert len(sys.argv) == 4
-  rule = build_svg()
-  rule.build(*sys.argv[1:])
+    assert len(sys.argv) == 4
+    rule = build_svg()
+    rule.build(*sys.argv[1:])
 
 # Exit fast:
-performance.exit()
+performance.exit(sys.argv[2])

doc/user_guide/user_guide.tex

@@ -445,6 +445,7 @@ \subsubsection{Using Redo} \label{Using Redo}
 redo all
 redo clean
 redo clean_all
+redo clear_cache
 redo templates
 redo publish
 redo targets
@@ -473,6 +474,7 @@ \subsubsection{Using Redo} \label{Using Redo}
   \item \textbf{\texttt{redo recursive}} - runs \texttt{redo all} from this directory and all below it
   \item \textbf{\texttt{redo clean}} - removes all \textit{build/} directories from this directory and all below it
   \item \textbf{\texttt{redo clean\_all}} - removes all \textit{build/} directories from this entire repository
+  \item \textbf{\texttt{redo clear\_cache}} - remove the model cache that the build system uses to increase performance, see Section \ref{The Model Cache}
   \item \textbf{\texttt{redo templates}} - finds and builds all autocoded templates (in \textit{build/template/}) that can be built from this directory
   \item \textbf{\texttt{redo test\_all}} - finds and runs all unit tests from this directory an all below it. Note: Add a \textit{.skip\_test} file to a directory to exclude it from being run by \texttt{redo test\_all}.
   \item \textbf{\texttt{redo coverage\_all}} - finds and runs all unit tests from this directory an all below it and generates coverage reports for each. Note: Add a \textit{.skip\_test} or \textit{.skip\_coverage} file to a directory to exclude it from being run by \texttt{redo coverage\_all}.
@@ -532,12 +534,13 @@ \subsubsection{Using Redo} \label{Using Redo}
 \end{minted}
 \vspace{5mm} %5mm vertical space
 
-Very rarely, \texttt{redo} can get itself into an unknown state. If things are not behaving correctly you can wipe out the redo database, and force things to rebuild fresh, which often fixes any issues that arise. The best procedure to do this is:
+Very rarely, \texttt{redo} can get itself into an unknown state. If things are not behaving correctly you can wipe out the redo database, and force things to rebuild fresh, which often fixes any issues that arise.To be safe, you can also remove the model cache that the build system uses to increase performance. The best procedure to do this is:
 
 \vspace{5mm} %5mm vertical space
 \begin{minted}{text}
 > rm -rf ~/.redo      # remove the redo database
 > redo clean_all      # clean the entire repository
+> redo clear_cache    # remove the model cache
 > redo <your_command> # retry the redo command that was not working
 \end{minted}
 \vspace{5mm} %5mm vertical space
@@ -6714,6 +6717,20 @@ \subsubsection{Extending the Build System} \label{Extending the Build System}
 
 Note that adding some build rules, such as compiling for a new programming language, will require more work than the procedure presents above. In that case, it is recommended to understand the \textit{set up} and \textit{database} portions of the build system located in \textit{redo/database}.
 
+\subsubsection{Model Caching} \label{The Model Cache}
+
+When building a target, the Adamant build system spends a lot of time reading YAML models from the file system, validating them, and then using them to generate outputs. Since most of these models do not change often on disk, the build system caches them in a database in \textit{/temp} to speed up build times. It is important when designing python models to correctly track the dependencies so that cached entries can be refreshed when a YAML model, or a YAML model's dependency, has changed. To do this, ensure that your python model correctly implements the \texttt{get\_dependencies} method, which should return a list of all YAML files that the current YAML file depends on. For example, a \texttt{*.component.yaml} always depends on its IDed entity models: \texttt{*.data\_products.yaml}, \texttt{*.data\_commands.yaml}, etc. \\
+
+If you suspect that the cache is messed up, or you want to ensure a completely clean build, you can clear the cache by running:
+
+\vspace{5mm} %5mm vertical space
+\begin{minted}{text}
+> redo clear_cache
+\end{minted}
+\vspace{5mm} %5mm vertical space
+
+which will remove the entire cache database from the system. It will be recreated next time you build a new target.
+
 \subsection{The Generator System} \label{Adding Generators}
 
 The primary generator/autocoding system for Adamant is contained within the \texttt{gen/} directory. Most generators follow a similar pattern. The input is usually a human written and human readable \href{http://yaml.org}{\textcolor{blue}{YAML}} file which is first validated by a schema and then ingested into a python data structure called a model. These models are then output into many different autogenerated text files using the \href{http://jinja.pocoo.org}{\textcolor{blue}{Jinja2}} templating engine. Modifying existing generators or adding your own is intended to be a straight forward process, however some investigation will be required on the reader's part, as the minute details cannot be fully documented here.

gen/generators/basic.py

@@ -141,6 +141,7 @@ def __init__(
     ):
         # Set the generator model class and type name:
         self.model_cls = model_class
+        self._model_obj = {}
         self.model_type = self.model_cls.__name__.lower()
         self.has_dependencies = has_dependencies
         self.camel_case_filename = camel_case_filename
@@ -158,6 +159,12 @@ def __init__(
         self.extension = ext[1:]
         self.descriptor = name.split("name")[-1]
 
+    # Cache model object for speed:
+    def model_object(self, input_filename):
+        if input_filename not in self._model_obj:
+            self._model_obj[input_filename] = self.model_cls(input_filename)
+        return self._model_obj[input_filename]
+
     def input_file_regex(self):
         return r".*\." + self.model_type + r"\.yaml$"
 
@@ -235,7 +242,7 @@ def output_filename(self, input_filename):
         return dirname + os.sep + build_dir + os.sep + output_filename
 
     def _generate_output(self, input_filename, methods_to_call_on_model_obj=[]):
-        model_obj = self.model_cls(input_filename)
+        model_obj = self.model_object(input_filename)
 
         # Call any desired methods on the model object. This performance feature is used to
         # add functionality to some models that are needed by only some generators,
@@ -265,7 +272,7 @@ def generate(self, input_filename, methods_to_call_on_model_obj=[]):
     # Depend on the primary model dependencies:
     def depends_on(self, input_filename):
         if self.has_dependencies:
-            m = self.model_cls(input_filename)
+            m = self.model_object(input_filename)
             if hasattr(m, "get_dependencies"):
                 # import sys
                 # sys.stderr.write("depending on: " + str(m.get_dependencies()) + "\n")