Merge pull request #450 from stfc/448_make_public_example

448 make public example

.github/workflows/unit-tests.yml

@@ -84,6 +84,9 @@ jobs:
         # fparser should work even under limited terminal conditions so set
         # LC_ALL (this is only relevant for versions before Python 3.7).
         LC_ALL=POSIX pytest --cov=fparser --cov-report=xml src/fparser
+    - name: Test examples
+      run: |
+        make -C example test
     - name: Upload coverage to Codecov with GitHub Action
       uses: codecov/codecov-action@v4
       env:

CHANGELOG.md

@@ -18,6 +18,9 @@ Modifications by (in alphabetical order):
 * P. Vitt, University of Siegen, Germany
 * A. Voysey, UK Met Office
 
+11/10/2024 PR #450 for #448. Adds an example script for removing all protected/private
+           attributes from a parse tree.
+
 15/07/2024 PR #438 for #437. Fix type guard statement bug. 
 
 24/04/2024 PR #444 for #443. Adds an option to the reader to handle code

doc/source/conf.py

@@ -10,7 +10,7 @@
 
 project = 'fparser'
 copyright = '2017-2024, Science and Technology Facilities Council'
-author = 'Andrew Porter, Rupert Ford, Balthasar Reuter and Pearu Peterson'
+author = 'Andrew Porter, Rupert Ford, Balthasar Reuter, Joerg Henrichs and Pearu Peterson'
 
 version = fparser._get_version()
 release = fparser._get_version()
@@ -55,3 +55,26 @@
 
 # Generate the Doxygen documentation
 subprocess.call('cd ..; doxygen doxygen.config', shell=True)
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    # 'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author,
+#  documentclass [howto/manual]).
+latex_documents = [
+    ('index', 'fparser.tex', 'fparser Documentation',
+     'Andrew Porter, Rupert Ford, Balthasar Reuter, \\\\ '
+     'Joerg Henrichs and Pearu Peterson', 'manual'),
+]

doc/source/examples.rst

@@ -0,0 +1,149 @@
+.. -*- rest -*-
+
+..
+    Copyright (c) 2024 Science and Technology Facilities Council.
+
+    All rights reserved.
+
+    Modifications made as part of the fparser project are distributed
+    under the following license:
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+    1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+.. _examples:
+
+Examples
+========
+
+The distribution comes with a set of examples showing how fparser can
+be used. At the same time, some of these examples are actually useful tools
+that are used in other projects.
+
+
+fparser2_f2008.py
+^^^^^^^^^^^^^^^^^
+This is a very small example code that shows how a Fortran code, given as
+a string, is parsed, and then converted back into Fortran. It just prints
+out the re-created Fortran source code.
+
+
+create_dependencies.py
+^^^^^^^^^^^^^^^^^^^^^^
+This file analyses the dependencies between a set of Fortran files, based on
+the ``Use`` statements in each file. It assumes that the module name in the
+``use`` statement corresponds to the name of the file (adding one of
+.F90/.f90/.x90). Only files in the current directory will be tested, so
+external dependencies will not be listed. Its output
+is in a format that can be immediately used in a Makefile::
+
+    ../create_dependencies.py  *.f90
+    b.o: a.o
+    c.o: a.o b.o
+
+
+Ignoring error handling, the simplified main part of this code
+that is related to fparser is::
+
+    reader = FortranFileReader(filename)
+    parser = ParserFactory().create(std="f2003")
+    parse_tree = parser(reader)
+
+    # Collect all used modules in a list
+    all_use = set()
+    for node in walk(parse_tree, Use_Stmt):
+        use_name = str(node.items[2])
+        # A more sophisticated mapping could be used here.
+        # But for now just assume that the name in the use statement
+        # with an added ".o" is the required object file:
+        obj_dependency = use_name + ".o"
+        all_use.add(use_name)
+    # Now ``all_use`` contains all .o files that ``filename`` depends on
+
+The rest of the example is related to creating the proper format for
+a Makefile.
+
+.. note:: It would be straight-forward to loop over all files twice, first
+          to collect the module names provided by each file, then use this
+          information
+
+
+make_public.py
+^^^^^^^^^^^^^^
+This example removes all ``private`` and ``protected`` attributes in any
+declaration. In general these attributes are important and should
+obviously not be removed in order to give the compiler more information
+about intended use of the variables. But `PSyclone 
+<https://github.com/stfc/PSyclone>`_ offers a feature called
+Kernel Extraction, which automatically writes all variables read and written
+in a code section to a file. It also then creates a stand-alone driver program
+that will read this file, execute the kernel, and compare the results with
+the original results.
+
+Since PSyclone will follow the call tree, the code must be able to read even
+variables declared as ``private`` (to write them into the output file), and
+a driver program must be able to modify ``private`` and ``protected``
+variables in modules. If the driver creation is used, the 
+`Fab <https://github.com/MetOffice/fab>`_ based build system will
+remove all ``private`` and ``protected`` attributes in a separate build phase,
+so that the kernel extraction and driver creation works as expected.
+
+A short example, which shows how a ``Access_Stmt`` like ``private :: a`` is
+removed::
+
+    for node in walk(parse_tree, Access_Stmt):
+        if node.items[0] == "PRIVATE":
+            # Find the node in the parent, and remove it:
+            node.parent.children.remove(node)
+
+
+Modifying some of the fparser data structures can be more difficult, since
+they are often based on Python tuples, which cannot be modified. The following
+example from ``make_public.py`` shows how the middle element of a three-element
+tuple is replaced with None::
+
+    type_decl.items = (type_decl.items[0], None, type_decl.items[2])
+
+
+split_file.py
+~~~~~~~~~~~~~
+This script splits one Fortran source file into several files, each containing
+one top level module, subroutine, function or program. Each file uses the name
+of the program unit (module-, subroutine-, function-, program name). The
+extension will be ``.F90`` if there are preprocessor directives in the file,
+and ``.f90`` otherwise.
+
+Additionally, ``split_file.py`` will create a Makefile to build either the
+binary (if a program is found in the file), or all object files. If any of
+the environment variables ``F90``, ``F90FLAGS``, and ``LDFLAGS`` are set at
+run time of the script, it will use these values as default values in the
+makefile. But by setting these environment variables when running ``make``,
+these defaults can always be overwritten. The Makefile also has a ``clean``
+target, which will remove all ``.mod``, object, and the program file (if
+available). It uses the ``create_dependencies.py`` script to add the
+required dependencies to the Makefile.

doc/source/index.rst

@@ -79,5 +79,6 @@ Welcome to fparser's documentation!
    introduction
    fparser
    fparser2
+   examples
    developers_guide
    reference_guide

doc/source/license.rst

@@ -5,8 +5,9 @@ License
 =======
 
 | Modified work Copyright (c) 2017-2021 Science and Technology Facilities Council
-| Authors: **Andrew Porter** and **Rupert Ford**, STFC Daresbury Laboratory, and
-|          **Balthasar Reuter**, ECMWF
+| Authors: **Andrew Porter** and **Rupert Ford**, STFC Daresbury Laboratory,
+|          **Balthasar Reuter**, ECMWF, and
+|          **Joerg Henrichs**, Bureau of Meteorology
 | Original work Copyright (c) 1999-2008 **Pearu Peterson**
 | All rights reserved.
 

example/Makefile

@@ -33,17 +33,25 @@
 # ------------------------------------------------------------------------------
 # Author: J. Henrichs, Bureau of Meteorology
 
-.PHONY: test create_dependency fparser2_f2008
+# A simple Makefile driver to test the various examples.
 
-test:  create_dependencies split_file fparser2_f2008
+
+.PHONY: test create_dependency fparser2_f2008 make_public split_file
+
+test:  create_dependencies fparser2_f2008 make_public split_file
 
 create_dependencies:
-	(cd test_files/create_dependencies; ../../create_dependencies.py *.f90)
+	(cd test_files/create_dependencies; \
+	 ../../create_dependencies.py *.f90 | diff correct_dependencies -)
+
+fparser2_f2008:
+	python ./fparser2_f2008.py
+
+make_public:
+	./make_public.py test_files/make_public.f90 | diff test_files/make_public_correct.f90 -
 
 split_file:
 	(cd test_files/split_file; \
 		rm -f Makefile func.f90 mod1.f90 sub.f90 test_prog.f90; \
-		../../split_file.py test.f90)
-
-fparser2_f2008:
-	python ./fparser2_f2008.py
+		../../split_file.py test.f90; \
+		cat mod1.f90 sub.f90 func.f90 test_prog.f90 | diff -B test.f90 -)

example/make_public.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2024, Science and Technology Facilities Council.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ------------------------------------------------------------------------------
+# Author: Joerg Henrichs, Bureau of Meteorology
+
+"""This file contains a script that will remove any private or protected
+declaration in a Fortran file. This is used by PSyclone's kernel extraction
+and driver creation feature, see
+https://psyclone.readthedocs.io/en/latest/psyke.html.
+for details.
+
+Usage:  make_public.py  file1.f90
+"""
+
+import sys
+
+from fparser.common.readfortran import FortranFileReader
+from fparser.two.Fortran2003 import (
+    Access_Stmt,
+    Access_Spec,
+    Attr_Spec,
+    Binding_Private_Stmt,
+    Private_Components_Stmt,
+    Protected_Stmt,
+)
+from fparser.two.parser import ParserFactory
+from fparser.two.utils import walk
+
+
+# -----------------------------------------------------------------------------
+def remove_private(filename):
+    """Simple function that removes all private and protected declarations.
+    :param str filename: the file in which to remove private and protected
+    """
+    reader = FortranFileReader(filename)
+    parser = ParserFactory().create(std="f2008")
+    parse_tree = parser(reader)
+    # A useful print to see the actual rules
+    # print(repr(parse_tree))
+
+    # Loop over all access and protected statements. Note that a
+    # `protected_stmt` is not an access statement, so it needs to
+    # be listed additionally:
+    for node in walk(
+        parse_tree,
+        (Access_Stmt, Protected_Stmt, Private_Components_Stmt, Binding_Private_Stmt),
+    ):
+        # A Private_Components_Stms has no items:
+        if isinstance(
+            node, (Binding_Private_Stmt, Private_Components_Stmt)
+        ) or node.items[0] in ["PRIVATE", "PROTECTED"]:
+            # Find the node in the parent, and remove it:
+            node.parent.children.remove(node)
+
+    for node in walk(parse_tree, Access_Spec):
+        if str(node) == "PRIVATE":
+            node.string = "PUBLIC"
+
+    all_nodes = list(walk(parse_tree, Attr_Spec))
+    for node in all_nodes:
+        if str(node) == "PROTECTED":
+            # This is a tuple, so we can't simply remove the attribute
+            node.parent.items = tuple(i for i in node.parent.items if i is not node)
+            # If all items in the Attr_Spec are removed, we need to replace the
+            # Attr_Spec in the parent-parent (Type_Declaration) with None,
+            # otherwise fparser will create e.g. `real, :: a`
+            if len(node.parent.items) == 0:
+                # Again all tuples, which we can't modify, so we need to
+                # recreate the tuple but replace the attr_spec with None
+                type_decl = node.parent.parent
+                type_decl.items = (type_decl.items[0], None, type_decl.items[2])
+
+    return parse_tree
+
+
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    user_filename = sys.argv[1]
+    modified_parse_tree = remove_private(user_filename)
+    print(modified_parse_tree)

example/split_file.py

@@ -192,6 +192,7 @@ def main():
             sys.exit(-1)
         with open(filename, mode="w", encoding="utf-8") as f_out:
             f_out.write(str(unit))
+            f_out.write("\n")
         all_filenames.append(filename)
         all_objs.append(f"{unit_name}.o")
 

example/test_files/create_dependencies/correct_dependencies

@@ -0,0 +1,2 @@
+b.o: a.o
+c.o: a.o b.o