PETSc fixes (#3884)

* Ravel swarm.getField values to work with updated petsc4py API
* Update some install dependencies to work with latest PETSc/SLEPc.

---------

Co-authored-by: Scott MacLachlan <[email protected]>

docker/Dockerfile.env

@@ -13,7 +13,7 @@ RUN apt-get update \
                  cmake gfortran git libopenblas-serial-dev \
                  libtool python3-dev python3-pip python3-tk python3-venv \
                  python3-requests zlib1g-dev libboost-dev sudo gmsh \
-                 bison flex ninja-build \
+                 bison flex ninja-build pkg-config \
                  libocct-ocaf-dev libocct-data-exchange-dev \
                  swig graphviz \
                  libcurl4-openssl-dev libxml2-dev \

docs/source/download.rst

@@ -135,6 +135,7 @@ they have the system dependencies:
 * zlib
 * flex, bison
 * Ninja
+* pkg-config
 
 Firedrake has been successfully installed on Windows 10 using the
 Windows Subsystem for Linux. There are more detailed instructions for

firedrake/mesh.py

@@ -1966,7 +1966,7 @@ def _renumber_entities(self, reorder):
         if reorder:
             swarm = self.topology_dm
             parent = self._parent_mesh.topology_dm
-            swarm_parent_cell_nums = swarm.getField("DMSwarm_cellid")
+            swarm_parent_cell_nums = swarm.getField("DMSwarm_cellid").ravel()
             parent_renum = self._parent_mesh._dm_renumbering.getIndices()
             pStart, _ = parent.getChart()
             parent_renum_inv = np.empty_like(parent_renum)
@@ -2063,7 +2063,7 @@ def cell_parent_cell_list(self):
         """Return a list of parent mesh cells numbers in vertex only
         mesh cell order.
         """
-        cell_parent_cell_list = np.copy(self.topology_dm.getField("parentcellnum"))
+        cell_parent_cell_list = np.copy(self.topology_dm.getField("parentcellnum").ravel())
         self.topology_dm.restoreField("parentcellnum")
         return cell_parent_cell_list[self.cell_closure[:, -1]]
 
@@ -2082,7 +2082,7 @@ def cell_parent_base_cell_list(self):
         """
         if not isinstance(self._parent_mesh, ExtrudedMeshTopology):
             raise AttributeError("Parent mesh is not extruded")
-        cell_parent_base_cell_list = np.copy(self.topology_dm.getField("parentcellbasenum"))
+        cell_parent_base_cell_list = np.copy(self.topology_dm.getField("parentcellbasenum").ravel())
         self.topology_dm.restoreField("parentcellbasenum")
         return cell_parent_base_cell_list[self.cell_closure[:, -1]]
 
@@ -2103,7 +2103,7 @@ def cell_parent_extrusion_height_list(self):
         """
         if not isinstance(self._parent_mesh, ExtrudedMeshTopology):
             raise AttributeError("Parent mesh is not extruded.")
-        cell_parent_extrusion_height_list = np.copy(self.topology_dm.getField("parentcellextrusionheight"))
+        cell_parent_extrusion_height_list = np.copy(self.topology_dm.getField("parentcellextrusionheight").ravel())
         self.topology_dm.restoreField("parentcellextrusionheight")
         return cell_parent_extrusion_height_list[self.cell_closure[:, -1]]
 
@@ -2123,7 +2123,7 @@ def mark_entities(self, tf, label_value, label_name=None):
     @utils.cached_property  # TODO: Recalculate if mesh moves
     def cell_global_index(self):
         """Return a list of unique cell IDs in vertex only mesh cell order."""
-        cell_global_index = np.copy(self.topology_dm.getField("globalindex"))
+        cell_global_index = np.copy(self.topology_dm.getField("globalindex").ravel())
         self.topology_dm.restoreField("globalindex")
         return cell_global_index
 
@@ -2158,8 +2158,8 @@ def _make_input_ordering_sf(swarm, nroots, ilocal):
         # ilocal = None -> leaves are swarm points [0, 1, 2, ...).
         # ilocal can also be Firedrake cell numbers.
         sf = PETSc.SF().create(comm=swarm.comm)
-        input_ranks = swarm.getField("inputrank")
-        input_indices = swarm.getField("inputindex")
+        input_ranks = swarm.getField("inputrank").ravel()
+        input_indices = swarm.getField("inputindex").ravel()
         nleaves = len(input_ranks)
         if ilocal is not None and nleaves != len(ilocal):
             swarm.restoreField("inputrank")
@@ -3652,7 +3652,7 @@ def _pic_swarm_in_mesh(
     # local_points[halo_indices] (it also updates local_points[~halo_indices]`, not changing any values there).
     # If some index of local_points_reduced corresponds to a missing point, local_points_reduced[index] is not updated
     # when we reduce and it does not update any leaf data, i.e., local_points, when we bcast.
-    owners = swarm.getField("DMSwarm_rank")
+    owners = swarm.getField("DMSwarm_rank").ravel()
     halo_indices, = np.where(owners != parent_mesh.comm.rank)
     halo_indices = halo_indices.astype(IntType)
     n = coords.shape[0]
@@ -3868,13 +3868,13 @@ def _dmswarm_create(
 
     # NOTE ensure that swarm.restoreField is called for each field too!
     swarm_coords = swarm.getField("DMSwarmPIC_coor").reshape((num_vertices, gdim))
-    swarm_parent_cell_nums = swarm.getField("DMSwarm_cellid")
-    field_parent_cell_nums = swarm.getField("parentcellnum")
+    swarm_parent_cell_nums = swarm.getField("DMSwarm_cellid").ravel()
+    field_parent_cell_nums = swarm.getField("parentcellnum").ravel()
     field_reference_coords = swarm.getField("refcoord").reshape((num_vertices, tdim))
-    field_global_index = swarm.getField("globalindex")
-    field_rank = swarm.getField("DMSwarm_rank")
-    field_input_rank = swarm.getField("inputrank")
-    field_input_index = swarm.getField("inputindex")
+    field_global_index = swarm.getField("globalindex").ravel()
+    field_rank = swarm.getField("DMSwarm_rank").ravel()
+    field_input_rank = swarm.getField("inputrank").ravel()
+    field_input_index = swarm.getField("inputindex").ravel()
     swarm_coords[...] = coords
     swarm_parent_cell_nums[...] = plex_parent_cell_nums
     field_parent_cell_nums[...] = parent_cell_nums
@@ -3895,8 +3895,8 @@ def _dmswarm_create(
     swarm.restoreField("DMSwarm_cellid")
 
     if extruded:
-        field_base_parent_cell_nums = swarm.getField("parentcellbasenum")
-        field_extrusion_heights = swarm.getField("parentcellextrusionheight")
+        field_base_parent_cell_nums = swarm.getField("parentcellbasenum").ravel()
+        field_extrusion_heights = swarm.getField("parentcellextrusionheight").ravel()
         field_base_parent_cell_nums[...] = base_parent_cell_nums
         field_extrusion_heights[...] = extrusion_heights
         swarm.restoreField("parentcellbasenum")

scripts/firedrake-install

@@ -742,7 +742,7 @@ def get_petsc_options(minimal=False):
         petsc_options.add("--download-suitesparse")
         petsc_options.add("--download-pastix")
         # Required by pastix
-        petsc_options.add("--download-hwloc")
+        petsc_options.update({"--download-hwloc", "--download-netlib-lapack", "--with-netlib-lapack-c-bindings"})
         if osname == "Darwin":
             petsc_options.add("--download-hwloc-configure-arguments=--disable-opencl")
         # Serial mesh partitioner
@@ -1517,6 +1517,7 @@ if mode == "install" or not args.update_script:
     if osname == "Darwin":
         package_manager = "brew"
         required_packages = ["gcc",
+                             "pkg-config",
                              "autoconf",
                              "make",
                              "automake",
@@ -1535,6 +1536,7 @@ if mode == "install" or not args.update_script:
                              "flex",   # for ptscotch
                              "cmake",
                              "gfortran",
+                             "pkg-config",
                              "git",
                              "libcurl4-openssl-dev",
                              "pkgconf",  # for p4est

tests/firedrake/vertexonly/test_swarm.py

@@ -208,7 +208,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
         exclude_halos = True
 
     # Get point coords on current MPI rank
-    localpointcoords = np.copy(swarm.getField("DMSwarmPIC_coor"))
+    localpointcoords = np.copy(swarm.getField("DMSwarmPIC_coor").ravel())
     swarm.restoreField("DMSwarmPIC_coor")
     if len(inputpointcoords.shape) > 1:
         localpointcoords = np.reshape(localpointcoords, (-1, inputpointcoords.shape[1]))
@@ -218,11 +218,11 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
     nptslocal = len(localpointcoords)
     nptsglobal = MPI.COMM_WORLD.allreduce(nptslocal, op=MPI.SUM)
     # Get parent PETSc cell indices on current MPI rank
-    localparentcellindices = np.copy(swarm.getField("DMSwarm_cellid"))
+    localparentcellindices = np.copy(swarm.getField("DMSwarm_cellid").ravel())
     swarm.restoreField("DMSwarm_cellid")
 
     # also get the global coordinate numbering
-    globalindices = np.copy(swarm.getField("globalindex"))
+    globalindices = np.copy(swarm.getField("globalindex").ravel())
     swarm.restoreField("globalindex")
 
     # Tests
@@ -233,7 +233,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
 
     # get custom fields on swarm - will fail if didn't get created
     for name, size, dtype in other_fields:
-        f = swarm.getField(name)
+        f = swarm.getField(name).ravel()
         assert len(f) == size*nptslocal
         assert f.dtype == dtype
         swarm.restoreField(name)
@@ -330,7 +330,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
     # Check that the rank numbering is correct. Since we know all points are at
     # the midpoints of cells, there should be no disagreement about cell
     # ownership and the voting algorithm should have no effect.
-    owned_ranks = np.copy(swarm.getField("DMSwarm_rank"))
+    owned_ranks = np.copy(swarm.getField("DMSwarm_rank").ravel())
     swarm.restoreField("DMSwarm_rank")
     if exclude_halos:
         assert np.array_equal(owned_ranks, inputlocalpointcoordranks)
@@ -339,7 +339,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
         assert np.all(np.isin(inputlocalpointcoordranks, owned_ranks))
 
     # check that the input rank is correct
-    input_ranks = np.copy(swarm.getField("inputrank"))
+    input_ranks = np.copy(swarm.getField("inputrank").ravel())
     swarm.restoreField("inputrank")
     if exclude_halos:
         assert np.all(input_ranks == input_rank)
@@ -351,7 +351,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
         assert np.all(input_ranks < parentmesh.comm.size)
 
     # check that the input index is correct
-    input_indices = np.copy(swarm.getField("inputindex"))
+    input_indices = np.copy(swarm.getField("inputindex").ravel())
     swarm.restoreField("inputindex")
     if exclude_halos:
         assert np.array_equal(input_indices, input_local_coord_indices)
@@ -365,7 +365,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
 
     # check we have unique parent cell numbers, which we should since we have
     # points at cell midpoints
-    parentcellnums = np.copy(swarm.getField("parentcellnum"))
+    parentcellnums = np.copy(swarm.getField("parentcellnum").ravel())
     swarm.restoreField("parentcellnum")
     assert len(np.unique(parentcellnums)) == len(parentcellnums)
 
@@ -378,7 +378,7 @@ def test_pic_swarm_in_mesh(parentmesh, redundant, exclude_halos):
     ):
         swarm.setPointCoordinates(localpointcoords, redundant=False,
                                   mode=PETSc.InsertMode.INSERT_VALUES)
-        petsclocalparentcellindices = np.copy(swarm.getField("DMSwarm_cellid"))
+        petsclocalparentcellindices = np.copy(swarm.getField("DMSwarm_cellid").ravel())
         swarm.restoreField("DMSwarm_cellid")
         if exclude_halos:
             assert np.all(petsclocalparentcellindices == localparentcellindices)

tests/firedrake/vertexonly/test_vertex_only_fs.py

@@ -116,7 +116,7 @@ def functionspace_tests(vm):
     h.dat.data_wo_with_halos[:] = -1
     h.interpolate(g)
     # Exclude points which we know are missing - these should all be equal to -1
-    input_ordering_parent_cell_nums = vm.input_ordering.topology_dm.getField("parentcellnum")
+    input_ordering_parent_cell_nums = vm.input_ordering.topology_dm.getField("parentcellnum").ravel()
     vm.input_ordering.topology_dm.restoreField("parentcellnum")
     idxs_to_include = input_ordering_parent_cell_nums != -1
     assert np.allclose(h.dat.data_ro_with_halos[idxs_to_include], np.prod(vm.input_ordering.coordinates.dat.data_ro_with_halos[idxs_to_include].reshape(-1, vm.input_ordering.geometric_dimension()), axis=1))
@@ -221,7 +221,7 @@ def vectorfunctionspace_tests(vm):
     h.dat.data_wo_with_halos[:] = -1
     h.interpolate(g)
     # Exclude points which we know are missing - these should all be equal to -1
-    input_ordering_parent_cell_nums = vm.input_ordering.topology_dm.getField("parentcellnum")
+    input_ordering_parent_cell_nums = vm.input_ordering.topology_dm.getField("parentcellnum").ravel()
     vm.input_ordering.topology_dm.restoreField("parentcellnum")
     idxs_to_include = input_ordering_parent_cell_nums != -1
     assert np.allclose(h.dat.data_ro[idxs_to_include], 2*vm.input_ordering.coordinates.dat.data_ro_with_halos[idxs_to_include])

tests/firedrake/vertexonly/test_vertex_only_mesh_generation.py

@@ -199,7 +199,7 @@ def verify_vertexonly_mesh(m, vm, inputvertexcoords, name):
     # Correct parent cell numbers
     stored_vertex_coords = np.copy(vm.topology_dm.getField("DMSwarmPIC_coor")).reshape((vm.num_cells(), gdim))
     vm.topology_dm.restoreField("DMSwarmPIC_coor")
-    stored_parent_cell_nums = np.copy(vm.topology_dm.getField("parentcellnum"))
+    stored_parent_cell_nums = np.copy(vm.topology_dm.getField("parentcellnum").ravel())
     vm.topology_dm.restoreField("parentcellnum")
     assert len(stored_vertex_coords) == len(stored_parent_cell_nums)
     if MPI.COMM_WORLD.size == 1: