diff --git a/packages/panzer/adapters-stk/test/evaluator_tests/CMakeLists.txt b/packages/panzer/adapters-stk/test/evaluator_tests/CMakeLists.txt index 3b2202563a32..d871d0375cb0 100644 --- a/packages/panzer/adapters-stk/test/evaluator_tests/CMakeLists.txt +++ b/packages/panzer/adapters-stk/test/evaluator_tests/CMakeLists.txt @@ -43,6 +43,13 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NUM_MPI_PROCS 2 ) +TRIBITS_ADD_EXECUTABLE_AND_TEST( + tGatherSolution_BlockedTpetra + SOURCES tpetra_blocked_gather_solution.cpp ${UNIT_TEST_DRIVER} + COMM serial mpi + NUM_MPI_PROCS 2 + ) + TRIBITS_ADD_EXECUTABLE_AND_TEST( tScatterResidual_Tpetra SOURCES tpetra_scatter_residual.cpp ${UNIT_TEST_DRIVER} diff --git a/packages/panzer/adapters-stk/test/evaluator_tests/tpetra_blocked_gather_solution.cpp b/packages/panzer/adapters-stk/test/evaluator_tests/tpetra_blocked_gather_solution.cpp new file mode 100644 index 000000000000..279956f8d6eb --- /dev/null +++ b/packages/panzer/adapters-stk/test/evaluator_tests/tpetra_blocked_gather_solution.cpp @@ -0,0 +1,721 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +/////////////////////////////////////////////////////////////////////////////// +// +// Include Files +// +/////////////////////////////////////////////////////////////////////////////// + +// C++ +#include +#include +#include + +// Kokkos +#include "Kokkos_View_Fad.hpp" + +// Panzer +#include "PanzerAdaptersSTK_config.hpp" +#include "Panzer_BasisIRLayout.hpp" +#include "Panzer_BlockedTpetraLinearObjFactory.hpp" +#include "Panzer_BlockedDOFManager.hpp" +#include "Panzer_DOFManager.hpp" +#include "Panzer_Evaluator_WithBaseImpl.hpp" +#include "Panzer_FieldManagerBuilder.hpp" +#include "Panzer_GatherOrientation.hpp" +#include "Panzer_PureBasis.hpp" +#include "Panzer_STKConnManager.hpp" +#include "Panzer_STK_Interface.hpp" +#include "Panzer_STK_SetupUtilities.hpp" +#include "Panzer_STK_SquareQuadMeshFactory.hpp" +#include "Panzer_STK_Version.hpp" +#include "Panzer_Workset.hpp" +#include "Panzer_LOCPair_GlobalEvaluationData.hpp" +#include "Panzer_GlobalEvaluationDataContainer.hpp" + +// Teuchos +#include "Teuchos_DefaultMpiComm.hpp" +#include "Teuchos_GlobalMPISession.hpp" +#include "Teuchos_OpaqueWrapper.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "Teuchos_UnitTestHarness.hpp" + +// Thyra +#include "Thyra_ProductVectorBase.hpp" +#include "Thyra_VectorStdOps.hpp" + +// Tpetra +#include "Tpetra_Vector.hpp" + +// user_app +#include "user_app_EquationSetFactory.hpp" + +typedef double ScalarT; +using LocalOrdinalT = panzer::LocalOrdinal; +using GlobalOrdinalT = panzer::GlobalOrdinal; + +typedef Tpetra::Vector VectorType; +typedef Tpetra::Operator OperatorType; +typedef Tpetra::CrsMatrix CrsMatrixType; +typedef Tpetra::CrsGraph CrsGraphType; +typedef Tpetra::Map MapType; +typedef Tpetra::Import ImportType; +typedef Tpetra::Export ExportType; + +typedef Thyra::TpetraLinearOp ThyraLinearOp; + +typedef panzer::BlockedTpetraLinearObjFactory BlockedTpetraLinObjFactoryType; +typedef panzer::TpetraLinearObjFactory TpetraLinObjFactoryType; +typedef panzer::BlockedTpetraLinearObjContainer BlockedTpetraLinObjContainerType; +typedef panzer::TpetraLinearObjContainer TpetraLinObjContainerType; + +namespace panzer +{ + + Teuchos::RCP buildBasis(std::size_t worksetSize, const std::string &basisName); + void testInitialization(const Teuchos::RCP &ipb); + Teuchos::RCP buildMesh(int elemX, int elemY); + void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream &out, bool &success); + + // Test without tangent fields in gather evaluator + TEUCHOS_UNIT_TEST(tpetra_assembly, gather_solution_no_tangents) + { + testGatherScatter(false, out, success); + } + + // Test with tangent fields in gather evaluator + TEUCHOS_UNIT_TEST(tpetra_assembly, gather_solution_tangents) + { + testGatherScatter(true, out, success); + } + + // enable_tangents determines whether tangent fields dx/dp are added to gather evaluator. + // These are used when computing df/dx*dx/dp with the tangent evaluation type + void testGatherScatter(const bool enable_tangents, Teuchos::FancyOStream &out, bool &success) + { +#ifdef HAVE_MPI + Teuchos::RCP> tComm = Teuchos::rcp(new Teuchos::MpiComm(MPI_COMM_WORLD)); +#else + Teuchos::RCP> tComm = Teuchos::rcp(new Teuchos::SerialComm(MPI_COMM_WORLD)); +#endif + + int myRank = tComm->getRank(); + int numProcs = tComm->getSize(); + + const std::size_t workset_size = 4 / numProcs; + const std::string fieldName1_q1 = "U"; + const std::string fieldName2_q1 = "V"; + const std::string fieldName_qedge1 = "B"; + const int num_tangent = enable_tangents ? 5 : 0; + + Teuchos::RCP mesh = buildMesh(2, 2); + + // build input physics block + Teuchos::RCP basis_q1 = buildBasis(workset_size, "Q1"); + Teuchos::RCP basis_qedge1 = buildBasis(workset_size, "QEdge1"); + + Teuchos::RCP ipb = Teuchos::parameterList(); + testInitialization(ipb); + + const int default_int_order = 1; + std::string eBlockID = "eblock-0_0"; + Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); + panzer::CellData cellData(workset_size, mesh->getCellTopology("eblock-0_0")); + Teuchos::RCP gd = panzer::createGlobalData(); + Teuchos::RCP physicsBlock = + Teuchos::rcp(new PhysicsBlock(ipb, eBlockID, default_int_order, cellData, eqset_factory, gd, false)); + + Teuchos::RCP> work_sets = panzer_stk::buildWorksets(*mesh, physicsBlock->elementBlockID(), + physicsBlock->getWorksetNeeds()); + TEST_EQUALITY(work_sets->size(), 1); + + // build connection manager and field manager + const Teuchos::RCP conn_manager = Teuchos::rcp(new panzer_stk::STKConnManager(mesh)); + Teuchos::RCP blocked_dofManager = Teuchos::rcp(new panzer::BlockedDOFManager(conn_manager, MPI_COMM_WORLD)); + + blocked_dofManager->addField(fieldName1_q1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); + blocked_dofManager->addField(fieldName2_q1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); + blocked_dofManager->addField(fieldName_qedge1, Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_qedge1->getIntrepid2Basis()))); + + std::vector > fieldOrder(3); + fieldOrder[0].push_back(fieldName1_q1); + fieldOrder[1].push_back(fieldName_qedge1); + fieldOrder[2].push_back(fieldName2_q1); + blocked_dofManager->setFieldOrder(fieldOrder); + + blocked_dofManager->buildGlobalUnknowns(); + + // setup linear object factory + ///////////////////////////////////////////////////////////// + + Teuchos::RCP t_lof = Teuchos::rcp(new BlockedTpetraLinObjFactoryType(tComm.getConst(), blocked_dofManager)); + Teuchos::RCP> lof = t_lof; + Teuchos::RCP loc = t_lof->buildGhostedLinearObjContainer(); + t_lof->initializeGhostedContainer(LinearObjContainer::X, *loc); + loc->initialize(); + + Teuchos::RCP t_loc = Teuchos::rcp_dynamic_cast(loc); + Teuchos::RCP> x_vec = t_loc->get_x_th(); + Thyra::assign(x_vec.ptr(), 123.0 + myRank); + + // need a place to evaluate the tangent fields, so we create a + // unblocked DOFManager and LOF and set up if needed + std::vector> tangentContainers; + Teuchos::RCP dofManager = Teuchos::rcp(new panzer::DOFManager(conn_manager, MPI_COMM_WORLD)); + Teuchos::RCP tangent_lof = Teuchos::rcp(new TpetraLinObjFactoryType(tComm.getConst(), dofManager)); + Teuchos::RCP> parent_tangent_lof = tangent_lof; + + if (enable_tangents) + { + using Teuchos::RCP; + using Teuchos::rcp; + using Teuchos::rcp_dynamic_cast; + using Thyra::ProductVectorBase; + using LOCPair = panzer::LOCPair_GlobalEvaluationData; + + std::vector tangent_fieldOrder; + for (int i(0); i < num_tangent; ++i) { + std::stringstream ssedge; + ssedge << fieldName_qedge1 << " Tangent " << i; + std::stringstream ss1, ss2; + ss1 << fieldName1_q1 << " Tangent " << i; + ss2 << fieldName2_q1 << " Tangent " << i; + + dofManager->addField(ss1.str(), Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); + dofManager->addField(ss2.str(), Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_q1->getIntrepid2Basis()))); + dofManager->addField(ssedge.str(), Teuchos::rcp(new panzer::Intrepid2FieldPattern(basis_qedge1->getIntrepid2Basis()))); + tangent_fieldOrder.push_back(ss1.str()); + tangent_fieldOrder.push_back(ss2.str()); + tangent_fieldOrder.push_back(ssedge.str()); + } + dofManager->setFieldOrder(tangent_fieldOrder); + dofManager->buildGlobalUnknowns(); + + // generate and evaluate some fields + Teuchos::RCP tangent_loc = tangent_lof->buildGhostedLinearObjContainer(); + tangent_lof->initializeGhostedContainer(LinearObjContainer::X, *tangent_loc); + tangent_loc->initialize(); + + for (int i(0); i < num_tangent; ++i) + { + auto locPair = Teuchos::rcp(new LOCPair(tangent_lof, panzer::LinearObjContainer::X)); + + auto global_t_loc = rcp_dynamic_cast(locPair->getGlobalLOC()); + Teuchos::RCP> global_x_vec = global_t_loc->get_x_th(); + Thyra::assign(global_x_vec.ptr(), 0.123 + myRank + i); + + auto ghosted_t_loc = rcp_dynamic_cast(locPair->getGhostedLOC()); + Teuchos::RCP> ghosted_x_vec = ghosted_t_loc->get_x_th(); + Thyra::assign(ghosted_x_vec.ptr(), 0.123 + myRank + i); + + tangentContainers.push_back(locPair); + } // end loop over the tangents + } // end if (enable_tangents) + + // setup field manager, add evaluator under test + ///////////////////////////////////////////////////////////// + + PHX::FieldManager fm; + + std::vector derivative_dimensions; + derivative_dimensions.push_back(12); + fm.setKokkosExtendedDataTypeDimensions(derivative_dimensions); + + std::vector tan_derivative_dimensions; + if (enable_tangents) + tan_derivative_dimensions.push_back(num_tangent); + else + tan_derivative_dimensions.push_back(0); + fm.setKokkosExtendedDataTypeDimensions(tan_derivative_dimensions); + + Teuchos::RCP evalField_q1, evalField_qedge1; + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName1_q1); + names->push_back(fieldName2_q1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_q1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName_qedge1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_qedge1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName1_q1); + names->push_back(fieldName2_q1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_q1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName_qedge1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_qedge1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName1_q1); + names->push_back(fieldName2_q1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_q1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + if (enable_tangents) + { + RCP>> tangent_names = + rcp(new std::vector>(2)); + for (int i = 0; i < num_tangent; ++i) + { + std::stringstream ss1, ss2; + ss1 << fieldName1_q1 << " Tangent " << i; + ss2 << fieldName2_q1 << " Tangent " << i; + (*tangent_names)[0].push_back(ss1.str()); + (*tangent_names)[1].push_back(ss2.str()); + } + pl.set("Tangent Names", tangent_names); + } + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + names->push_back(fieldName_qedge1); + + Teuchos::ParameterList pl; + pl.set("Basis", basis_qedge1); + pl.set("DOF Names", names); + pl.set("Indexer Names", names); + + if (enable_tangents) + { + RCP>> tangent_names = + rcp(new std::vector>(1)); + for (int i = 0; i < num_tangent; ++i) + { + std::stringstream ss; + ss << fieldName_qedge1 << " Tangent " << i; + (*tangent_names)[0].push_back(ss.str()); + } + pl.set("Tangent Names", tangent_names); + } + + Teuchos::RCP> evaluator = lof->buildGather(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); + + fm.registerEvaluator(evaluator); + fm.requireField(*evaluator->evaluatedFields()[0]); + } + + if (enable_tangents) + { + for (int i = 0; i < num_tangent; ++i) + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + RCP> tangent_names = rcp(new std::vector); + names->push_back(fieldName1_q1); + names->push_back(fieldName2_q1); + { + std::stringstream ss1, ss2; + ss1 << fieldName1_q1 << " Tangent " << i; + ss2 << fieldName2_q1 << " Tangent " << i; + tangent_names->push_back(ss1.str()); + tangent_names->push_back(ss2.str()); + } + + Teuchos::ParameterList pl; + pl.set("Basis", basis_q1); + pl.set("DOF Names", tangent_names); + pl.set("Indexer Names", tangent_names); + + { + std::stringstream ss; + ss << "Tangent Container " << i; + pl.set("Global Data Key", ss.str()); + } + + Teuchos::RCP> evaluator = + parent_tangent_lof->buildGatherTangent(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 2); + + fm.registerEvaluator(evaluator); + } + for (int i = 0; i < num_tangent; ++i) + { + using Teuchos::RCP; + using Teuchos::rcp; + RCP> names = rcp(new std::vector); + RCP> tangent_names = rcp(new std::vector); + names->push_back(fieldName_qedge1); + { + std::stringstream ss; + ss << fieldName_qedge1 << " Tangent " << i; + tangent_names->push_back(ss.str()); + } + + Teuchos::ParameterList pl; + pl.set("Basis", basis_qedge1); + pl.set("DOF Names", tangent_names); + pl.set("Indexer Names", tangent_names); + + { + std::stringstream ss; + ss << "Tangent Container " << i; + pl.set("Global Data Key", ss.str()); + } + + Teuchos::RCP> evaluator = + parent_tangent_lof->buildGatherTangent(pl); + + TEST_EQUALITY(evaluator->evaluatedFields().size(), 1); + + fm.registerEvaluator(evaluator); + } + } + + panzer::Traits::SD sd; + + panzer::Workset &workset = (*work_sets)[0]; + workset.alpha = 0.0; + workset.beta = 2.0; // derivatives multiplied by 2 + workset.time = 0.0; + workset.evaluate_transient_terms = false; + + sd.worksets_ = work_sets; + + fm.postRegistrationSetup(sd); + + panzer::Traits::PED ped; + ped.gedc->addDataObject("Solution Gather Container", loc); + if (enable_tangents) + { + for (int i(0); i < num_tangent; ++i) + { + std::stringstream ss; + ss << "Tangent Container " << i; + ped.gedc->addDataObject(ss.str(), tangentContainers[i]); + } + } + + fm.preEvaluate(ped); + fm.evaluateFields(workset); + fm.postEvaluate(0); + + fm.preEvaluate(ped); + fm.evaluateFields(workset); + fm.postEvaluate(0); + + fm.preEvaluate(ped); + fm.evaluateFields(workset); + fm.postEvaluate(0); + + // test Residual fields + { + PHX::MDField + fieldData1_q1(fieldName1_q1, basis_q1->functional); + PHX::MDField + fieldData2_q1(fieldName2_q1, basis_qedge1->functional); + + fm.getFieldData(fieldData1_q1); + fm.getFieldData(fieldData2_q1); + + TEST_EQUALITY(fieldData1_q1.extent(0), Teuchos::as(4 / numProcs)); + TEST_EQUALITY(fieldData1_q1.extent(1), 4); + TEST_EQUALITY(fieldData2_q1.extent(0), Teuchos::as(4 / numProcs)); + TEST_EQUALITY(fieldData2_q1.extent(1), 4); + TEST_EQUALITY(fieldData1_q1.size(), Teuchos::as(4 * 4 / numProcs)); + TEST_EQUALITY(fieldData2_q1.size(), Teuchos::as(4 * 4 / numProcs)); + + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); + + for (unsigned int i = 0; i < fieldData1_q1.extent(0); i++) + for (unsigned int j = 0; j < fieldData1_q1.extent(1); j++) + TEST_EQUALITY(fieldData1_q1_h(i, j), 123.0 + myRank); + + for (unsigned int i = 0; i < fieldData2_q1.extent(0); i++) + for (unsigned int j = 0; j < fieldData2_q1.extent(1); j++) + TEST_EQUALITY(fieldData2_q1_h(i, j), 123.0 + myRank); + } + { + PHX::MDField + fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); + + fm.getFieldData(fieldData_qedge1); + + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + + TEST_EQUALITY(fieldData_qedge1.extent(0), Teuchos::as(4 / numProcs)); + TEST_EQUALITY(fieldData_qedge1.extent(1), 4); + TEST_EQUALITY(fieldData_qedge1.size(), Teuchos::as(4 * 4 / numProcs)); + + for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) + for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); pt++) + TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); + } + + // test Jacobian fields + { + PHX::MDField + fieldData1_q1(fieldName1_q1, basis_q1->functional); + PHX::MDField + fieldData2_q1(fieldName2_q1, basis_qedge1->functional); + + fm.getFieldData(fieldData1_q1); + fm.getFieldData(fieldData2_q1); + + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); + + for (unsigned int cell = 0; cell < fieldData1_q1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData1_q1.extent(1); pt++) + { + TEST_EQUALITY(fieldData1_q1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), 12); + } + } + for (unsigned int cell = 0; cell < fieldData2_q1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData2_q1.extent(1); pt++) + { + TEST_EQUALITY(fieldData2_q1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), 12); + } + } + } + { + PHX::MDField + fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); + + fm.getFieldData(fieldData_qedge1); + + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + + for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); ++pt) + { + TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), 12); + } + } + } + + // test Tangent fields + { + PHX::MDField + fieldData1_q1(fieldName1_q1, basis_q1->functional); + PHX::MDField + fieldData2_q1(fieldName2_q1, basis_qedge1->functional); + + fm.getFieldData(fieldData1_q1); + fm.getFieldData(fieldData2_q1); + + auto fieldData1_q1_h = Kokkos::create_mirror_view(fieldData1_q1.get_static_view()); + auto fieldData2_q1_h = Kokkos::create_mirror_view(fieldData2_q1.get_static_view()); + Kokkos::deep_copy(fieldData1_q1_h, fieldData1_q1.get_static_view()); + Kokkos::deep_copy(fieldData2_q1_h, fieldData2_q1.get_static_view()); + + for (unsigned int cell = 0; cell < fieldData1_q1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData1_q1.extent(1); pt++) + { + if (enable_tangents) + { + TEST_EQUALITY(fieldData1_q1_h(cell, pt).val(), 123.0 + myRank); + TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), num_tangent); + for (int i = 0; i < num_tangent; ++i) + TEST_EQUALITY(fieldData1_q1_h(cell, pt).dx(i), 0.123 + myRank + i); + } + else + { + TEST_EQUALITY(fieldData1_q1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData1_q1_h(cell, pt).availableSize(), 0); + } + } + } + for (unsigned int cell = 0; cell < fieldData2_q1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData2_q1.extent(1); pt++) + { + if (enable_tangents) + { + TEST_EQUALITY(fieldData2_q1_h(cell, pt).val(), 123.0 + myRank); + TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), num_tangent); + for (int i = 0; i < num_tangent; ++i) + { + TEST_EQUALITY(fieldData2_q1_h(cell, pt).dx(i), 0.123 + myRank + i); + TEST_EQUALITY(fieldData2_q1_h(cell, pt).dx(i), 0.123 + myRank + i); + } + } + else + { + TEST_EQUALITY(fieldData2_q1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData2_q1_h(cell, pt).availableSize(), 0); + } + } + } + } + { + PHX::MDField + fieldData_qedge1(fieldName_qedge1, basis_qedge1->functional); + + fm.getFieldData(fieldData_qedge1); + + auto fieldData_qedge1_h = Kokkos::create_mirror_view(fieldData_qedge1.get_static_view()); + Kokkos::deep_copy(fieldData_qedge1_h, fieldData_qedge1.get_static_view()); + + for (unsigned int cell = 0; cell < fieldData_qedge1.extent(0); ++cell) + { + for (unsigned int pt = 0; pt < fieldData_qedge1.extent(1); ++pt) + { + if (enable_tangents) + { + TEST_EQUALITY(fieldData_qedge1_h(cell, pt).val(), 123.0 + myRank); + TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), num_tangent); + for (int i = 0; i < num_tangent; ++i) + TEST_EQUALITY(fieldData_qedge1_h(cell, pt).dx(i), 0.123 + myRank + i); + } + else + { + TEST_EQUALITY(fieldData_qedge1_h(cell, pt), 123.0 + myRank); + TEST_EQUALITY(fieldData_qedge1_h(cell, pt).availableSize(), 0); + } + } + } + } + } + + Teuchos::RCP buildBasis(std::size_t worksetSize, const std::string &basisName) + { + Teuchos::RCP topo = + Teuchos::rcp(new shards::CellTopology(shards::getCellTopologyData>())); + + panzer::CellData cellData(worksetSize, topo); + return Teuchos::rcp(new panzer::PureBasis(basisName, 1, cellData)); + } + + Teuchos::RCP buildMesh(int elemX, int elemY) + { + Teuchos::RCP pl = rcp(new Teuchos::ParameterList); + pl->set("X Blocks", 1); + pl->set("Y Blocks", 1); + pl->set("X Elements", elemX); + pl->set("Y Elements", elemY); + + panzer_stk::SquareQuadMeshFactory factory; + factory.setParameterList(pl); + Teuchos::RCP mesh = factory.buildUncommitedMesh(MPI_COMM_WORLD); + factory.completeMeshConstruction(*mesh, MPI_COMM_WORLD); + + return mesh; + } + + void testInitialization(const Teuchos::RCP &ipb) + { + // Physics block + ipb->setName("test physics"); + { + Teuchos::ParameterList &p = ipb->sublist("a"); + p.set("Type", "Energy"); + p.set("Prefix", ""); + p.set("Model ID", "solid"); + p.set("Basis Type", "HGrad"); + p.set("Basis Order", 1); + p.set("Integration Order", 1); + } + { + Teuchos::ParameterList &p = ipb->sublist("b"); + p.set("Type", "Energy"); + p.set("Prefix", "ION_"); + p.set("Model ID", "solid"); + p.set("Basis Type", "HCurl"); + p.set("Basis Order", 1); + p.set("Integration Order", 1); + } + } + +} diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra.hpp index aec43b41dfbc..6d9bde9d1a3b 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra.hpp @@ -163,7 +163,7 @@ class GatherSolution_BlockedTpetra public: GatherSolution_BlockedTpetra(const Teuchos::RCP & indexer) - : gidIndexer_(indexer) {} + : globalIndexer_(indexer) {} GatherSolution_BlockedTpetra(const Teuchos::RCP & indexer, const Teuchos::ParameterList& p); @@ -176,13 +176,13 @@ class GatherSolution_BlockedTpetra void evaluateFields(typename TRAITS::EvalData d); virtual Teuchos::RCP clone(const Teuchos::ParameterList & pl) const - { return Teuchos::rcp(new GatherSolution_BlockedTpetra(gidIndexer_,pl)); } + { return Teuchos::rcp(new GatherSolution_BlockedTpetra(globalIndexer_,pl)); } private: typedef typename panzer::Traits::Tangent EvalT; typedef typename panzer::Traits::Tangent::ScalarT ScalarT; - //typedef typename panzer::Traits::RealType RealT; + typedef typename panzer::Traits::RealType RealT; typedef BlockedTpetraLinearObjContainer ContainerType; typedef Tpetra::Vector VectorType; @@ -194,10 +194,14 @@ class GatherSolution_BlockedTpetra // maps the local (field,element,basis) triplet to a global ID // for scattering - Teuchos::RCP gidIndexer_; + Teuchos::RCP globalIndexer_; std::vector fieldIds_; // field IDs needing mapping + //! Returns the index into the Thyra ProductVector sub-block. Size + //! of number of fields to scatter + std::vector productVectorBlockIndex_; + std::vector< PHX::MDField > gatherFields_; std::vector indexerNames_; @@ -206,9 +210,16 @@ class GatherSolution_BlockedTpetra Teuchos::RCP > blockedContainer_; + //! Local indices for unknowns + PHX::View worksetLIDs_; + + //! Offset into the cell lids for each field. Size of number of fields to scatter. + std::vector> fieldOffsets_; + // Fields for storing tangent components dx/dp of solution vector x bool has_tangent_fields_; - std::vector< std::vector< PHX::MDField > > tangentFields_; + std::vector< std::vector< PHX::MDField > > tangentFields_; + PHX::ViewOfViews<2,PHX::View> tangentFieldsVoV_; GatherSolution_BlockedTpetra(); }; diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp index b0ef54fdd70b..52488585d37e 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_GatherSolution_BlockedTpetra_impl.hpp @@ -8,8 +8,8 @@ // ***************************************************************************** // @HEADER -#ifndef PANZER_GATHER_SOLUTION_BLOCKED_EPETRA_IMPL_HPP -#define PANZER_GATHER_SOLUTION_BLOCKED_EPETRA_IMPL_HPP +#ifndef PANZER_GATHER_SOLUTION_BLOCKED_TPETRA_IMPL_HPP +#define PANZER_GATHER_SOLUTION_BLOCKED_TPETRA_IMPL_HPP #include "Teuchos_Assert.hpp" #include "Phalanx_DataLayout.hpp" @@ -216,7 +216,7 @@ panzer::GatherSolution_BlockedTpetra & indexer, const Teuchos::ParameterList& p) - : gidIndexer_(indexer) + : globalIndexer_(indexer) , has_tangent_fields_(false) { typedef std::vector< std::vector > vvstring; @@ -250,7 +250,7 @@ GatherSolution_BlockedTpetra( tangentFields_[fd].resize(tangent_field_names[fd].size()); for (std::size_t i=0; i(tangent_field_names[fd][i],basis->functional); + PHX::MDField(tangent_field_names[fd][i],basis->functional); this->addDependentField(tangentFields_[fd][i]); } } @@ -268,17 +268,60 @@ GatherSolution_BlockedTpetra( // ********************************************************************** template void panzer::GatherSolution_BlockedTpetra:: -postRegistrationSetup(typename TRAITS::SetupData /* d */, +postRegistrationSetup(typename TRAITS::SetupData d, PHX::FieldManager& /* fm */) { TEUCHOS_ASSERT(gatherFields_.size() == indexerNames_.size()); - fieldIds_.resize(gatherFields_.size()); + const Workset & workset_0 = (*d.worksets_)[0]; + const std::string blockId = this->wda(workset_0).block_id; + fieldIds_.resize(gatherFields_.size()); + fieldOffsets_.resize(gatherFields_.size()); + productVectorBlockIndex_.resize(gatherFields_.size()); + int maxElementBlockGIDCount = -1; for (std::size_t fd = 0; fd < gatherFields_.size(); ++fd) { - // get field ID from DOF manager - const std::string& fieldName = indexerNames_[fd]; - fieldIds_[fd] = gidIndexer_->getFieldNum(fieldName); + + const std::string fieldName = indexerNames_[fd]; + const int globalFieldNum = globalIndexer_->getFieldNum(fieldName); // Field number in the aggregate BlockDOFManager + productVectorBlockIndex_[fd] = globalIndexer_->getFieldBlock(globalFieldNum); + const auto& subGlobalIndexer = globalIndexer_->getFieldDOFManagers()[productVectorBlockIndex_[fd]]; + fieldIds_[fd] = subGlobalIndexer->getFieldNum(fieldName); // Field number in the sub-global-indexer + + const std::vector& offsets = subGlobalIndexer->getGIDFieldOffsets(blockId,fieldIds_[fd]); + fieldOffsets_[fd] = PHX::View("GatherSolution_BlockedTpetra(Tangent):fieldOffsets",offsets.size()); + auto hostOffsets = Kokkos::create_mirror_view(fieldOffsets_[fd]); + for (std::size_t i=0; i < offsets.size(); ++i) + hostOffsets(i) = offsets[i]; + Kokkos::deep_copy(fieldOffsets_[fd], hostOffsets); + maxElementBlockGIDCount = std::max(subGlobalIndexer->getElementBlockGIDCount(blockId),maxElementBlockGIDCount); + } + + // We will use one workset lid view for all fields, but has to be + // sized big enough to hold the largest elementBlockGIDCount in the + // ProductVector. + worksetLIDs_ = PHX::View("ScatterResidual_BlockedTpetra(Tangent):worksetLIDs", + gatherFields_[0].extent(0), + maxElementBlockGIDCount); + + // Set up storage for tangentFields using view of views + // We also need storage for the number of tangent fields associated with + // each gatherField + + if (has_tangent_fields_) { + + size_t inner_vector_max_size = 0; + for (std::size_t fd = 0; fd < tangentFields_.size(); ++fd) + inner_vector_max_size = std::max(inner_vector_max_size,tangentFields_[fd].size()); + tangentFieldsVoV_.initialize("GatherSolution_BlockedTpetra::tangentFieldsVoV_",gatherFields_.size(),inner_vector_max_size); + + for (std::size_t fd = 0; fd < gatherFields_.size(); ++fd) { + for (std::size_t i=0; i void panzer::GatherSolution_BlockedTpetra:: evaluateFields(typename TRAITS::EvalData workset) { - using Teuchos::RCP; - using Teuchos::ArrayRCP; - using Teuchos::ptrFromRef; - using Teuchos::rcp_dynamic_cast; - - using Thyra::VectorBase; - using Thyra::SpmdVectorBase; - using Thyra::ProductVectorBase; + using Teuchos::RCP; + using Teuchos::ArrayRCP; + using Teuchos::ptrFromRef; + using Teuchos::rcp_dynamic_cast; - Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout)); - out.setShowProcRank(true); - out.setOutputToRootOnly(-1); + using Thyra::VectorBase; + using Thyra::SpmdVectorBase; + using Thyra::ProductVectorBase; - std::vector > GIDs; - std::vector LIDs; + Teuchos::FancyOStream out(Teuchos::rcpFromRef(std::cout)); + out.setShowProcRank(true); + out.setOutputToRootOnly(-1); - // for convenience pull out some objects from workset - std::string blockId = this->wda(workset).block_id; - const std::vector & localCellIds = this->wda(workset).cell_local_ids; + const PHX::View & localCellIds = this->wda(workset).getLocalCellIDs(); - Teuchos::RCP > x; - if (useTimeDerivativeSolutionVector_) - x = rcp_dynamic_cast >(blockedContainer_->get_dxdt()); - else - x = rcp_dynamic_cast >(blockedContainer_->get_x()); + Teuchos::RCP > blockedSolution; + if (useTimeDerivativeSolutionVector_) + blockedSolution = rcp_dynamic_cast >(blockedContainer_->get_dxdt()); + else + blockedSolution = rcp_dynamic_cast >(blockedContainer_->get_x()); - // gather operation for each cell in workset - for(std::size_t worksetCellIndex=0;worksetCellIndexgetFieldDOFManagers()[productVectorBlockIndex_[fieldIndex]]; + const std::string blockId = this->wda(workset).block_id; + const int num_dofs = globalIndexer_->getFieldDOFManagers()[productVectorBlockIndex_[fieldIndex]]->getElementBlockGIDCount(blockId); + blockIndexer->getElementLIDs(localCellIds,worksetLIDs_,num_dofs); + currentWorksetLIDSubBlock = productVectorBlockIndex_[fieldIndex]; + } - gidIndexer_->getElementGIDsPair(cellLocalId,GIDs,blockId); + const int blockRowIndex = productVectorBlockIndex_[fieldIndex]; + const auto& subblockSolution = *((rcp_dynamic_cast>(blockedSolution->getNonconstVectorBlock(blockRowIndex),true))->getTpetraVector()); + const auto kokkosSolution = subblockSolution.getLocalViewDevice(Tpetra::Access::ReadOnly); - // caculate the local IDs for this element - LIDs.resize(GIDs.size()); - for(std::size_t i=0;i x_map = blockedContainer_->getMapForBlock(GIDs[i].first); + // Class data fields for lambda capture + const PHX::View fieldOffsets = fieldOffsets_[fieldIndex]; + const PHX::View worksetLIDs = worksetLIDs_; + const PHX::View fieldValues = gatherFields_[fieldIndex].get_static_view(); - LIDs[i] = x_map->getLocalElement(GIDs[i].second); - } + if (has_tangent_fields_) { + const int numTangents = tangentFields_[fieldIndex].size(); + const auto tangentFieldsDevice = tangentFieldsVoV_.getViewDevice(); + const auto kokkosTangents = Kokkos::subview(tangentFieldsDevice,fieldIndex,Kokkos::ALL()); + Kokkos::parallel_for(Kokkos::RangePolicy(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { + for (int basis=0; basis < static_cast(fieldOffsets.size()); ++basis) { + const int rowLID = worksetLIDs(cell,fieldOffsets(basis)); + fieldValues(cell,basis).zero(); + fieldValues(cell,basis).val() = kokkosSolution(rowLID,0); + for (int i_tangent=0; i_tangent(0,workset.num_cells), KOKKOS_LAMBDA (const int& cell) { + for (int basis=0; basis < static_cast(fieldOffsets.size()); ++basis) { + const int rowLID = worksetLIDs(cell,fieldOffsets(basis)); + fieldValues(cell,basis).zero(); + fieldValues(cell,basis) = kokkosSolution(rowLID,0); + } + }); + } + } - // loop over the fields to be gathered - Teuchos::ArrayRCP local_x; - for (std::size_t fieldIndex=0; fieldIndexgetFieldBlock(fieldNum); - - // grab local data for inputing - RCP > block_x = rcp_dynamic_cast >(x->getNonconstVectorBlock(indexerId)); - block_x->getLocalData(ptrFromRef(local_x)); - - const std::vector & elmtOffset = gidIndexer_->getGIDFieldOffsets(blockId,fieldNum); - - // loop over basis functions and fill the fields - for(std::size_t basis=0;basis