From 313569cdca304469f2b15a54ec666826cf8d8426 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Tue, 4 Jul 2017 12:56:07 +0200 Subject: [PATCH 01/14] Add `using namespace std` to the headers here --- .../invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 1 + lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 54592989f..6a5021873 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -37,6 +37,7 @@ #include "qphix_singleton.h" using namespace QDP; +using namespace std; namespace Chroma { diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h index 0b99e51e4..ed3ab193a 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h @@ -38,7 +38,9 @@ #include "qphix_singleton.h" #include "actions/ferm/invert/qphix/qphix_vec_traits.h" #include "qphix/blas_new_c.h" + using namespace QDP; +using namespace std; namespace Chroma { From 881923345575f2c97bb156bfa7548d57ac46246d Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Tue, 22 Aug 2017 18:19:23 +0200 Subject: [PATCH 02/14] Start --- ...ssolver_mdagm_clover_qphix_iter_refine_w.h | 241 +++++++++++++++--- 1 file changed, 208 insertions(+), 33 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 6a5021873..bd83960ca 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -276,31 +276,54 @@ namespace Chroma t_boundary, toDouble(aniso_coeffs[0]), toDouble(aniso_coeffs[3])); - - M_inner=new QPhiX::EvenOddCloverOperator(u_packed_i, - clov_packed_i[1], - invclov_packed_i[0], - geom_inner, - t_boundary, - toDouble(aniso_coeffs[0]), - toDouble(aniso_coeffs[3])); - - - bicgstab_inner_solver = new QPhiX::InvBiCGStab((*M_inner), invParam.MaxIter); - - mixed_solver = new QPhiX::InvRichardsonMultiPrec((*M_outer), (*bicgstab_inner_solver), toDouble(invParam.Delta), invParam.MaxIter); + + M_inner = new QPhiX::EvenOddCloverOperator( + u_packed_i, + clov_packed_i[1], + invclov_packed_i[0], + geom_inner, + t_boundary, + toDouble(aniso_coeffs[0]), + toDouble(aniso_coeffs[3])); + + switch (invParam.SolverType) { + case CG: { + QDPIO::cout << "Creating the CG Solver" << std::endl; + cg_solver = new QPhiX::InvCG::Vec, + VecTraits::Soa, + VecTraits::compress12>(*M, invParam.MaxIter); + } + case BICGSTAB: { + QDPIO::cout << "Creating the BiCGStab Solver" << std::endl; + bicgstab_inner_solver = + new QPhiX::InvBiCGStab( + *M_inner, invParam.MaxIter); + + mixed_solver = new QPhiX::InvRichardsonMultiPrec( + *M_outer, *bicgstab_inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + } break; + default: + QDPIO::cerr << "UNKNOWN Solver Type" << std::endl; + QDP_abort(1); + } } - - - //! Destructor - ~MdagMSysSolverQPhiXCloverIterRefine() - { - - // Need to unalloc all the memory... - QDPIO::cout << "Destructing" << std::endl; + } + + //! Destructor + ~MdagMSysSolverQPhiXCloverIterRefine() + { + + // Need to unalloc all the memory... + QDPIO::cout << "Destructing" << std::endl; #ifndef QDP_IS_QDPJIT geom_outer->free(psi_qphix); @@ -354,7 +377,34 @@ namespace Chroma //! Return the subset on which the operator acts const Subset& subset() const {return A->subset();} - + + SystemSolverResults_t operator()( + T &psi, const T &chi, Chroma::AbsChronologicalPredictor4D &predictor) const + { + + START_CODE(); + StopWatch swatch; + swatch.reset(); + swatch.start(); + SystemSolverResults_t res; + switch (invParam.SolverType) { + case CG: + res = cgSolve(psi, chi, predictor); + break; + case BICGSTAB: + res = biCGStabSolve(psi, chi, predictor); + break; + default: + QDPIO::cout << "Unknown Solver " << std::endl; + break; + } + + swatch.stop(); + QDPIO::cout << "QPHIX_MDAGM_ITER_REFINE_SOLVER: total time: " + << swatch.getTimeInSeconds() << " (sec)" << endl; + END_CODE(); + return res; + } SystemSolverResults_t operator()(T& psi, const T& chi) const { @@ -366,18 +416,147 @@ namespace Chroma return res; } + + SystemSolverResults_t + cgSolve(T &psi, const T &chi, AbsChronologicalPredictor4D &predictor) const + { + SystemSolverResults_t res; + Handle< LinearOperator > MdagM( new MdagMLinOp(A) ); + + T Y; + Y[ A->subset() ] = psi; // Y is initial guess + + try { + // Try to cast the predictor to a two step predictor + AbsTwoStepChronologicalPredictor4D& two_step_predictor = + dynamic_cast& >(predictor); + + + // Predict Y and X separately + two_step_predictor.predictY(Y,*A,chi); + two_step_predictor.predictX(psi,*MdagM, chi); + } + catch( std::bad_cast) { + + // Not a 2 step predictor. Predict X + // Then MX = Y is a good guess. + predictor(psi,*MdagM, chi); + (*A)(Y,psi,PLUS); + + } + + +#ifndef QDP_IS_QDPJIT + QDPIO::cout << "Packing" << std::endl << std::flush ; + QPhiX::qdp_pack_cb_spinor<>(Y, tmp_qphix, *geom_outer,1); // Initial Guess for Y + QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom_outer,1); // Initial Guess for X + QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom_outer,1); // RHS +#else + tmp_qphix = (QPhiX_Spinor *)(Y.getFjit()) + cbsize_in_blocks; + psi_qphix = (QPhiX_Spinor *)(psi.getFjit()) + cbsize_in_blocks; + chi_qphix = (QPhiX_Spinor *)(chi.getFjit()) + cbsize_in_blocks; +#endif + QDPIO::cout << "Done" << std::endl << std::flush; + double rsd_final; + int num_cb_sites = Layout::vol()/2; + + unsigned long site_flops1=0; + unsigned long mv_apps1=0; + int n_count1=0; + + unsigned long site_flops2=0; + unsigned long mv_apps2=0; + int n_count2=0; + + QDPIO::cout << "Starting Y solve" << std::endl << std::flush ; + double start = omp_get_wtime(); + (*mixed_solver)(tmp_qphix,chi_qphix, toDouble(invParam.RsdTarget), n_count1, rsd_final, site_flops1, mv_apps1, -1, invParam.VerboseP); + double end = omp_get_wtime(); + + + unsigned long total_flops = (site_flops1 + (1320+504+1320+504+48)*mv_apps1)*num_cb_sites; + double gflops = (double)(total_flops)/(1.0e9); + double total_time = end - start; + + Double r_final = sqrt(toDouble(rsd_final)); + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: " << n_count1 << " iters, rsd_sq_final=" << rsd_final << " ||r||/||b|| (acc) = " << r_final <(tmp_qphix, Y, *geom_outer,1); +#endif + double norm2Y; + QPhiX::norm2Spinor(norm2Y, tmp_qphix, *geom_outer, n_blas_simt); + r_final = sqrt(toDouble(rsd_final)); + + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: " << n_count2 << " iters, rsd_sq_final=" << rsd_final << " ||r||/||b|| (acc) = " << r_final << std::endl; + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: Solver Time="<< total_time <<" (sec) Performance=" << gflops / total_time << " GFLOPS" << std::endl; + +#ifndef QDP_IS_QDPJIT + QPhiX::qdp_unpack_cb_spinor<>(psi_qphix, psi, *geom_outer,1); +#endif + + try { + // Try to cast the predictor to a two step predictor + AbsTwoStepChronologicalPredictor4D& two_step_predictor = + dynamic_cast& >(predictor); + two_step_predictor.newYVector(Y); + two_step_predictor.newXVector(psi); + + } + catch( std::bad_cast) { + + // Not a 2 step predictor. Predict X + // Then MX = Y is a good guess. + predictor.newVector(psi); + } + + // Chi Should now hold the result spinor + // Check it against chroma. -- reuse Y as the residuum + Y[ A->subset() ] = chi; + { + T tmp,tmp2; + (*A)(tmp, psi, PLUS); + (*A)(tmp2, tmp, MINUS); + + Y[ A->subset() ] -= tmp2; + } + + Double r2 = norm2(Y,A->subset()); + Double b2 = norm2(chi, A->subset()); + Double rel_resid = sqrt(r2/b2); + res.resid=rel_resid; + res.n_count = n_count1 + n_count2; + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: total_iters="<& predictor) const + SystemSolverResults_t + biCgStabSolve(T &psi, const T &chi, AbsChronologicalPredictor4D &predictor) const { - START_CODE(); - StopWatch swatch; - swatch.reset(); swatch.start(); - /* Factories here later? */ SystemSolverResults_t res; Handle< LinearOperator > MdagM( new MdagMLinOp(A) ); @@ -503,10 +682,6 @@ namespace Chroma QDP_abort(1); } #endif - swatch.stop(); - QDPIO::cout << "QPHIX_MDAGM_SOLVER: total time: " << swatch.getTimeInSeconds() << " (sec)" << endl; - END_CODE(); - return res; } From 44d1b6d007ad374916b26ce95b59a39b64dc7791 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Tue, 22 Aug 2017 18:19:40 +0200 Subject: [PATCH 03/14] Remove unneeded code --- ...ssolver_mdagm_clover_qphix_iter_refine_w.h | 45 ++----------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index bd83960ca..2cd3216f0 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -420,39 +420,16 @@ namespace Chroma SystemSolverResults_t cgSolve(T &psi, const T &chi, AbsChronologicalPredictor4D &predictor) const { - SystemSolverResults_t res; - Handle< LinearOperator > MdagM( new MdagMLinOp(A) ); - - T Y; - Y[ A->subset() ] = psi; // Y is initial guess - - try { - // Try to cast the predictor to a two step predictor - AbsTwoStepChronologicalPredictor4D& two_step_predictor = - dynamic_cast& >(predictor); - - - // Predict Y and X separately - two_step_predictor.predictY(Y,*A,chi); - two_step_predictor.predictX(psi,*MdagM, chi); - } - catch( std::bad_cast) { - - // Not a 2 step predictor. Predict X - // Then MX = Y is a good guess. - predictor(psi,*MdagM, chi); - (*A)(Y,psi,PLUS); - - } + SystemSolverResults_t res; + Handle> MdagM(new MdagMLinOp(A)); + predictor(psi, *MdagM, chi); #ifndef QDP_IS_QDPJIT QDPIO::cout << "Packing" << std::endl << std::flush ; - QPhiX::qdp_pack_cb_spinor<>(Y, tmp_qphix, *geom_outer,1); // Initial Guess for Y QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom_outer,1); // Initial Guess for X QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom_outer,1); // RHS #else - tmp_qphix = (QPhiX_Spinor *)(Y.getFjit()) + cbsize_in_blocks; psi_qphix = (QPhiX_Spinor *)(psi.getFjit()) + cbsize_in_blocks; chi_qphix = (QPhiX_Spinor *)(chi.getFjit()) + cbsize_in_blocks; #endif @@ -464,10 +441,6 @@ namespace Chroma unsigned long mv_apps1=0; int n_count1=0; - unsigned long site_flops2=0; - unsigned long mv_apps2=0; - int n_count2=0; - QDPIO::cout << "Starting Y solve" << std::endl << std::flush ; double start = omp_get_wtime(); (*mixed_solver)(tmp_qphix,chi_qphix, toDouble(invParam.RsdTarget), n_count1, rsd_final, site_flops1, mv_apps1, -1, invParam.VerboseP); @@ -478,18 +451,6 @@ namespace Chroma double gflops = (double)(total_flops)/(1.0e9); double total_time = end - start; - Double r_final = sqrt(toDouble(rsd_final)); - QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: " << n_count1 << " iters, rsd_sq_final=" << rsd_final << " ||r||/||b|| (acc) = " << r_final <(tmp_qphix, Y, *geom_outer,1); From 70c44dad95e35c327a64a30830fad584891eafcd Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Tue, 22 Aug 2017 18:24:51 +0200 Subject: [PATCH 04/14] Fix function name --- .../invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 2cd3216f0..3519d9b4e 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -516,7 +516,7 @@ namespace Chroma * \return syssolver results */ SystemSolverResults_t - biCgStabSolve(T &psi, const T &chi, AbsChronologicalPredictor4D &predictor) const + biCGStabSolve(T &psi, const T &chi, AbsChronologicalPredictor4D &predictor) const { SystemSolverResults_t res; Handle< LinearOperator > MdagM( new MdagMLinOp(A) ); From 1a4fe46df3130fc2043ec70d6a96e7c4307aa78d Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Wed, 23 Aug 2017 09:50:39 +0200 Subject: [PATCH 05/14] More changes --- ...ssolver_mdagm_clover_qphix_iter_refine_w.h | 178 +++++++++--------- 1 file changed, 87 insertions(+), 91 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 3519d9b4e..a345c2449 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -287,14 +287,26 @@ namespace Chroma toDouble(aniso_coeffs[3])); switch (invParam.SolverType) { - case CG: { + case CG: QDPIO::cout << "Creating the CG Solver" << std::endl; cg_solver = new QPhiX::InvCG::Vec, VecTraits::Soa, VecTraits::compress12>(*M, invParam.MaxIter); - } - case BICGSTAB: { + + bool constexpr MdagM = true; + mixed_solver = new QPhiX::InvRichardsonMultiPrec( + *M_outer, *bicgstab_inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + + case BICGSTAB: QDPIO::cout << "Creating the BiCGStab Solver" << std::endl; bicgstab_inner_solver = new QPhiX::InvBiCGStab( @@ -309,7 +321,7 @@ namespace Chroma InnerSoa, comp12>( *M_outer, *bicgstab_inner_solver, toDouble(invParam.Delta), invParam.MaxIter); - } break; + break; default: QDPIO::cerr << "UNKNOWN Solver Type" << std::endl; QDP_abort(1); @@ -422,91 +434,78 @@ namespace Chroma { SystemSolverResults_t res; Handle> MdagM(new MdagMLinOp(A)); - - predictor(psi, *MdagM, chi); + predictor(psi, (*MdagM), chi); #ifndef QDP_IS_QDPJIT - QDPIO::cout << "Packing" << std::endl << std::flush ; - QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom_outer,1); // Initial Guess for X - QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom_outer,1); // RHS + // Pack Spinors psi and chi + QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom, 1); + QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom, 1); #else - psi_qphix = (QPhiX_Spinor *)(psi.getFjit()) + cbsize_in_blocks; - chi_qphix = (QPhiX_Spinor *)(chi.getFjit()) + cbsize_in_blocks; + psi_qphix = (QPhiX_Spinor *)(psi.getFjit()) + cbsize_in_blocks; + chi_qphix = (QPhiX_Spinor *)(chi.getFjit()) + cbsize_in_blocks; #endif - QDPIO::cout << "Done" << std::endl << std::flush; - double rsd_final; - int num_cb_sites = Layout::vol()/2; - - unsigned long site_flops1=0; - unsigned long mv_apps1=0; - int n_count1=0; - - QDPIO::cout << "Starting Y solve" << std::endl << std::flush ; - double start = omp_get_wtime(); - (*mixed_solver)(tmp_qphix,chi_qphix, toDouble(invParam.RsdTarget), n_count1, rsd_final, site_flops1, mv_apps1, -1, invParam.VerboseP); - double end = omp_get_wtime(); - - - unsigned long total_flops = (site_flops1 + (1320+504+1320+504+48)*mv_apps1)*num_cb_sites; - double gflops = (double)(total_flops)/(1.0e9); - double total_time = end - start; + double rsd_final; + unsigned long site_flops = 0; + unsigned long mv_apps = 0; + + double start = omp_get_wtime(); + int my_isign = 1; + (*mixed_solver)(psi_qphix, + chi_qphix, + toDouble(invParam.RsdTarget), + res.n_count, + rsd_final, + site_flops, + mv_apps, + my_isign, + invParam.VerboseP); + double end = omp_get_wtime(); #ifndef QDP_IS_QDPJIT - // Want Norm of Y - QPhiX::qdp_unpack_cb_spinor<>(tmp_qphix, Y, *geom_outer,1); + QPhiX::qdp_unpack_cb_spinor<>(psi_qphix, psi, *geom, 1); #endif - double norm2Y; - QPhiX::norm2Spinor(norm2Y, tmp_qphix, *geom_outer, n_blas_simt); - r_final = sqrt(toDouble(rsd_final)); - QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: " << n_count2 << " iters, rsd_sq_final=" << rsd_final << " ||r||/||b|| (acc) = " << r_final << std::endl; - QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: Solver Time="<< total_time <<" (sec) Performance=" << gflops / total_time << " GFLOPS" << std::endl; + predictor.newVector(psi); -#ifndef QDP_IS_QDPJIT - QPhiX::qdp_unpack_cb_spinor<>(psi_qphix, psi, *geom_outer,1); -#endif - - try { - // Try to cast the predictor to a two step predictor - AbsTwoStepChronologicalPredictor4D& two_step_predictor = - dynamic_cast& >(predictor); - two_step_predictor.newYVector(Y); - two_step_predictor.newXVector(psi); - - } - catch( std::bad_cast) { + // Chi Should now hold the result spinor + // Check it against chroma. + { + T r = chi; + T tmp, tmp2; + (*A)(tmp, psi, PLUS); + (*A)(tmp2, tmp, MINUS); + r[A->subset()] -= tmp2; - // Not a 2 step predictor. Predict X - // Then MX = Y is a good guess. - predictor.newVector(psi); - } + Double r2 = norm2(r, A->subset()); + Double b2 = norm2(chi, A->subset()); + Double rel_resid = sqrt(r2 / b2); + res.resid = rel_resid; + QDPIO::cout << "QPHIX_CLOVER_CG_SOLVER: " << res.n_count + << " iters, rsd_sq_final=" << rel_resid << std::endl; - // Chi Should now hold the result spinor - // Check it against chroma. -- reuse Y as the residuum - Y[ A->subset() ] = chi; - { - T tmp,tmp2; - (*A)(tmp, psi, PLUS); - (*A)(tmp2, tmp, MINUS); + QDPIO::cout << "QPHIX_CLOVER_CG_SOLVER: || r || / || b || = " << rel_resid + << std::endl; - Y[ A->subset() ] -= tmp2; - } +#if 0 + if ( !toBool ( rel_resid < invParam.RsdTarget*invParam.RsdToleranceFactor ) ) { + QDPIO::cout << "SOLVE FAILED" << std::endl; + QDP_abort(1); + } +#endif + } - Double r2 = norm2(Y,A->subset()); - Double b2 = norm2(chi, A->subset()); - Double rel_resid = sqrt(r2/b2); - res.resid=rel_resid; - res.n_count = n_count1 + n_count2; - QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_BICGSTAB_SOLVER: total_iters="<::SoaInner, MixedVecTraits::compress12> > M_inner; - // Inner BiCGStab - Handle< QPhiX::InvBiCGStab::VecInner, - MixedVecTraits::SoaInner, - MixedVecTraits::compress12> > bicgstab_inner_solver; - - - // Outer solver - Handle< QPhiX::InvRichardsonMultiPrec::Vec, - MixedVecTraits::Soa, - MixedVecTraits::compress12, - InnerReal, - MixedVecTraits::VecInner, - MixedVecTraits::SoaInner, - MixedVecTraits::compress12> > mixed_solver; - + // Inner solver, can be CG or BiCGStab + Handle::VecInner, + MixedVecTraits::SoaInner, + MixedVecTraits::compress12>> + inner_solver; + + // Outer solver, will be Richardson solver. + Handle::Vec, + MixedVecTraits::Soa, + MixedVecTraits::compress12>> + mixed_solver; + QPhiX_Clover* invclov_packed[2]; QPhiX_Clover* clov_packed[2]; QPhiX_Gauge* u_packed[2]; From 7c5acbb6f68f54009dcfca7c3a93eec98a6eff10 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Wed, 23 Aug 2017 09:59:35 +0200 Subject: [PATCH 06/14] Fix compilation errors --- ...ssolver_mdagm_clover_qphix_iter_refine_w.h | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index a345c2449..aa21f9a22 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -31,6 +31,7 @@ #include "qphix/qdp_packer.h" #include "qphix/clover.h" #include "qphix/invbicgstab.h" +#include "qphix/invcg.h" #include "qphix/inv_richardson_multiprec.h" #include "actions/ferm/invert/qphix/qphix_vec_traits.h" @@ -289,10 +290,11 @@ namespace Chroma switch (invParam.SolverType) { case CG: QDPIO::cout << "Creating the CG Solver" << std::endl; - cg_solver = new QPhiX::InvCG::Vec, - VecTraits::Soa, - VecTraits::compress12>(*M, invParam.MaxIter); + inner_solver = + new QPhiX::InvCG::Vec, + VecTraits::Soa, + VecTraits::compress12>(*M_inner, invParam.MaxIter); bool constexpr MdagM = true; mixed_solver = new QPhiX::InvRichardsonMultiPrec( - *M_outer, *bicgstab_inner_solver, toDouble(invParam.Delta), invParam.MaxIter); - + *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + case BICGSTAB: QDPIO::cout << "Creating the BiCGStab Solver" << std::endl; - bicgstab_inner_solver = - new QPhiX::InvBiCGStab( - *M_inner, invParam.MaxIter); + inner_solver = new QPhiX::InvBiCGStab( + *M_inner, invParam.MaxIter); mixed_solver = new QPhiX::InvRichardsonMultiPrec( - *M_outer, *bicgstab_inner_solver, toDouble(invParam.Delta), invParam.MaxIter); - break; + *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + break; default: QDPIO::cerr << "UNKNOWN Solver Type" << std::endl; QDP_abort(1); } } - } - //! Destructor ~MdagMSysSolverQPhiXCloverIterRefine() { @@ -438,8 +437,8 @@ namespace Chroma #ifndef QDP_IS_QDPJIT // Pack Spinors psi and chi - QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom, 1); - QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom, 1); + QPhiX::qdp_pack_cb_spinor<>(psi, psi_qphix, *geom_outer, 1); + QPhiX::qdp_pack_cb_spinor<>(chi, chi_qphix, *geom_outer, 1); #else psi_qphix = (QPhiX_Spinor *)(psi.getFjit()) + cbsize_in_blocks; chi_qphix = (QPhiX_Spinor *)(chi.getFjit()) + cbsize_in_blocks; @@ -462,7 +461,7 @@ namespace Chroma double end = omp_get_wtime(); #ifndef QDP_IS_QDPJIT - QPhiX::qdp_unpack_cb_spinor<>(psi_qphix, psi, *geom, 1); + QPhiX::qdp_unpack_cb_spinor<>(psi_qphix, psi, *geom_outer, 1); #endif predictor.newVector(psi); From ae987ca9096459126bda2f8dba71dbd795febbc0 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Wed, 23 Aug 2017 10:05:53 +0200 Subject: [PATCH 07/14] Compiles, does it work, though? --- .../syssolver_mdagm_clover_qphix_iter_refine_w.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index aa21f9a22..01f50e8c4 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -288,7 +288,7 @@ namespace Chroma toDouble(aniso_coeffs[3])); switch (invParam.SolverType) { - case CG: + case CG: { QDPIO::cout << "Creating the CG Solver" << std::endl; inner_solver = new QPhiX::InvCG( *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); - case BICGSTAB: + break; + } + case BICGSTAB: { QDPIO::cout << "Creating the BiCGStab Solver" << std::endl; inner_solver = new QPhiX::InvBiCGStab( *M_inner, invParam.MaxIter); @@ -323,9 +325,12 @@ namespace Chroma comp12>( *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); break; - default: + } + default: { QDPIO::cerr << "UNKNOWN Solver Type" << std::endl; QDP_abort(1); + break; + } } } From 2fc022c0a661747c9fe26d1c884f3fbc28c742e6 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Wed, 23 Aug 2017 15:36:52 +0200 Subject: [PATCH 08/14] Construct with inner template parameters --- .../qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 01f50e8c4..1b8436baa 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -290,11 +290,8 @@ namespace Chroma switch (invParam.SolverType) { case CG: { QDPIO::cout << "Creating the CG Solver" << std::endl; - inner_solver = - new QPhiX::InvCG::Vec, - VecTraits::Soa, - VecTraits::compress12>(*M_inner, invParam.MaxIter); + inner_solver = new QPhiX::InvCG( + *M_inner, invParam.MaxIter); bool constexpr MdagM = true; mixed_solver = new QPhiX::InvRichardsonMultiPrec Date: Thu, 24 Aug 2017 09:44:14 +0200 Subject: [PATCH 09/14] Fix Handle type --- .../qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 1b8436baa..5ba9f50d0 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -699,11 +699,7 @@ namespace Chroma inner_solver; // Outer solver, will be Richardson solver. - Handle::Vec, - MixedVecTraits::Soa, - MixedVecTraits::compress12>> - mixed_solver; + Handle> mixed_solver; QPhiX_Clover* invclov_packed[2]; QPhiX_Clover* clov_packed[2]; From 121b68bb5c55c0129d063d8abfb7fb546053843c Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Thu, 24 Aug 2017 10:32:03 +0200 Subject: [PATCH 10/14] USe a unique_ptr instead --- ...ssolver_mdagm_clover_qphix_iter_refine_w.h | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 5ba9f50d0..d9ee4d413 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -37,6 +37,8 @@ #include "actions/ferm/invert/qphix/qphix_vec_traits.h" #include "qphix_singleton.h" +#include + using namespace QDP; using namespace std; @@ -293,17 +295,16 @@ namespace Chroma inner_solver = new QPhiX::InvCG( *M_inner, invParam.MaxIter); - bool constexpr MdagM = true; - mixed_solver = new QPhiX::InvRichardsonMultiPrec( - *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + mixed_solver.reset(new QPhiX::InvRichardsonMultiPrec( + *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter)); break; } @@ -312,15 +313,15 @@ namespace Chroma inner_solver = new QPhiX::InvBiCGStab( *M_inner, invParam.MaxIter); - mixed_solver = new QPhiX::InvRichardsonMultiPrec( - *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter); + mixed_solver.reset(new QPhiX::InvRichardsonMultiPrec( + *M_outer, *inner_solver, toDouble(invParam.Delta), invParam.MaxIter)); break; } default: { @@ -699,7 +700,8 @@ namespace Chroma inner_solver; // Outer solver, will be Richardson solver. - Handle> mixed_solver; + std::unique_ptr> + mixed_solver; QPhiX_Clover* invclov_packed[2]; QPhiX_Clover* clov_packed[2]; From c867ce16df20a607d949011acbd06a2aabfefc47 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Thu, 24 Aug 2017 12:04:49 +0200 Subject: [PATCH 11/14] Revert "Add `using namespace std` to the headers here" This reverts commit 313569cdca304469f2b15a54ec666826cf8d8426. --- .../invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 1 - lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index ea3e43a27..a6fbbe4bb 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -40,7 +40,6 @@ #include using namespace QDP; -using namespace std; namespace Chroma { diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h index 0105a1129..643aa9b04 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_w.h @@ -38,9 +38,7 @@ #include "qphix_singleton.h" #include "actions/ferm/invert/qphix/qphix_vec_traits.h" #include "qphix/blas_new_c.h" - using namespace QDP; -using namespace std; namespace Chroma { From 97bfbaf4086d0381f94cf2bb104a0ced24452874 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Thu, 24 Aug 2017 12:06:34 +0200 Subject: [PATCH 12/14] swatch call is done outside --- .../invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index a6fbbe4bb..334de9dac 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -643,10 +643,6 @@ namespace Chroma QDP_abort(1); } #endif - swatch.stop(); - QDPIO::cout << "QPHIX_MDAGM_SOLVER: total time: " << swatch.getTimeInSeconds() << " (sec)" << std::endl; - END_CODE(); - return res; } From a4b0f515a74ba479296ee02eb9bb004bafce4ee6 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Thu, 24 Aug 2017 12:06:49 +0200 Subject: [PATCH 13/14] Need an std:: --- .../invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 334de9dac..2ebc381fd 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -414,7 +414,7 @@ namespace Chroma swatch.stop(); QDPIO::cout << "QPHIX_MDAGM_ITER_REFINE_SOLVER: total time: " - << swatch.getTimeInSeconds() << " (sec)" << endl; + << swatch.getTimeInSeconds() << " (sec)" << std::endl; END_CODE(); return res; } From c47343f447d25e3b74dd0b254ff86d139e437940 Mon Sep 17 00:00:00 2001 From: Martin Ueding Date: Wed, 30 Aug 2017 22:15:02 +0200 Subject: [PATCH 14/14] Update the name in the output --- .../qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h index 2ebc381fd..b3bf351fa 100644 --- a/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h +++ b/lib/actions/ferm/invert/qphix/syssolver_mdagm_clover_qphix_iter_refine_w.h @@ -481,10 +481,10 @@ namespace Chroma Double b2 = norm2(chi, A->subset()); Double rel_resid = sqrt(r2 / b2); res.resid = rel_resid; - QDPIO::cout << "QPHIX_CLOVER_CG_SOLVER: " << res.n_count + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_CG_SOLVER: " << res.n_count << " iters, rsd_sq_final=" << rel_resid << std::endl; - QDPIO::cout << "QPHIX_CLOVER_CG_SOLVER: || r || / || b || = " << rel_resid + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_CG_SOLVER: || r || / || b || = " << rel_resid << std::endl; #if 0 @@ -501,7 +501,7 @@ namespace Chroma double gflops = (double)(total_flops) / (1.0e9); double total_time = end - start; - QDPIO::cout << "QPHIX_CLOVER_CG_SOLVER: Solver Time=" << total_time + QDPIO::cout << "QPHIX_CLOVER_ITER_REFINE_CG_SOLVER: Solver Time=" << total_time << " (sec) Performance=" << gflops / total_time << " GFLOPS" << std::endl;