diff --git a/.github/workflows/good_defines.txt b/.github/workflows/good_defines.txt index c0f6c73287..cdfaba0e22 100644 --- a/.github/workflows/good_defines.txt +++ b/.github/workflows/good_defines.txt @@ -1,3 +1,4 @@ +ALLOW_GPU_PRINTF AMREX_DEBUG AMREX_PARTICLES AMREX_SPACEDIM diff --git a/Docs/source/mpi_plus_x.rst b/Docs/source/mpi_plus_x.rst index 7d88e7be77..33df1d1224 100644 --- a/Docs/source/mpi_plus_x.rst +++ b/Docs/source/mpi_plus_x.rst @@ -130,6 +130,37 @@ To enable this, compile with:: USE_HIP = TRUE +Printing Warnings from GPU Kernels +================================== + +.. index:: USE_GPU_PRINTF + +Castro will output warnings if several assumptions are violated (often +triggering a retry in the process). On GPUs, printing from a kernel +(using ``printf()``) can increase the number of registers a kernel needs, +causing performance problems. As a result, warnings are disabled by +wrapping them in ``#ifndef AMREX_USE_GPU``. + +However, for debugging GPU runs, sometimes we want to see these +warnings. The build option ``USE_GPU_PRINTF=TRUE`` will enable these +(by setting the preprocessor flag ``ALLOW_GPU_PRINTF``). + +.. note:: + + Not every warning has been enabled for GPUs. + +.. tip:: + + On AMD architectures, it seems necessary to use unbuffered I/O. This + can be accomplished in the job submission script (for SLURM) by doing + + :: + + srun -u ./Castro... + + + + Working at Supercomputing Centers ================================= diff --git a/Exec/Make.Castro b/Exec/Make.Castro index 910a577563..5b1575820b 100644 --- a/Exec/Make.Castro +++ b/Exec/Make.Castro @@ -136,6 +136,10 @@ ifeq ($(USE_GPU),TRUE) endif endif +ifeq ($(USE_GPU_PRINTF),TRUE) + DEFINES += -DALLOW_GPU_PRINTF +endif + CASTRO_AUTO_SOURCE_DIR := $(TmpBuildDir)/castro_sources/$(optionsSuffix).EXE diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp index e79cf0574e..3108b7794f 100644 --- a/Source/driver/Castro.cpp +++ b/Source/driver/Castro.cpp @@ -3300,6 +3300,9 @@ Castro::check_for_negative_density () std::cout << "Invalid X[" << n << "] = " << X << " in zone " << i << ", " << j << ", " << k << " with density = " << rho << "\n"; +#elif defined(ALLOW_GPU_PRINTF) + AMREX_DEVICE_PRINTF("Invalid X[%d] = %g in zone (%d,%d,%d) with density = %g\n", + n, X, i, j, k, rho); #endif X_check_failed = 1; } @@ -3310,6 +3313,10 @@ Castro::check_for_negative_density () return {rho_check_failed, X_check_failed}; }); +#ifdef ALLOW_GPU_PRINTF + std::fflush(nullptr); +#endif + } ReduceTuple hv = reduce_data.value();