Merge branch 'development' into plot_max_level

.clang-tidy

@@ -26,6 +26,7 @@ Checks: >
        -modernize-use-trailing-return-type,
      performance-*,
        -performance-avoid-endl,
+       -performance-enum-size,
      portability-*,
      readability-*,
        -readability-avoid-const-params-in-decls,
@@ -38,6 +39,8 @@ Checks: >
        -readability-isolate-declaration,
        -readability-magic-numbers,
        -readability-named-parameter,
+       -readability-redundant-inline-specifier,
+       -readability-redundant-member-init,
        -readability-simplify-boolean-expr,
        -readability-static-accessed-through-instance,
      mpi-*,

.codespell-ignore-words

@@ -12,3 +12,4 @@ nd
 ue
 bion
 aas
+checkin

.editorconfig

@@ -0,0 +1,49 @@
+# http://EditorConfig.org
+#
+# precedence of rules is bottom to top
+
+# this is the top-most EditorConfig file
+root = true
+
+
+[*.{c,h,cpp,hpp,H,py}]
+# 4 space indentation
+indent_style = space
+indent_size = 4
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+# unix-style newlines
+end_of_line = lf
+
+# newline ending in files
+insert_final_newline = true
+
+
+[*.md]
+# two end of line whitespaces are newlines without a paragraph
+trim_trailing_whitespace = false
+
+
+[*.rst]
+# Enforce UTF-8 encoding
+charset = utf-8
+
+# Unix-style newlines
+end_of_line = lf
+
+# Newline ending in files
+insert_final_newline = true
+
+# 3 space indentation
+indent_style = space
+indent_size = 3
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+[{Makefile,GNUmakefile,Make.*}]
+# TABs are part of its syntax
+indent_style = tab
+indent_size = unset

.github/workflows/check_params.py

@@ -48,7 +48,7 @@ def doit(castro_dir):
                 # remove comments
                 idx = line.find("#")
                 if idx > 0:
-                    line = line[idx:]
+                    line = line[:idx]
 
                 found_param = pattern.match(line)
                 if not found_param:

.github/workflows/dependencies_hip.sh

@@ -17,10 +17,23 @@ set -eu -o pipefail
 #   failed files the given number of times.
 echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
 
-# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
+# Ref.: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/ubuntu.html
+
+# Make the directory if it doesn't exist yet.
+# This location is recommended by the distribution maintainers.
+sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+
+# Download the key, convert the signing-key to a full
+# keyring required by apt and store in the keyring directory
+wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
+    gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+
 curl -O https://repo.radeon.com/rocm/rocm.gpg.key
 sudo apt-key add rocm.gpg.key
-echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-debian}/ ubuntu main" \
+
+source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ...
+
+echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \
   | sudo tee /etc/apt/sources.list.d/rocm.list
 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
   | sudo tee -a /etc/profile.d/rocm.sh
@@ -43,8 +56,10 @@ sudo apt-get install -y --no-install-recommends \
     roctracer-dev   \
     rocprofiler-dev \
     rocrand-dev     \
-    rocprim-dev     \
-    hiprand-dev
+    rocprim-dev
+
+# hiprand-dev is a new package that does not exist in old versions
+sudo apt-get install -y --no-install-recommends hiprand-dev || true
 
 # activate
 #

.github/workflows/flame_wave-compare.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Run flame_wave
         run: |
           cd Exec/science/flame_wave
-          ./Castro2d.gnu.DEBUG.ex inputs_2d.testsuite max_step=2 castro.sum_interval=1 amr.plot_files_output=0 amr.checkpoint_files_output=0
+          ./Castro2d.gnu.DEBUG.ex inputs_2d.testsuite max_step=2 castro.sum_interval=1 amr.plot_files_output=1 amr.checkpoint_files_output=0
 
       - name: Check grid_diag.out
         run: |
@@ -47,3 +47,10 @@ jobs:
         run: |
           cd Exec/science/flame_wave
           diff species_diag.out ci-benchmarks/species_diag.out
+
+      - name: Check job_info parameters
+        run: |
+          cd Exec/science/flame_wave
+          line_no=`grep -n "Inputs File Parameters" plt00000/job_info | cut -d ":" -f 1`
+          tail -n +$(($line_no -1)) plt00000/job_info > tmp.txt
+          diff tmp.txt ci-benchmarks/job_info_params.txt

.github/workflows/gh-pages.yml

@@ -44,7 +44,7 @@ jobs:
         run: ./deploy_docs_action.sh
 
       - name: Deploy
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./out

.github/workflows/good_defines.txt

@@ -1,3 +1,4 @@
+ALLOW_GPU_PRINTF
 AMREX_DEBUG
 AMREX_PARTICLES
 AMREX_SPACEDIM

.github/workflows/style/check_trailing_whitespaces.sh

@@ -6,6 +6,7 @@ find . -type d \( -name .git \
                   -o -path ./paper \
                   -o -name build -o -name install \
                   -o -name tmp_build_dir -o -name tmp_install_dir \
+                  -o -name ci-benchmarks \
                   -o -path ./util/gcem \
                \) -prune -o \
        -type f \( \( -name "*.H" -o -name "*.h" -o -name "*.hh" -o -name "*.hpp" \

.github/workflows/wdmerger_collision-compare.yml

@@ -48,5 +48,5 @@ jobs:
       - name: Check the extrema
         run: |
           cd Exec/science/wdmerger
-          ../../../external/amrex/Tools/Plotfile/fextrema.gnu.ex plt00095 > fextrema.out
+          ../../../external/amrex/Tools/Plotfile/fextrema.gnu.ex plt00086 > fextrema.out
           diff fextrema.out ci-benchmarks/wdmerger_collision_2D.out

.zenodo.json

@@ -17,6 +17,11 @@
             "name": "Bell, John",
             "orcid": "0000-0002-5749-334X"
         },
+        {
+            "affiliation": "Department of Physics and Astronomy, Stony Brook University",
+            "name": "Chen, Zhi",
+            "orcid": "0000-0002-2839-107X"
+        },
         {
             "name": "Harpole, Alice",
             "orcid": "0000-0002-1530-781X"

CHANGES.md

@@ -1,3 +1,60 @@
+# 24.07
+
+  * Reorganizing of the existing 2-shock and HLL Riemann solvers
+    (#2887, #2888, #2889, #2890)
+
+  * Some clean-up, accuracy improvements, and optimization of the
+    exact Riemann solver in preparation for coupling it to simulations
+    (#2868, #2869, #2875)
+
+  * a new problem setup, `subch_planar`, for exploring He detonations
+    (#2870, #2873, #2881, #2883, #2886, #2877, #2878)
+
+  * clean-up the computation of the shock variable for hybrid Riemann
+    solves (#2865)
+
+  * code clean-ups (#2852, #2871, #2891)
+
+  * documentation improvements (#2843, #2879)
+
+  * add an EditorConfig (#2872)
+
+  * add the circular detonation problem to test multidimensional shock
+    algorithms (#2858)
+
+# 24.06
+
+  * Doc updates (#2839, #2842, #2846, #2851, #2854, #2860)
+
+  * Sync problem and Castro runtime parameters to recent Microphysics
+    changes (#2838, #2845, #2861)
+
+  * fix parsing in the `check_params.py` script (#2850)
+
+  * code cleaning (#2840, #2842, #2843)
+
+# 24.05
+
+  * Changed how the shock flag is computed.  It is now computed once,
+    at the start of a timestep.  It also takes into account sources
+    such that the pressure jump is only the pressure that is available
+    to drive a shock.  (#2726, #2728)
+
+  * Fixed a boundary condition check in gravity (#2833)
+
+  * Some coverity and clang-tidy fixes (#2810, #2811, #2835, #2831,
+    #2832)
+
+  * Fix the wdmerger initialization when the two stars are overlapping
+    (#2829)
+
+  * Fixed the Castro retry mechanism when amr.subcycling_mode = None
+    (#2821, #2824, #2826)
+
+  * Added more amr parameters to the `job_info` file (#2828)
+
+  * Added OpenMP to the SDC burn loop (#2770)
+
 # 24.04
 
   * Some clang-tidy fixes (#2779, #2780, #2781, #2784, #2786, #2787, #2790,

Diagnostics/DustCollapse/GNUmakefile

@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 # programs to be compiled
 ALL: dustcollapse_$(DIM)d.ex
 

Diagnostics/Radiation/GNUmakefile

@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 ALL: radhelp
 
 radhelp:

Diagnostics/Sedov/GNUmakefile

@@ -10,8 +10,6 @@ USE_OMP = FALSE
 
 USE_REACT = FALSE
 
-USE_ACC = FALSE
-
 # programs to be compiled
 ALL: sedov_$(DIM)d.ex
 

Docs/source/Hydrodynamics.rst

@@ -609,7 +609,7 @@ behavior:
 Compute Primitive Variables
 ---------------------------
 
-We compute the primtive variables from the conserved variables.
+We compute the primitive variables from the conserved variables.
 
 -  :math:`\rho, \rho e`: directly copy these from the conserved state
    vector

Docs/source/Verification.rst

@@ -495,7 +495,7 @@ test problem. A hot sphere is centered at the origin in a spherical
 geometry. The spectrum from this sphere follows a Planck
 distribution. The ambient medium is at a much lower temperature. A
 frequency-dependent opacity makes the domain optically thin for high
-frequecies and optically thick for low frequency. At long times, the
+frequencies and optically thick for low frequency. At long times, the
 solution will be a combination of the blackbody radiation from the
 ambient medium plus the radiation that propagated from the hot sphere.
 An analytic solution exists :cite:`graziani:2008` which gives the

Docs/source/conf.py

@@ -82,7 +82,7 @@ def get_version():
 
 # General information about the project.
 project = 'Castro'
-copyright = '2018-2022, Castro development team'
+copyright = '2018-2024, Castro development team'
 author = 'Castro development team'
 
 html_logo = "castro_logo_hot_200.png"
@@ -218,6 +218,12 @@ def get_version():
 ]
 
 
+# -- Options for linkcheck
+
+linkcheck_retries = 3
+linkcheck_timeout = 100
+
+
 # -- Options for Texinfo output -------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples

Docs/source/gpu.rst

@@ -0,0 +1,58 @@
+*********************
+GPU Programming Model
+*********************
+
+CPUs and GPUs have separate memory, which means that working on both
+the host and device may involve managing the transfer of data between
+the memory on the host and that on the GPU.
+
+In Castro, the core design when running on GPUs is that all of the compute
+should be done on the GPU.
+
+When we compile with ``USE_CUDA=TRUE`` or ``USE_HIP=TRUE``, AMReX will allocate
+a pool of memory on the GPUs and all of the ``StateData`` will be stored there.
+As long as we then do all of the computation on the GPUs, then we don't need
+to manage any of the data movement manually.
+
+.. note::
+
+   We can tell AMReX to allocate the data using managed-memory by
+   setting:
+
+   ::
+
+      amrex.the_arena_is_managed = 1
+
+   This is generally not needed.
+
+The programming model used throughout Castro is C++-lambda-capturing
+by value.  We access the ``FArrayBox`` stored in the ``StateData``
+``MultiFab`` by creating an ``Array4`` object.  The ``Array4`` does
+not directly store a copy of the data, but instead has a pointer to
+the data in the ``FArrayBox``.  When we capture the ``Array4`` by
+value in the GPU kernel, the GPU gets access to the pointer to the
+underlying data.
+
+
+Most AMReX functions will work on the data directly on the GPU (like
+``.setVal()``).
+
+In rare instances where we might need to operate on the data on the
+host, we can force a copy to the host, do the work, and then copy
+back.  For an example, see the reduction done in  ``Gravity.cpp``.
+
+.. note::
+
+   For a thorough discussion of how the AMReX GPU offloading works
+   see :cite:`amrex-ecp`.
+
+
+Runtime parameters
+------------------
+
+The main exception for all data being on the GPUs all the time are the
+runtime parameters.  At the moment, these are allocated as managed
+memory and stored in global memory.  This is simply to make it easier
+to read them in and initialize them on the CPU at runtime.
+
+

Docs/source/index.rst

@@ -21,6 +21,7 @@ https://github.com/amrex-astro/Castro
    mpi_plus_x
    FlowChart
    software
+   gpu
    problem_setups
    timestepping
    creating_a_problem