Move Comm into a file

Add force module that uses KRS

input/in.lj

@@ -5,7 +5,7 @@ atom_style	atomic
 
 newton off
 lattice		fcc 0.8442
-region		box block 0 80 0 80 0 80
+region		box block 0 1 0 1 0 1
 create_box	1 box
 create_atoms	1 box
 mass		1 2.0
@@ -20,4 +20,4 @@ neigh_modify	every 20 one 50
 fix		1 all nve
 thermo 10
 
-run		100
+run		2

src/CMakeLists.txt

@@ -15,14 +15,6 @@ if(ENABLE_MPI)
    target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_MPI)
 endif()
 
-# Select a default set of options. We can export this as CMake options later
-if (ENABLE_KOKKOS_REMOTE_SPACES)
-  target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
-  target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_SCALAR)
-  #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO)
-  target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_GLOBAL)
-endif()
-
 target_include_directories(ExaMiniMD PRIVATE ${Kokkos_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${SUBDIRECTORIES})
 target_link_libraries(ExaMiniMD PRIVATE $<$<BOOL:${ENABLE_MPI}>:MPI::MPI_CXX> Kokkos::kokkos $<$<BOOL:${ENABLE_KOKKOS_REMOTE_SPACES}>:Kokkos::kokkosremotespaces>)
 

src/binning_types/binning_kksort.cpp

@@ -71,7 +71,7 @@ namespace {
 void BinningKKSort::create_binning(T_X_FLOAT dx_in, T_X_FLOAT dy_in, T_X_FLOAT dz_in, int halo_depth, bool do_local, bool do_ghost, bool sort) {
   if(do_local||do_ghost) {
     nhalo = halo_depth;
-    std::pair<T_INT,T_INT> range(do_local?0:system->N_local,
+    Kokkos::pair<T_INT,T_INT> range(do_local?0:system->N_local,
                                  do_ghost?system->N_local+system->N_ghost:system->N_local);
 
     nbinx = T_INT(system->sub_domain_x/dx_in);

src/comm_lib.cpp

@@ -0,0 +1,88 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#include <comm_lib.h>
+#include <assert.h>
+
+#if EXAMINIMD_ENABLE_MPI
+#include <mpi.h>
+#endif
+
+#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
+#include <Kokkos_RemoteSpaces.hpp>
+#endif
+
+void comm_lib_init(int argc, char* argv[]) {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+  int mpi_thread_level_available;
+  int mpi_thread_level_required = MPI_THREAD_MULTIPLE;
+
+#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL
+  mpi_thread_level_required = MPI_THREAD_SINGLE;
+#endif
+
+  MPI_Init_thread(&argc, &argv, mpi_thread_level_required,
+                  &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_init_thread(mpi_thread_level_required, &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+#endif
+
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  MPI_Comm mpi_comm;
+  nvshmemx_init_attr_t attr;
+  mpi_comm      = MPI_COMM_WORLD;
+  attr.mpi_comm = &mpi_comm;
+  nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr);
+#endif
+}
+
+void comm_lib_finalize() {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_finalize();
+#endif
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  nvshmem_finalize();
+#endif
+  MPI_Finalize();
+#endif
+#endif
+}

src/comm_lib.h

@@ -0,0 +1,47 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#pragma once
+
+#ifndef COMM_INIT_H
+#define COMM_INIT_H
+
+void comm_lib_init(int argc, char* argv[]);
+void comm_lib_finalize();
+
+#endif

src/comm_types/CMakeLists.txt

@@ -1,9 +1,3 @@
 FILE(GLOB SRCS *.cpp)
 target_sources(ExaMiniMD PRIVATE ${SRCS})
-
-if (!ENABLE_MPI AND !ENABLE_KOKKOS_REMOTE_SPACES) 
-    # Skip MPI module
-    list(FILTER SRCS EXCLUDE REGEX ".*comm_mpi\\.cpp$")
-endif()
-
 target_sources(ExaMiniMD PRIVATE ${SRCS})

src/comm_types/comm_mpi.cpp

@@ -36,6 +36,8 @@
 //  Questions? Contact Christian R. Trott ([email protected])
 //************************************************************************
 
+#include <string>
+
 #if defined(EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
 #include<comm_mpi.h>
 
@@ -197,14 +199,14 @@ void CommMPI::exchange() {
   s = *system;
   N_local = system->N_local;
   N_ghost = 0;
-  //printf("System A: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
+  printf("System A: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
   Kokkos::parallel_for("CommMPI::exchange_self",
             Kokkos::RangePolicy<TagExchangeSelf, Kokkos::IndexType<T_INT> >(0,N_local), *this);
 
   T_INT N_total_recv = 0;
   T_INT N_total_send = 0;
 
-  //printf("System B: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
+  printf("System B: %i %lf %lf %lf %i\n",s.N_local,s.x(21,0),s.x(21,1),s.x(21,2),s.type(21));
   for(phase = 0; phase < 6; phase ++) {
     proc_num_send[phase] = 0;
     proc_num_recv[phase] = 0;
@@ -390,7 +392,8 @@ void CommMPI::exchange_halo() {
 };
 
 void CommMPI::update_halo() {
-#ifndef SHMEMTESTS_USE_HALO
+
+#if !defined(SHMEMTESTS_USE_HALO) && defined(EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
   return;
 #else
   Kokkos::Profiling::pushRegion("Comm::update_halo");
@@ -478,7 +481,13 @@ void CommMPI::update_force() {
   Kokkos::Profiling::popRegion();
 };
 
-const char* CommMPI::name() { return "CommMPI"; }
+const char* CommMPI::name() {
+    comm_name = std::string("CommMPI");
+    #ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
+    comm_name += "Distrib";
+    #endif
+    return comm_name.c_str();
+}
 
 int CommMPI::process_rank() { return proc_rank; }
 int CommMPI::num_processes() { return proc_size; }

src/comm_types/comm_mpi.h

@@ -70,7 +70,6 @@ class CommMPI: public Comm {
   System s;
 
   // Owned Variables
-
   int phase; // Communication Phase
   int proc_neighbors_recv[6]; // Neighbor for each phase
   int proc_neighbors_send[6]; // Neighbor for each phase
@@ -81,6 +80,8 @@ class CommMPI: public Comm {
   int proc_rank;         // My Process rank
   int proc_size;         // Number of processes
 
+  std::string comm_name;
+
   T_INT num_ghost[6];
   T_INT ghost_offsets[6];
 
@@ -503,7 +504,9 @@ class CommMPI: public Comm {
   KOKKOS_INLINE_FUNCTION
   void operator() (const TagCreateGlobalIndecies,
                    const T_INT& i) const {
+    #ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
     s.global_index(i) = N_MAX_MASK * proc_rank + i; 
+    #endif
   }
 
   const char* name();

src/comm_types/comm_serial.h

@@ -52,7 +52,6 @@
       }
 #endif
 
-
 #if !defined(MODULES_OPTION_CHECK) && !defined(COMM_MODULES_INSTANTIATION)
 #ifndef COMM_SERIAL_H
 #define COMM_SERIAL_H

src/examinimd.cpp

@@ -83,8 +83,6 @@ void ExaMiniMD::init(int argc, char* argv[]) {
 #undef FORCE_MODULES_INSTANTIATION
   else comm->error("Invalid ForceType");
   for(int line = 0; line < input->force_coeff_lines.extent(0); line++) {
-    //input->input_data.print_line(input->force_coeff_lines(line));
-    //printf("init_coeff: %i %i\n",line,input->input_data.words_in_line(input->force_coeff_lines(line)));
     force->init_coeff(input->input_data.words_in_line(input->force_coeff_lines(line)),
                       input->input_data.words[input->force_coeff_lines(line)]);
   }
@@ -160,7 +158,7 @@ void ExaMiniMD::init(int argc, char* argv[]) {
       } else {
         printf("\n");
         printf("Step Temp E_pair TotEng CPU\n");
-        printf("     %i %lf %lf %lf %lf\n",step,T,PE,PE+KE,0.0);
+        printf("%i %lf %lf %lf %lf\n",step,T,PE,PE+KE,0.0);
       }
     }
   }
@@ -221,11 +219,12 @@ void ExaMiniMD::run(int nsteps) {
         neighbor->create_neigh_list(system,binning,force->half_neigh,false);
       neigh_time += neigh_timer.seconds();
     } else {
-      // Exchange Halo
+      // Exchange Halo data
       comm_timer.reset();
       comm->update_halo();
       comm_time += comm_timer.seconds();
     }
+    Kokkos::Experimental::DefaultRemoteMemorySpace::fence();
 
     // Zero out forces
     force_timer.reset();
@@ -249,7 +248,7 @@ void ExaMiniMD::run(int nsteps) {
     integrator->final_integrate();
 
     // On output steps print output
-    if(step%input->thermo_rate==0) {
+    //if(step%input->thermo_rate==0) {
       T_FLOAT T = temp.compute(system);
       T_FLOAT PE = pote.compute(system,binning,neighbor,force)/system->N;
       T_FLOAT KE = kine.compute(system)/system->N;
@@ -264,7 +263,7 @@ void ExaMiniMD::run(int nsteps) {
           last_time = time;
         }
       }
-    }
+   // }
 
     if(input->dumpbinaryflag)
       dump_binary(step);

src/force_types/CMakeLists.txt

@@ -1,11 +1,25 @@
 FILE(GLOB SRCS *.cpp)
 
-
-#Skip lj_ideal, snap and cell
+#Skip snap and cell
 #TODO: SNAP is outdates and should likely be removed all together as it is
 list(FILTER SRCS EXCLUDE REGEX ".*lj_cell\\.cpp$")
-list(FILTER SRCS EXCLUDE REGEX ".*lj_idial_neigh\\.cpp$")
 list(FILTER SRCS EXCLUDE REGEX ".*snap_neigh\\.cpp$")
 
-target_sources(ExaMiniMD PRIVATE ${SRCS})
+# Skip force-type module if Kokkos Remote Spaces is not enabled
+if (ENABLE_KOKKOS_REMOTE_SPACES)
+    message(STATUS "Building with support for force_lj_neigh_distrib")
+    list(FILTER SRCS EXCLUDE REGEX ".*lj_neigh\\.cpp$")
+    target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+    target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_SCALAR)
+    #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO)
+    #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO_LOCAL)
+    #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_LOCAL_GLOBAL)
+    target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_GLOBAL)
+else()
+    #Otherwise exclude
+    list(FILTER SRCS EXCLUDE REGEX ".*distrib\\.cpp$")
+endif()
+
 
+
+target_sources(ExaMiniMD PRIVATE ${SRCS})

src/force_types/force_lj_neigh.cpp

@@ -37,7 +37,6 @@
 //************************************************************************
 
 #include<force_lj_neigh_impl.h>
-
 #define FORCETYPE_DECLARE_TEMPLATE_MACRO(NeighType) ForceLJNeigh<NeighType>
 #define FORCE_MODULES_TEMPLATE
 #include<modules_neighbor.h>