Fix GCC GPU running on Delta by reverting to host-side buffers

CMakeLists.txt

@@ -22,6 +22,9 @@ set(PARTHENON_LINT_DEFAULT OFF CACHE BOOL "KHARMA Override")
 set(PARTHENON_DISABLE_HDF5_COMPRESSION OFF CACHE BOOL "KHARMA Override")
 # Don't build sparse (selectively-allocated) variable support
 set(PARTHENON_DISABLE_SPARSE ON CACHE BOOL "KHARMA Override")
+# Set to move MPI buffers to host; slower but less crashy
+# Favor setting this per-machine in machines/
+set(PARTHENON_ENABLE_HOST_COMM_BUFFERS OFF CACHE BOOL "KHARMA Override")
 
 # Parthenon internal build options
 set(BUILD_TESTING OFF CACHE BOOL "KHARMA Override")

machines/delta.sh

@@ -13,8 +13,10 @@ then
   HOST_ARCH=ZEN3
   DEVICE_ARCH=AMPERE80
   MPI_EXE=mpirun
+  NPROC=64
 
   module purge
+  module load cmake
 
   if [[ $ARGS == *"cuda"* ]]
   then
@@ -23,8 +25,11 @@ then
     MPI_EXTRA_ARGS="--map-by ppr:4:node:pe=16"
     MPI_NUM_PROCS=4
 
+    # Device-side buffers are broken on some Nvidia machines
+    EXTRA_FLAGS="-DPARTHENON_ENABLE_HOST_COMM_BUFFERS=ON $EXTRA_FLAGS"
+
     # Load common GPU modules
-    module load modtree/gpu hdf5 cmake
+    module load modtree/gpu cmake
 
     if [[ $ARGS == *"latest"* ]]; then
       # nvhpc only on request, MPI crashes
@@ -35,13 +40,13 @@ then
       C_NATIVE=gcc
       CXX_NATIVE=g++
     else
-      module load nvhpc
-      #C_NATIVE=nvc
-      #CXX_NATIVE=nvc++
+      module load nvhpc_latest/22.11 openmpi
+      C_NATIVE=nvc
+      CXX_NATIVE=nvc++
     fi
   else
     # CPU Compile
-    module load modtree/cpu gcc hdf5 cmake
+    module load modtree/cpu gcc
     MPI_NUM_PROCS=1
   fi
 fi

make.sh

@@ -76,8 +76,7 @@ if [[ "$(which python3 2>/dev/null)" == *"conda"* ]]; then
   echo
   echo "make.sh note:"
   echo "It looks like you have Anaconda loaded."
-  echo "Anaconda loads a serial version of HDF5 which may make this compile impossible."
-  echo "If you run into trouble, deactivate your environment with 'conda deactivate'"
+  echo "This is usually okay, but double-check the line 'Found MPI_CXX:' below!"
 fi
 # Save arguments if we've changed them
 # Used in run.sh for loading the same modules/etc.
@@ -314,7 +313,7 @@ if [[ "$ARGS" == *"clean"* ]]; then
     -DKokkos_ENABLE_CUDA=$ENABLE_CUDA \
     -DKokkos_ENABLE_SYCL=$ENABLE_SYCL \
     -DKokkos_ENABLE_HIP=$ENABLE_HIP \
-    "$EXTRA_FLAGS"
+    $EXTRA_FLAGS
 
   if [[ "$ARGS" == *"dryrun"* ]]; then
     set +x