enable make check for folder examples/Pytorch_DDP

Author: wkliao

examples/Makefile

@@ -28,29 +28,32 @@ OUTPUT_DIR = _tmp_output
 all:
 
 check: ptest4
+	cd Pytorch_DDP && make check
+
 ptests: ptest3 ptest4 ptest8
+	cd Pytorch_DDP && make ptests
 
 ptest3:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    examples: Parallel testing on 3 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=3; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""
 
 ptest4:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    examples: Parallel testing on 4 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=4; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""
 
 ptest8:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    examples: Parallel testing on 8 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=8; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""
 

examples/Pytorch_DDP/Makefile

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2024, Northwestern University and Argonne National Laboratory
+# See COPYRIGHT notice in top-level directory.
+#
+
+check_PROGRAMS = torch_ddp_skeleton.py
+
+
+TESTS_ENVIRONMENT  = export check_PROGRAMS="${check_PROGRAMS}";
+TESTS_ENVIRONMENT += export PNETCDF_DIR="${PNETCDF_DIR}";
+
+OUTPUT_DIR = _tmp_output
+
+all:
+
+check: ptest4
+ptests: ptest3 ptest4 ptest8
+
+ptest3:
+	@mkdir -p ${OUTPUT_DIR}
+	@echo "======================================================================"
+	@echo "    examples/Pytorch_DDP: Parallel testing on 3 MPI processes"
+	@echo "======================================================================"
+	@${TESTS_ENVIRONMENT} export NPROC=3; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
+	@echo ""
+
+ptest4:
+	@mkdir -p ${OUTPUT_DIR}
+	@echo "======================================================================"
+	@echo "    examples/Pytorch_DDP: Parallel testing on 4 MPI processes"
+	@echo "======================================================================"
+	@${TESTS_ENVIRONMENT} export NPROC=4; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
+	@echo ""
+
+ptest8:
+	@mkdir -p ${OUTPUT_DIR}
+	@echo "======================================================================"
+	@echo "    examples/Pytorch_DDP: Parallel testing on 8 MPI processes"
+	@echo "======================================================================"
+	@${TESTS_ENVIRONMENT} export NPROC=8; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
+	@echo ""
+
+clean:
+	rm -rf ${OUTPUT_DIR}
+

examples/Pytorch_DDP/parallel_run.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright (C) 2024, Northwestern University and Argonne National Laboratory
+# See COPYRIGHT notice in top-level directory.
+#
+
+# Exit immediately if a command exits with a non-zero status.
+set -e
+
+# Get the directory containing this script
+if test "x$NPROC" = x ; then
+    NPROC=4
+fi
+
+# get output folder from command line
+if test "$#" -gt 0 ; then
+   args=("$@")
+   OUT_DIR="${args[0]}"
+   # check if output folder exists
+   if ! test -d $OUT_DIR ; then
+      echo "Error: output folder \"$OUT_DIR\" does not exist."
+      exit 1
+   fi
+else
+   # output folder is not set at command line, use current folder
+   OUT_DIR="."
+fi
+# echo "OUT_DIR=$OUT_DIR"
+
+for prog in $check_PROGRAMS; do
+   printf '%-60s' "Testing $prog"
+
+   if test $prog = "torch_ddp_skeleton.py" ; then
+      CMD="mpiexec -n $NPROC python $prog -q"
+   fi
+   $CMD
+   status=$?
+   if [ $status -ne 0 ]; then
+      echo " ---- FAIL"
+   else
+      echo " ---- PASS"
+   fi
+done
+

examples/Pytorch_DDP/torch_ddp_skeleton.py

@@ -5,11 +5,13 @@
 # This is a skeleton program to show how to run Pytorch distributed environment
 # with MPI
 
-import os
+import os, argparse
 import torch
 import torch.distributed as dist
 from mpi4py import MPI
 
+verbose = True
+
 class distributed():
     def get_size(self):
         if dist.is_available() and dist.is_initialized():
@@ -217,13 +219,23 @@ def init_parallel():
 
 #----< main() >----------------------------------------------------------------
 def main():
+    global verbose
+
     # initialize parallel environment
     comm, device = init_parallel()
 
     rank = comm.get_rank()
     nprocs = comm.get_size()
 
-    print("nprocs = ", nprocs, " rank = ",rank," device = ", device)
+    # Get command-line arguments
+    args = None
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-q", help="Quiet mode (reports when fail)", action="store_true")
+    args = parser.parse_args()
+    if args.q: verbose = False
+
+    if verbose:
+        print("nprocs = ", nprocs, " rank = ",rank," device = ", device)
 
     comm.finalize()
 

test/Makefile

@@ -58,25 +58,25 @@ ptests: ptest3 ptest4 ptest8
 
 ptest3:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    test: Parallel testing on 3 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=3; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""
 
 ptest4:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    test: Parallel testing on 4 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=4; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""
 
 ptest8:
 	@mkdir -p ${OUTPUT_DIR}
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@echo "    test: Parallel testing on 8 MPI processes"
-	@echo "==========================================================="
+	@echo "======================================================================"
 	@${TESTS_ENVIRONMENT} export NPROC=8; ./parallel_run.sh ${OUTPUT_DIR} || exit 1
 	@echo ""