Skip to content

Using ALCF

Jump to bottom Edit New page
v4m4 edited this page Nov 14, 2014 · 15 revisions

Table of Contents

Links

Kevin's Stock MTXM code

Whether you are on Blue Gene/P or Blue Gene/Q, you will want to compile Kevin Stock's MTXM code. This must be done outside of the MADNESS build system proper with the XL compiler as GCC does not support double hummer or QPX intrinsic. This is straightforward to accomplish, but the code is buried in a non-intuitive directory. 

cd ${MADNESS_TOP_LEVEL}/src/madness/tensor/new_mtxmq/bests
setenv TARGET BGQ)
make

Blue Gene/Q

MADNESS with Elemental enable hangs using the non-legacy MPI wrapper scripts, so please use the legacy MPI wrapper scripts until this issue is resolved

IBM XL v. 12

Presently working but requires V1R1M2 or later driver. May 2012 or later PTF of IBM XL C/C++ for Blue Gene, V12.1 is required.

You should always use the latest version of the IBM compilers, which must be set in Softenv (at ALCF - other sites might use modules, etc.) to ensure a consistent compiler toolchain. You can put e.g. @ibm-compilers-nov2012 in ~/.soft or do soft add @ibm-compilers-nov2012 from the command line. When newer compilers are available at ALCF, replace nov2012 as appropriate. 

#!/bin/bash -x

export LANG=C

export XL_TOP="${IBM_MAIN_DIR}"

export LIBS="${LIBS} -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack"
export LIBS="${LIBS} -L/home/naromero/madness-gcc-trunk/src/lib/tensor/new_mtxmq/bests -lMADMTXM"
export LIBS="${LIBS} -L/soft/libraries/essl/current/lib64 -lesslbg"
export LIBS="${LIBS} -L${XL_TOP}/xlmass/bg/7.3/bglib64 -lmass"
export LIBS="${LIBS} -L${XL_TOP}/xlf/bg/14.1/bglib64 -lxlf90_r -lxlopt -lxl"
export LIBS="${LIBS} -L${XL_TOP}/xlsmp/bg/3.1/bglib64 -lxlsmp "

# this should not be necessary any more but there is no hard in including it
export CPPFLAGS="-D"

export CXXFLAGS="${CPPFLAGS} -O2 -qdebug=IgnoreCvOnTopOfFunctionTypes" 
export CFLAGS=" -O2"
export FFLAGS=" -O2"

export MPICC=mpixlc_r 
export MPICXX=mpixlcxx_r 
export CC=mpixlc_r 
export CXX=mpixlcxx_r 
#export F77=bgxlf90_r 
export F77=mpixlf77_r

./configure \
   --with-google-test=/home/naromero/gtest-1.6.0-XLMAY2012 \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=IBM \
   --host=powerpc64-bgq-linux \
   --with-fortran-int32

It is possible to use the XL built-in TR1(which is actually the TR1 from GCC 4.4.6) instead of the MADNESS TR1 implementation, but it requires the use of an interim fix as well as an additional compiler flag. I recommend going thise route. 

export CXXFLAGS="${CPPFLAGS} -O2 -qdebug=IgnoreCvOnTopOfFunctionTypes -qlanglvl=variadictemplates -qpath=C:<path_to_interimfix>/MADNESS_MPQC_esp/interimfix/130214/exe"

where <path_to_interimfix></path_to_interimfix> is 

/gpfs/vesta_scratch/projects
on Vesta and 
/projects
on Cetus and Mira.

Notes

  • MADNESS compiled with XL occasionally gives NAN.
  • MADNESS with Elemental support cannot be linked by XL. PMR filed.

Clang build Instructions

Environmental variables tested in Cetus, Vesta, and Mira.

See instruction on HOWTO install Elemental 0.84 here link.

Your ~/.soft file must have: 

+mpiwrapper-bgclang
And it must have disabled any other wrappers of MPI, ie, +mpiwrapper-xl.

This is an example of HOWTO compile with Clang+Elemental+LibXC+libMADMXM. 

export LIBS="${MATH_LIBS} ${LIBS} -L/home/avazquez/public/install/lib  -lMADMTXM "
export CPPFLAGS="-D__bgq__"

export CXXFLAGS="${CPPFLAGS} -O2 -std=c++0x "
export CFLAGS="-O2"
export FFLAGS="-O2"


export CLANG_ROOT=/home/projects/llvm
export CLANG_MPI_ROOT=/home/projects/llvm/mpi/bgclang

export CC=${CLANG_ROOT}/wbin/bgclang
export CXX=${CLANG_ROOT}/wbin/bgclang++11
export F77=${GCC_ROOT}/bin/${GCC_NAME}-gfortran
export MPICC=${CLANG_MPI_ROOT}/bin/mpiclang
export MPICXX=${CLANG_MPI_ROOT}/bin/mpiclang++11

./configure \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=GNU \
   --with-elemental="/home/avazquez/public/install/local/elemental-0.84"\
   --with-libxc="/home/avazquez/public/install"\
   --host=powerpc64-bgq-linux \
   --with-fortran-int32

GCC build Instructions

Environmental variables tested in Cetus, Vesta, and Mira.

Incomplete, MADNESS support only GCC/GNU 4.8.3

Running with the Elemental, a parallel dense linear algebra package

Elemental 0.84 is only supported. Elemental support can be activated with the following configure flag: 

--with-elemental=elemental-0.84_path

MADNESS using Elemental MRRR algorithm for solving the Generalized EVP which is part of the LCAO initialization. MRRR algorithm is implemented in an external library which is bundled with Elemental and built by default. MRRR spawns its own set of threads by default which interfere with the MADNESS RT. It is necessary to set the following flag in your job submission script: 

--env PMR_NUM_THREADS=1

Elemental Build Script and Toolchain for Clang + ESSL

Source for Elemental http://libelemental.org/pub/releases/Elemental-0.84-p1.tgz

Create a build subdirectory. And execute cmake command inside as: build.sh: 

/soft/buildtools/cmake/current/bin/cmake \
      -D CMAKE_TOOLCHAIN_FILE=../BGQ-Mira-clang-essl.cmake \
      -D ELEM_EXAMPLES=ON -D ELEM_TESTS=ON \
      -D CMAKE_INSTALL_PREFIX=$HOME/public/install/local/elemental-0.84
Where BGQ-Mira-clang-essl.cmake is link. Then run: 
make -j4 && make install

BGQ-Mira-clang-essl.cmake Toolchain for no threaded version of Elemental 

set(GCC_ROOT  "/bgsys/drivers/ppcfloor/gnu-linux")
set(GCC_NAME  "powerpc64-bgq-linux")
set(CLANG_ROOT "/home/projects/llvm")
set(CLANG_MPI_ROOT "/home/projects/llvm/mpi/bgclang")
#set(IBMCMP_ROOT "$ENV{IBM_MAIN_DIR}")
set(IBMCMP_ROOT "/soft/compilers/ibmcmp-feb2014")

set(BLAS_LIB "/soft/libraries/alcf/current/xl/BLAS/lib")
set(LAPACK_LIB "/soft/libraries/alcf/current/xl/LAPACK/lib")
set(ESSL_LIB "/soft/libraries/essl/current/essl/5.1/lib64")

set(MPI_ROOT   "/bgsys/drivers/ppcfloor/comm")
set(PAMI_ROOT  "/bgsys/drivers/ppcfloor/comm/sys")
set(SPI_ROOT   "/bgsys/drivers/ppcfloor/spi")

# The serial compilers
set(CMAKE_C_COMPILER       "${CLANG_ROOT}/wbin/bgclang")
set(CMAKE_CXX_COMPILER     "${CLANG_ROOT}/wbin/bgclang++11")
set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/${GCC_NAME}-gfortran")

# The MPI wrappers for the C and C++ compilers
set(MPI_C_COMPILER   "${CLANG_MPI_ROOT}/bin/mpiclang")
set(MPI_CXX_COMPILER "${CLANG_MPI_ROOT}/bin/mpiclang++11")

set(MPI_C_COMPILE_FLAGS    "")
set(MPI_CXX_COMPILE_FLAGS  "")
set(MPI_C_INCLUDE_PATH     "${MPI_ROOT}/include")
set(MPI_CXX_INCLUDE_PATH   "${MPI_ROOT}/include")
set(MPI_C_LINK_FLAGS       "-L${MPI_ROOT}/lib -L${PAMI_ROOT}/lib -L${SPI_ROOT}/lib")
set(MPI_CXX_LINK_FLAGS     "${MPI_C_LINK_FLAGS}")
set(MPI_C_LIBRARIES       "${MPI_C_LINK_FLAGS}   -lmpich-gcc -lopa-gcc -lmpl-gcc -lpami-gcc -lSPI -lSPI_cnk -lrt -lpthread -lstdc++ -lpthread")
set(MPI_CXX_LIBRARIES     "${MPI_CXX_LINK_FLAGS} -lmpichcxx-gcc ${MPI_C_LIBRARIES}")


if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES HybridDebug)
  set(CXX_FLAGS "-g")
else()
  set(CXX_FLAGS "-O3")
endif()

#set(CMAKE_THREAD_LIBS_INIT "-fopenmp")
#set(OpenMP_CXX_FLAGS "-fopenmp")

##############################################################

# set the search path for the environment coming with the compiler
# and a directory where you can install your own compiled software
set(CMAKE_FIND_ROOT_PATH
    /bgsys/drivers/ppcfloor
    /bgsys/drivers/ppcfloor/gnu-linux/powerpc64-bgq-linux
    /bgsys/drivers/ppcfloor/comm/xl
    /bgsys/drivers/ppcfloor/comm/sys
    /bgsys/drivers/ppcfloor/spi
)

# adjust the default behaviour of the FIND_XXX() commands:
# search headers and libraries in the target environment, search
# programs in the host environment
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

##############################################################

set(XLF_LIB "${IBMCMP_ROOT}/xlf/bg/14.1/bglib64")
set(XLSMP_LIB "${IBMCMP_ROOT}/xlsmp/bg/3.1/bglib64")
if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES PureRelease)
  set(MATH_LIBS "-L${ESSL_LIB} -lesslbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlomp_ser -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
else()
  set(MATH_LIBS "-L${ESSL_LIB} -lesslsmpbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslsmpbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlsmp -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
endif()

Running with Intel TBB

This is not well tested at all, but preliminary findings indicate that it probably works.

Obtain BG version of Intel TBB from the ANL repo: 

svn co https://repo.anl-external.org/repos/BlueTBB/tbb41_20130314oss tbb41_20130314oss

Build it: 

make extra_inc=big_iron.inc arch=bg

Append the following configure options: 

   --with-tbb=yes \
   --with-tbb-include=/home/naromero/tbb41_20130314oss/include \
   --with-tbb-lib=/home/naromero/tbb41_20130314oss/build/linux_bg_gcc_cc4.4.6_libc2.12_kernel2.6.32_debug

Build as you would normally. You will need to add the following flag to run due to some limitation in the BG version of Intel TBB: 

__TBB_DEFAULT_NUM_THREADS=1
Add a custom footer
Add a custom sidebar
Clone this wiki locally