Using ALCF

Table of Contents Links Kevin's Stock MTXM code MPI wrapper scripts Clang build Instructions GCC 4.7.2 build Instructions Running with the Elemental, a parallel dense linear algebra package Elemental Build Script and Toolchain for Clang + ESSL Running with Intel TBB

Links

Helpful Blue Gene/Q web sites:

ALCF Documentation
IBM library documentation
Documentation at Swedish Royal Institute of Technology
Jeff Hammond's Blue Gene/Q page

Kevin's Stock MTXM code

Whether you are on Blue Gene/P or Blue Gene/Q, you will want to compile Kevin Stock's MTXM code. This must be done outside of the MADNESS build system proper with the XL compiler as GCC does not support double hummer or QPX intrinsic. This is straightforward to accomplish, but the code is buried in a non-intuitive directory.

cd ${MADNESS_TOP_LEVEL}/src/madness/tensor/new_mtxmq/bests
setenv TARGET BGQ)
make

MPI wrapper scripts

MADNESS uses MPI_THREAD_MULTIPLE mode. In principle, this should work with the legacy MPI wrapper scripts. However, we have discovered some hangs so we recommend using the non-legacy MPI wrapper scripts instead. This is the default wrapper script on the ALCF Blue Gene/Q.

Clang build Instructions

Environmental variables tested in Cetus, Vesta, and Mira.

See instruction on HOWTO install Elemental 0.84 here link.

Your ~/.soft file must have:

+mpiwrapper-bgclang

And it must have disabled any other wrappers of MPI, ie, +mpiwrapper-xl.

This is an example of HOWTO compile with Clang+Elemental+LibXC+libMADMXM.

#!/bin/bash -x
export LANG=C
export IBMCMP_ROOT="/soft/compilers/ibmcmp-aug2014"
export LAPACK_LIB=/soft/libraries/alcf/current/xl/LAPACK/lib
export ESSL_LIB=/soft/libraries/essl/current/essl/5.1/lib64
export XLF_LIB=${IBMCMP_ROOT}/xlf/bg/14.1/bglib64
export XLSMP_LIB=${IBMCMP_ROOT}/xlsmp/bg/3.1/bglib64
export XLMASS_LIB=${IBMCMP_ROOT}/xlmass/bg/7.3/bglib64
export MATH_LIBS="-L${XLMASS_LIB} -lmassv -lmass -L${LAPACK_LIB} -llapack \
                  -L${ESSL_LIB} -lesslbg -L${XLF_LIB} -lxlf90_r \
                  -L${XLSMP_LIB} -lxlsmp -ldl  -lxlopt -lxlfmath -lxl \
                  -Wl,--allow-multiple-definition"

export LIBS="${MATH_LIBS} ${LIBS} -L/home/avazquez/public/install/lib  -lMADMTXM "
export CPPFLAGS="-D"

export CXXFLAGS="${CPPFLAGS} -O2 -std=c++0x "
export CFLAGS="-O2"
export FFLAGS="-O2"


export CLANG_ROOT=/home/projects/llvm
export CLANG_MPI_ROOT=/home/projects/llvm/mpi/bgclang

export CC=${CLANG_ROOT}/wbin/bgclang
export CXX=${CLANG_ROOT}/wbin/bgclang++11
export F77=${GCC_ROOT}/bin/${GCC_NAME}-gfortran
export MPICC=${CLANG_MPI_ROOT}/bin/mpiclang
export MPICXX=${CLANG_MPI_ROOT}/bin/mpiclang++11

./configure \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=GNU \
   --with-elemental="/home/avazquez/public/install/local/elemental-0.84"\
   --with-libxc="/home/avazquez/public/install"\
   --host=powerpc64-bgq-linux \
   --with-fortran-int32

GCC 4.7.2 build Instructions

You cannot compile MADNESS with the default version of GCC (4.4.7) because it lacks sufficient C++11 support. Because there are no compiler wrappers for GCC 4.7.2, you will need to use the existing GCC 4.4.7 MPI wrapper scripts and override the underlying compilers. This what the first two lines of this batch script accomplish.

!/bin/bash
export MPICH_CXX="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-g++"
export MPICH_CC="/bgsys/drivers/toolchain/V1R2M2-4.7.2/gnu-linux-4.7.2-efix014/bin/powerpc64-bgq-linux-gcc"
export LANG=C

export XL_TOP="${IBM_MAIN_DIR}"

export LIBS="${LIBS} -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack"
export LIBS="${LIBS} -L/home/naromero/madness-git-naromero/src/lib/tensor/new_mtxmq/bests -lMADMTXM"
export LIBS="${LIBS} -L/soft/libraries/essl/5.1.1-1/lib64 -lesslbg"
export LIBS="${LIBS} -L${XL_TOP}/xlmass/bg/7.3/bglib64 -lmass"
export LIBS="${LIBS} -L${XL_TOP}/xlf/bg/14.1/bglib64 -lxlf90_r -lxlopt -lxl -ldl"
export LIBS="${LIBS} -L${XL_TOP}/xlsmp/bg/3.1/bglib64 -lxlsmp "

export NOMALLOC="-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"

default
export CXXFLAGS="-mcpu=a2 -O3 -std=c++11 ${NOMALLOC}"
export CFLAGS="-mcpu=a2 -O3 ${NOMALLOC}"
export FFLAGS="-mcpu=a2 -O3"

export MPICC=mpicc
export MPICXX=mpicxx
export CC=mpicc
export CXX=mpicxx
export F77=bgxlf90_r 
export F77=mpif77
TBB_HOME=tbb43_20150316oss
../madness-git-naromero/configure \
   --with-google-test=/home/naromero/gtest-1.6.0-GCC4.7.2 \
   --enable-debugging=yes \
   --enable-optimal=no \
   --enable-optimization=no \
   --enable-warning=GNU \
   --host=powerpc64-bgq-linux \
   --with-fortran-int32 \
   --with-libxc="/home/naromero/libxc-2.0.x/bin" \
   --with-elemental=/home/naromero/elemental-0.84-gcc \
   --with-google-perf=/home/naromero/gperftools-2.2.1 \
   --with-tbb=yes \
   --with-tbb-lib=/home/naromero/${TBB_HOME}/build/linux_bg_gcc_4.7.2_release \
   --with-tbb-include=/home/naromero/${TBB_HOME}/include

Running with the Elemental, a parallel dense linear algebra package

Elemental 0.84 is only supported. Elemental support can be activated with the following configure flag:

--with-elemental=elemental-0.84_path

MADNESS using Elemental MRRR algorithm for solving the Generalized EVP which is part of the LCAO initialization. MRRR algorithm is implemented in an external library which is bundled with Elemental and built by default. MRRR spawns its own set of threads by default which interfere with the MADNESS RT. It is necessary to set the following flag in your job submission script:

--env PMR_NUM_THREADS=1

Elemental Build Script and Toolchain for Clang + ESSL

Source for Elemental http://libelemental.org/pub/releases/Elemental-0.84-p1.tgz

Create a build subdirectory. And execute cmake command inside as: build.sh:

/soft/buildtools/cmake/current/bin/cmake \
      -D CMAKE_TOOLCHAIN_FILE=../BGQ-Mira-clang-essl.cmake \
      -D ELEM_EXAMPLES=ON -D ELEM_TESTS=ON \
      -D CMAKE_INSTALL_PREFIX=$HOME/public/install/local/elemental-0.84

Where BGQ-Mira-clang-essl.cmake is link. Then run:

make -j 4 && make install

BGQ-Mira-clang-essl.cmake Toolchain for no threaded version of Elemental

 #set(CMAKE_SYSTEM_NAME BlueGeneQ-static)
set(CMAKE_BUILD_TYPE "PureRelease")

set(GCC_ROOT  "/bgsys/drivers/ppcfloor/gnu-linux")
set(GCC_NAME  "powerpc64-bgq-linux")
set(CLANG_ROOT "/home/projects/llvm")
set(CLANG_MPI_ROOT "/home/projects/llvm/mpi/bgclang")
 #set(IBMCMP_ROOT "$ENV{IBM_MAIN_DIR}")
set(IBMCMP_ROOT "/soft/compilers/ibmcmp-feb2014")

set(BLAS_LIB "/soft/libraries/alcf/current/xl/BLAS/lib")
set(LAPACK_LIB "/soft/libraries/alcf/current/xl/LAPACK/lib")
set(ESSL_LIB "/soft/libraries/essl/current/essl/5.1/lib64")

set(MPI_ROOT   "/bgsys/drivers/ppcfloor/comm")
set(PAMI_ROOT  "/bgsys/drivers/ppcfloor/comm/sys")
set(SPI_ROOT   "/bgsys/drivers/ppcfloor/spi")

The serial compilers
set(CMAKE_C_COMPILER       "${CLANG_ROOT}/wbin/bgclang")
set(CMAKE_CXX_COMPILER     "${CLANG_ROOT}/wbin/bgclang++11")
set(CMAKE_Fortran_COMPILER "${GCC_ROOT}/bin/${GCC_NAME}-gfortran")

The MPI wrappers for the C and C++ compilers
set(MPI_C_COMPILER   "${CLANG_MPI_ROOT}/bin/mpiclang")
set(MPI_CXX_COMPILER "${CLANG_MPI_ROOT}/bin/mpiclang++11")

set(MPI_C_COMPILE_FLAGS    "")
set(MPI_CXX_COMPILE_FLAGS  "")
set(MPI_C_INCLUDE_PATH     "${MPI_ROOT}/include")
set(MPI_CXX_INCLUDE_PATH   "${MPI_ROOT}/include")
set(MPI_C_LINK_FLAGS       "-L${MPI_ROOT}/lib -L${PAMI_ROOT}/lib -L${SPI_ROOT}/lib")
set(MPI_CXX_LINK_FLAGS     "${MPI_C_LINK_FLAGS}")
set(MPI_C_LIBRARIES       "${MPI_C_LINK_FLAGS}   -lmpich-gcc -lopa-gcc -lmpl-gcc -lpami-gcc -lSPI -lSPI_cnk -lrt -lpthread -lstdc++ -lpthread")
set(MPI_CXX_LIBRARIES     "${MPI_CXX_LINK_FLAGS} -lmpichcxx-gcc ${MPI_C_LIBRARIES}")


if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES HybridDebug)
  set(CXX_FLAGS "-g")
else()
  set(CXX_FLAGS "-O3")
endif()


set(CMAKE_THREAD_LIBS_INIT "-fopenmp")
set(OpenMP_CXX_FLAGS "-fopenmp")





set the search path for the environment coming with the compiler
and a directory where you can install your own compiled software

set(CMAKE_FIND_ROOT_PATH
    /bgsys/drivers/ppcfloor
    /bgsys/drivers/ppcfloor/gnu-linux/powerpc64-bgq-linux
    /bgsys/drivers/ppcfloor/comm/xl
    /bgsys/drivers/ppcfloor/comm/sys
    /bgsys/drivers/ppcfloor/spi
)


adjust the default behaviour of the FIND_XXX() commands:
search headers and libraries in the target environment, search
programs in the host environment

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)



set(XLF_LIB "${IBMCMP_ROOT}/xlf/bg/14.1/bglib64")
set(XLSMP_LIB "${IBMCMP_ROOT}/xlsmp/bg/3.1/bglib64")
if(CMAKE_BUILD_TYPE MATCHES PureDebug OR
   CMAKE_BUILD_TYPE MATCHES PureRelease)
  set(MATH_LIBS "-L${ESSL_LIB} -lesslbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlomp_ser -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
else()
  set(MATH_LIBS "-L${ESSL_LIB} -lesslsmpbg -L${LAPACK_LIB} -llapack -L${ESSL_LIB} -lesslsmpbg -L${XLF_LIB} -lxlf90_r -L${XLSMP_LIB} -lxlsmp -lxlopt -lxlfmath -lxl -lpthread -ldl -Wl,--allow-multiple-definition")
endif()

Running with Intel TBB

This is now robustly tested. Please report any errors to the MADNESS mailing list.

You will need Intel TBB 4.3 Update 4:

wget --no-check-certificate https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb43_20150316oss_src.tgz

Older version will not compile 'out of the box'.

Build it:

make extra_inc=big_iron.inc arch=bg

Append the following configure options:

   --with-tbb=yes \
   --with-tbb-include=/home/naromero/tbb41_20130314oss/include \
   --with-tbb-lib=/home/naromero/tbb41_20130314oss/build/linux_bg_gcc_cc4.4.6_libc2.12_kernel2.6.32_debug

Build as you would normally. Note that you will be limited to running on 1 rank per node (c1 mode).

Provide feedback

Saved searches

Use saved searches to filter your results more quickly