added configuration and build scripts for supermuc

dash-project · Jul 1, 2017 · 2780f9d · 2780f9d
1 parent e5d9752
commit 2780f9d
Show file tree

Hide file tree

Showing 10 changed files with 811 additions and 0 deletions.
diff --git a/hpccg/compile-all.sh b/hpccg/compile-all.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+[[ !  -z  "$1"  ]] || { echo "usage: $0 <ENVIRONMENT>  for building Makefile.ENVIRONMENT"; exit 1; }
+ENVIRONMENT="$1"
+
+for DIR in $(ls -d */); do
+  if [[ "$DIR" == "jobfiles/" ]];then continue; fi
+  if [[ "$DIR" == "results/" ]];then continue; fi
+  test -e $DIR/Makefile.$ENVIRONMENT || { echo "no Makefile for $ENVIRONMENT in $DIR"; continue; }
+
+  cd $DIR
+  echo "Build $DIR"
+  make clean
+  make -f Makefile.$ENVIRONMENT -j 10
+  cd ..
+done
diff --git a/hpccg/jobfiles/supermuc/README b/hpccg/jobfiles/supermuc/README
@@ -0,0 +1,11 @@
+# Run HPCCG on LRZ Supermuc
+
+## Load suitable environment
+
+## Compile and Install
+
+- execute `dash-apps/HPCCG/compile-all.sh supermuc`
+
+## Run 
+
+- Schedule desired jobfile using `bash schedule-job.sh <template>`
diff --git a/hpccg/jobfiles/supermuc/weakScaleAll-ibm-large.tpl.job b/hpccg/jobfiles/supermuc/weakScaleAll-ibm-large.tpl.job
@@ -0,0 +1,43 @@
+# Run HPCCG with IBM MPI on supermuc
+#!/bin/bash
+##
+#@ energy_policy_tag = hpccg 
+#@ minimize_time_to_solution = yes
+#@ job_type = parallel
+#@ class = micro 
+#@ node = 1
+###@ island_count= not needed for
+#### class general
+#@ island_count = 1 
+#@ total_tasks= <procs>
+## other example
+##@ tasks_per_node = 28
+#@ wall_clock_limit = 1:00:00
+#@ job_name = hpccg-weakScale-ibm-large-<procs> 
+#@ network.MPI = sn_all,not_shared,us
+#@ initialdir = ../../
+#@ output = $(home)/log/job.$(job_name).$(jobid).out
+#@ error =  $(home)/log/job.$(job_name).$(jobid).err
+#@ notification=always
+#@ notify_user=felix.moessbauer@campus.lmu.de
+#@ queue
+. /etc/profile
+. /etc/profile.d/modules.sh
+module load mpi.ibm
+module load papi
+module load hwloc
+
+
+for IMPL in $(ls -d */); do
+  if [[ "$IMPL" == "jobfiles/" ]]; then continue; fi 
+  if [[ "$IMPL" == "results/" ]]; then continue; fi 
+  echo "Execute $IMPL"
+  TOPDIR=$(pwd)
+  RESULTDIR="$TOPDIR/results/ibm/<procs>/$IMPL"
+  mkdir -p $RESULTDIR 
+  # place output files in correct dir
+  cd $RESULTDIR 
+  poe $TOPDIR/$IMPL/test_HPCCG 128 128 256
+  cd $TOPDIR
+done
+
diff --git a/hpccg/jobfiles/supermuc/weakScaleAll-ibm.tpl.job b/hpccg/jobfiles/supermuc/weakScaleAll-ibm.tpl.job
@@ -0,0 +1,43 @@
+# Run HPCCG with IBM MPI on supermuc
+#!/bin/bash
+##
+#@ energy_policy_tag = hpccg 
+#@ minimize_time_to_solution = yes
+#@ job_type = parallel
+#@ class = micro 
+#@ node = 1
+###@ island_count= not needed for
+#### class general
+#@ island_count = 1 
+#@ total_tasks= <procs>
+## other example
+##@ tasks_per_node = 28
+#@ wall_clock_limit = 1:00:00
+#@ job_name = hpccg-weakScale-ibm-<procs> 
+#@ network.MPI = sn_all,not_shared,us
+#@ initialdir = ../../
+#@ output = $(home)/log/job.$(job_name).$(jobid).out
+#@ error =  $(home)/log/job.$(job_name).$(jobid).err
+#@ notification=always
+#@ notify_user=felix.moessbauer@campus.lmu.de
+#@ queue
+. /etc/profile
+. /etc/profile.d/modules.sh
+module load mpi.ibm
+module load papi
+module load hwloc
+
+
+for IMPL in $(ls -d */); do
+  if [[ "$IMPL" == "jobfiles/" ]]; then continue; fi 
+  if [[ "$IMPL" == "results/" ]]; then continue; fi 
+  echo "Execute $IMPL"
+  TOPDIR=$(pwd)
+  RESULTDIR="$TOPDIR/results/ibm/<procs>/$IMPL"
+  mkdir -p $RESULTDIR 
+  # place output files in correct dir
+  cd $RESULTDIR 
+  poe $TOPDIR/$IMPL/test_HPCCG 64 64 64
+  cd $TOPDIR
+done
+
diff --git a/hpccg/jobfiles/supermuc/weakScaleAll-intel.tpl.job b/hpccg/jobfiles/supermuc/weakScaleAll-intel.tpl.job
@@ -0,0 +1,43 @@
+# Run HPCCG with IBM MPI on supermuc
+#!/bin/bash
+##
+#@ energy_policy_tag = hpccg 
+#@ minimize_time_to_solution = yes
+#@ job_type = MPICH 
+#@ class = micro 
+#@ node = 1
+###@ island_count= not needed for
+#### class general
+#@ island_count = 1 
+#@ total_tasks= <procs>
+## other example
+##@ tasks_per_node = 28
+#@ wall_clock_limit = 1:00:00
+#@ job_name = hpccg-weakScale-intel-<procs> 
+#@ network.MPI = sn_all,not_shared,us
+#@ initialdir = ../../
+#@ output = $(home)/log/job.$(job_name).$(jobid).out
+#@ error =  $(home)/log/job.$(job_name).$(jobid).err
+#@ notification=always
+#@ notify_user=felix.moessbauer@campus.lmu.de
+#@ queue
+. /etc/profile
+. /etc/profile.d/modules.sh
+module unload mpi.ibm
+module load mpi.intel
+module load papi
+module load hwloc
+
+for IMPL in $(ls -d */); do
+  if [[ "$IMPL" == "jobfiles/" ]]; then continue; fi 
+  if [[ "$IMPL" == "results/" ]]; then continue; fi 
+  echo "Execute $IMPL"
+  TOPDIR=$(pwd)
+  RESULTDIR="$TOPDIR/results/ibm/<procs>/$IMPL"
+  mkdir -p $RESULTDIR 
+  # place output files in correct dir
+  cd $RESULTDIR 
+  mpiexec -n <procs> $TOPDIR/$IMPL/test_HPCCG 64 64 64
+  cd $TOPDIR
+done
+
diff --git a/hpccg/src_dart/Makefile.supermuc b/hpccg/src_dart/Makefile.supermuc
@@ -0,0 +1,132 @@
+#@HEADER
+# ************************************************************************
+# 
+#               HPCCG: Simple Conjugate Gradient Benchmark Code
+#                 Copyright (2006) Sandia Corporation
+# 
+# Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+# license for use of this work by or on behalf of the U.S. Government.
+# 
+# This library is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 2.1 of the
+# License, or (at your option) any later version.
+#  
+# This library is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#  
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+# Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
+# 
+# ************************************************************************
+#@HEADER
+
+DART_ROOT=$(HOME)/opt/dash-0.3.0
+
+
+# Simple hand-tuned makefile.  Modify as necessary for your environment.
+# Questions? Contact Mike Heroux (maherou@sandia.gov).
+#
+
+#
+# 0) Specify compiler and linker:
+
+#CXX=/usr/local/bin/g++
+#LINKER=/usr/local/bin/g++
+CXX=mpiCC
+LINKER=mpiCC
+
+
+# 1) Build with MPI or not?
+#    If you want to run the program with MPI, make sure USE_MPI is set 
+#    to -DUSING_MPI
+
+USE_MPI =
+USE_MPI = -DUSING_MPI
+
+# 2) MPI headers:  
+#    If you:
+#    - Are building MPI mode (-DUSING_MPI is set above).
+#    - Do not have the MPI headers installed in a default search directory and
+#    - Are not using MPI compiler wrappers
+#    Then specify the path to your MPI header file (include a -I)
+
+#MPI_INC = -I/usr/MPICH/SDK.gcc/include
+
+DART_INC=-I$(DART_ROOT)/include
+
+
+# 3) Specify C++ compiler optimization flags (if any)
+#    Typically some reasonably high level of optimization should be used to 
+#    enhance performance.
+
+#IA32 with GCC: 
+#CPP_OPT_FLAGS = -O3 -funroll-all-loops -malign-double
+CPP_OPT_FLAGS = -O3 -ftree-vectorize -ftree-vectorizer-verbose=2
+
+CPP_STD_FLAGS = -std=c++11 -DDART_FULL_ALLOC
+CPP_STD_FLAGS = -std=c++11 
+
+#
+# 4) MPI library:
+#    If you:
+#    - Are building MPI mode (-DUSING_MPI is set above).
+#    - Do not have the MPI library installed a default search directory and
+#    - Are not using MPI compiler wrappers for linking
+#    Then specify the path to your MPI library (include -L and -l directives)
+
+#MPI_LIB = -L/usr/MPICH/SDK.gcc/lib -lmpich
+
+#
+# 5) Build with OpenMP or not?
+#    If you want to run the program with OpenMP, make sure USING_OMP is set
+#    to -DUSING_OMP
+
+USE_OMP = 
+#USE_OMP = -DUSING_OMP
+
+#
+# 6) OpenMP Compiler argument
+#    GCC and Intel compilers require -fopenmp and -openmp, resp.  Other compilers may differ.
+
+#OMP_FLAGS = -fopenmp
+#OMP_FLAGS = -openmp
+
+#
+# 7) System libraries: (May need to add -lg2c before -lm)
+
+SYS_LIB =-lm
+DART_LIB=-L$(DART_ROOT)/lib -L$(HWLOC_BASE)/lib -L$(PAPI_BASE)/lib -ldash-mpi -ldart-mpi -ldart-base -lhwloc -lpapi -lnuma 
+
+#
+# 6) Specify name if executable (optional):
+
+TARGET = test_HPCCG
+
+################### Derived Quantities (no modification required) ##############
+
+CXXFLAGS= $(CPP_OPT_FLAGS) $(CPP_STD_FLAGS) $(OMP_FLAGS) $(USE_OMP) $(USE_MPI) $(MPI_INC) $(DART_INC)
+
+LIB_PATHS= $(SYS_LIB) $(DART_LIB)
+
+TEST_CPP = main.cpp generate_matrix.cpp read_HPC_row.cpp \
+	  compute_residual.cpp mytimer.cpp dump_matlab_matrix.cpp \
+          HPC_sparsemv.cpp HPCCG.cpp waxpby.cpp ddot.cpp \
+          make_local_matrix.cpp exchange_externals.cpp \
+          YAML_Element.cpp YAML_Doc.cpp
+
+TEST_OBJ          = $(TEST_CPP:.cpp=.o)
+
+$(TARGET): $(TEST_OBJ)
+	$(LINKER) $(CPP_OPT_FLAGS) $(OMP_FLAGS) $(TEST_OBJ) $(LIB_PATHS) -o $(TARGET)
+
+test:
+	@echo "Not implemented yet..."
+
+clean:
+	@rm -f *.o  *~ $(TARGET) $(TARGET).exe test_HPCPCG