added SuperMUC subfolder, for DASH's plattform adjustments

dash-project · Mar 14, 2018 · 26e518f · 26e518f
1 parent c52720d
commit 26e518f
Show file tree

Hide file tree

Showing 22 changed files with 2,151 additions and 0 deletions.
diff --git a/cowichan/SuperMUC/Makefile b/cowichan/SuperMUC/Makefile
@@ -0,0 +1,26 @@
+include ./make.defs
+
+SUBDIRS   = randmat thresh winnow outer product chain
+
+
+.PHONY: all clean all-clean
+#### ------------------------------------------------------------
+all: dash
+
+clean: dash-clean
+all-clean: clean
+####################  </Top Level Targets>  #####################
+####################       <DASH>           #####################
+.PHONY: dash dash-clean $(SUBDIRS)
+#### ------------------------------------------------------------
+
+dash: MAKECMDGOALS = all
+dash: $(SUBDIRS)
+
+dash-clean: MAKECMDGOALS = clean
+dash-clean: $(SUBDIRS)
+
+$(SUBDIRS):
+	@$(MAKE) --no-print-directory -C $@ $(MAKECMDGOALS)
+####################    </DASH>    ##############################
+
diff --git a/cowichan/SuperMUC/Terminal_Color.h b/cowichan/SuperMUC/Terminal_Color.h
@@ -0,0 +1,66 @@
+#ifndef TERMINAL_COLOR_HEADER
+#define TERMINAL_COLOR_HEADER
+
+#define TERMINAL_COLORS
+
+#define BEGIN_COLOR "\x1B["
+#define END_COLOR   "\x1B[0m"
+
+// Color codes for colorized output on terminal
+enum Code {
+  FBLK   = 30,
+  FRED       ,
+  FGREEN     ,
+  FYEL       ,
+  FBLUE      ,
+  FMAG       ,
+  FCYN       ,
+  FWHT   = 37,
+  BRED   = 41,
+  BGREEN     ,
+  BYEL       ,
+  BBLUE      ,
+  BMAG       ,
+  BCYN       ,
+  BWHT   = 47
+};
+
+
+// Helper struct for fmt function
+template<typename T>
+struct sPar{
+  public:
+    const T& in;
+    const Code code;
+    const int width;
+    sPar(const T& in_, const Code code_, const int width_ = 0) 
+    :in(in_), code(code_), width(width_){};
+};
+
+// Function for formated output and more beautiful code
+template<typename T>
+inline const sPar<T> fmt( const T& in, const Code code, const int width = 0 ) {
+  return sPar<T>( in, code, width );
+}
+
+std::ostream& operator<<(std::ostream& os, const sPar<unsigned char>& par) {
+  #ifdef TERMINAL_COLORS
+    return os << BEGIN_COLOR << par.code << "m" << std::setw(par.width) << static_cast<uint>(par.in) << END_COLOR;
+  #else
+    return os << std::setw(par.width) << static_cast<uint>(par.in);
+  #endif
+}
+
+// overloading for transfer of several parameters
+template<typename T>
+std::ostream& operator<<(std::ostream& os, const sPar<T>& par) {
+  #ifdef TERMINAL_COLORS
+    return os << BEGIN_COLOR << par.code << "m" << std::setw(par.width) << par.in << END_COLOR;
+  #else
+    return os << std::setw(par.width) << par.in;
+  #endif
+}
+
+
+
+#endif
diff --git a/cowichan/SuperMUC/benchAll.sh b/cowichan/SuperMUC/benchAll.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+
+probIxStart=0
+probIxEnd=5
+jobsStart=1
+jobsEnd=24
+numberOfIterations=10
+nRowsCols=(100 400 4000 40000)
+thresh=(10 25 50 75 100)
+winnowNelts=(100 400 4000 40000)
+
+
+# #bench winnows and chains
+# for (( nRowsColsIX=3 ; nRowsColsIX<4 ; ++nRowsColsIX )); do
+  # for (( threshIX=3 ; threshIX<5 ; ++threshIX )); do
+    # for (( winnowNeltsIX=0 ; winnowNeltsIX<4 ; ++winnowNeltsIX )); do
+      # #bench winnows
+      # ./benchDash.sh 2 2 $jobsStart $jobsEnd $numberOfIterations ${nRowsCols[$nRowsColsIX]} ${thresh[$threshIX]} ${winnowNelts[$winnowNeltsIX]}
+      # #bench chains
+      # ./benchDash.sh 5 5 $jobsStart $jobsEnd $numberOfIterations ${nRowsCols[$nRowsColsIX]} ${thresh[$threshIX]} ${winnowNelts[$winnowNeltsIX]}
+    # done
+  # done
+# done
+
+
+# #bench randmats
+# for (( nRowsColsIX=2 ; nRowsColsIX<4 ; ++nRowsColsIX )); do
+  # ./benchDash.sh 0 0 $jobsStart $jobsEnd $numberOfIterations ${nRowsCols[$nRowsColsIX]} 0 0
+# done
+
+# #bench threshs
+# tmp=2
+# for (( nRowsColsIX=2 ; nRowsColsIX<4 ; ++nRowsColsIX )); do
+  # for (( threshIX=tmp ; threshIX<5 ; ++threshIX )); do
+  # tmp=0
+    # ./benchDash.sh 1 1 $jobsStart $jobsEnd $numberOfIterations ${nRowsCols[$nRowsColsIX]} ${thresh[$threshIX]} 0
+  # done
+# done
+
+
+#bench outers and products
+for (( winnowNeltsIX=0 ; winnowNeltsIX<4 ; ++winnowNeltsIX )); do
+    ./benchDash.sh 3 4 $jobsStart $jobsEnd $numberOfIterations 0 0 ${winnowNelts[$winnowNeltsIX]}
+done
diff --git a/cowichan/SuperMUC/chain/Makefile b/cowichan/SuperMUC/chain/Makefile
@@ -0,0 +1,24 @@
+include ../make.defs
+
+.PHONY: all
+all: chain
+#mpirun -n 4 ./$@ < main.in > rand.out2
+#diff -ws rand.out2 main.gold
+
+chain: chain.cpp ../randmat/randmat.h ../thresh/thresh.h ../winnow/winnow.h ../outer/outer.h ../product/product.h
+	$(CXX) -c $(INC) $<
+	$(CXX) -o $@ $@.o $(LIB)
+
+
+.PHONY: printenv
+printenv:
+	@echo "CXX           = $(CXX)"
+	@echo "DART_IMPL     = $(DART_IMPL)"
+	@echo "DASH_ROOT     = $(DASH_ROOT)"
+	@echo "INC           = $(INC)"
+	@echo "LIB           = $(LIB)"
+
+.PHONY: clean
+clean:
+	-rm chain
+	-rm chain.o
diff --git a/cowichan/SuperMUC/chain/chain.cpp b/cowichan/SuperMUC/chain/chain.cpp
@@ -0,0 +1,177 @@
+#include <libdash.h>
+#include <iostream>
+
+using std::cout;
+using std::endl;
+using std::cin;
+
+using dash::Shared;
+using dash::NArray;
+using dash::Array;
+
+using uint     = unsigned int  ;
+using uchar    = unsigned char ;
+using MATRIX_T = uchar         ;
+
+static int myid;
+
+#include "../randmat/randmat.h"
+#include "../thresh/thresh.h"
+// #include "../winnow/winnow_placeholder.h"
+#include "../winnow/winnow.h"
+#include "../outer/outer.h"
+#include "../product/product.h"
+
+#include <time.h>
+#include <stdio.h>
+#include <cstring>
+using std::strcmp;
+
+struct InputPar { uint nRowsCols, seed, thresh, winnow_nelts; } in;
+
+
+/* One unit has the job to read in the parameters.
+ * Because there's always a unit0, it reads the input parameter and
+ * distributes them to the rest of the units.
+ */
+inline void ReadPars( )
+{
+  Shared<InputPar> input_transfer;
+
+  if(0 == myid)
+  {
+    cin >> in.nRowsCols    ;    
+    cin >> in.seed         ;
+    cin >> in.thresh       ;
+    cin >> in.winnow_nelts ;
+
+    input_transfer.set(in) ;
+  }                        
+  input_transfer.barrier() ;
+  in = input_transfer.get();
+}
+
+/* both parameters musst have same size!
+ * copy from a dash::Array to std::vector
+ */
+inline void CopyFromDashToStd(
+  Array  <double> const & dashVector,
+  vector <double>       & loclVector)
+{
+  if(0 == myid)
+  {
+    double * vec = loclVector.data( );
+    for( double const i : dashVector )
+    {
+      *(vec++) = i;
+    }
+  }
+  BroadcastOuterVecToUnits( loclVector ); //defined in product.h
+}
+
+/* both parameters musst have same size!
+ * copy from a dash::Array to std::vector
+ */
+template<typename T, typename X, typename Y>
+inline void CopyFromDashToStd(
+  Array<T, X, Y>  const & dashVector,
+       vector<T>        & loclVector)
+{
+  if(0 == myid)
+  {
+    T * vec = loclVector.data( );
+    for( T const i : dashVector )
+    {
+      *(vec++) = i;
+    }
+  }
+
+  // using dash::Team        ;
+  // using dash::team_unit_t ;
+
+  // Team & team = dash::Team::All( );
+
+  dart_ret_t ret = dart_bcast(
+                    static_cast<void*>( loclVector.data() ),  // buf 
+                    loclVector.size() * sizeof(T)          ,  // nelem
+                    DART_TYPE_BYTE                         ,  // dtype
+                    dash::team_unit_t(0)                   ,  // root/source
+                    dash::Team::All( ).dart_id( )          ); // team
+
+  if( DART_OK != ret ) cout << "An error while BCAST has occured!" << endl;
+}
+
+
+// @echo $(nRowsCols) $(seed) $(thresh) $(winnow_nelts) | $(TBB_ROOT)/chain/expertpar/main
+int main( int argc, char* argv[] )
+{
+  dash::init( &argc,&argv );
+
+  struct timespec start, stop;
+  double accum;
+  int is_bench = 0;
+
+  for (int i = 1; i < argc; i++) {
+    if (!strcmp(argv[i], "--is_bench")) {
+      is_bench = 1;
+    }
+  }
+
+  myid = dash::myid( );
+  ReadPars( );
+
+  // initialize variables
+  NArray<MATRIX_T, 2> rand_mat      ( in.nRowsCols   , in.nRowsCols    );
+  NArray<bool    , 2> thresh_mask   ( in.nRowsCols   , in.nRowsCols    );
+  vector<value      > winnow_vecRes ( in.winnow_nelts                  ); //value defined in winnow.h
+  NArray<double  , 2> outer_mat     ( in.winnow_nelts, in.winnow_nelts );
+  Array <double     > outer_vec     ( in.winnow_nelts                  );
+  vector<double     > prod_vec      ( in.winnow_nelts                  );
+  // Array  < double             > result        ( in.winnow_nelts                  );
+  res_array_t winnow_dashRes; // defined in winnow.h
+
+  // after the run of outer "outer_vec" will be recycled/reused for the final output
+  auto & result = outer_vec;
+
+  if( clock_gettime( CLOCK_MONOTONIC, &start) == -1 ) {
+    perror( "clock gettime error 1" );
+    exit( EXIT_FAILURE );
+  }
+
+  // execute functions
+  Randmat( rand_mat, in.seed );
+
+  Thresh( rand_mat, thresh_mask, in.nRowsCols, in.nRowsCols, in.thresh );
+
+  // if(0 == myid){ winnow( in.nRowsCols, in.nRowsCols, rand_mat, thresh_mask, in.winnow_nelts, winnow_points );}
+  // BroadcastPointsToUnits( winnow_points );
+  Winnow( in.nRowsCols, in.nRowsCols, rand_mat, thresh_mask, in.winnow_nelts, winnow_dashRes );
+  CopyFromDashToStd( winnow_dashRes, winnow_vecRes );
+
+  Outer( winnow_vecRes, outer_mat, outer_vec, in.winnow_nelts );
+  CopyFromDashToStd( outer_vec, prod_vec );
+
+  Product( prod_vec, outer_mat, result, in.winnow_nelts );
+
+  if( clock_gettime( CLOCK_MONOTONIC, &stop) == -1 ) {
+    perror( "clock gettime error 2" );
+    exit( EXIT_FAILURE );
+  }
+
+  accum = ( stop.tv_sec - start.tv_sec ) + ( stop.tv_nsec - start.tv_nsec ) / 1e9;
+
+  if( 0 == myid ){
+    FILE* fp = fopen("./measurements.txt", "a");
+
+    if( !fp ) {
+        perror("File opening for benchmark results failed");
+        return EXIT_FAILURE;
+    }
+    // Lang, Problem, rows, cols, thresh, winnow_nelts, jobs, time
+    fprintf( fp, "DASH  ,Chain  ,%5u,%5u,%3u,%5u,%2u,%.9lf,isBench:%d\n", in.nRowsCols, in.nRowsCols, in.thresh, in.winnow_nelts, dash::Team::All().size(), accum, is_bench );
+    fclose ( fp );
+  }
+
+  if( !is_bench ){ PrintOutput( result, in.winnow_nelts ); }
+  dash::finalize( );
+}
diff --git a/cowichan/SuperMUC/make.defs b/cowichan/SuperMUC/make.defs
@@ -0,0 +1,12 @@
+DASH_ROOT = $(HOME)/opt/dash-0.3.0
+
+DART_IMPL = mpi
+
+INC=-I$(DASH_ROOT)/include
+LIB=-L$(DASH_ROOT)/lib -ldash-$(DART_IMPL) -ldart-$(DART_IMPL) -ldart-base
+
+#CXXFLAGS= -g -O0 -Wall -DDASH_DEBUG=1 -DDASH_ENABLE_DEFAULT_INDEX_TYPE_LONG
+#CXXFLAGS= -O3 -DDASH_ENABLE_DEFAULT_INDEX_TYPE_LONG
+ CXXFLAGS= -Ofast
+
+ CXX     = mpiCC -cxx=icpc -std=c++11 $(CXXFLAGS)
diff --git a/cowichan/SuperMUC/outer/Makefile b/cowichan/SuperMUC/outer/Makefile
@@ -0,0 +1,22 @@
+include ../make.defs
+
+.PHONY: all
+all: outer
+
+outer: outer.cpp outer.h
+	$(CXX) -c $(INC) $<
+	$(CXX) -o $@ $@.o $(LIB) 
+
+
+.PHONY: printenv
+printenv :
+	@echo "CXX           = $(CXX)"
+	@echo "DART_IMPL     = $(DART_IMPL)"
+	@echo "DASH_ROOT     = $(DASH_ROOT)"
+	@echo "INC           = $(INC)"
+	@echo "LIB           = $(LIB)"
+
+.PHONY: clean
+clean:
+	-rm outer
+	-rm outer.o