Few fixes to clean version respect to actual develop branch and fix e…

…rroneous version of mod_shympi_node.f
SHYFEM-model · georgu · Apr 23, 2021 · Dec 21, 2020 · Mar 11, 2021 · Mar 24, 2021
commit 46bf81836991ba1fa93f93326b65df12a4dcc4b8
diff --git a/Rules.make b/Rules.make
@@ -176,12 +176,11 @@ SOLVER = SPARSKIT
 
 ##############################################
 
-#to use PETSc compile with make using the PETSc solver, modify as well file pragma_directives.h to undef _use_SPK, and define _use_PETSc 
-#SOLVER=PETSC_AmgX  #to compile with make using the PETSc_AmgX solver, modify as well file pragma_directives.h to undef _use_SPK, and define both _use_PETSc and _use_AmgX 
 #PETSCDIR needed for both PETSc and PETSc_AmgX solvers
 PETSCDIR=
+
 # fill in next paths for PETSc_AmgX solver only
-AMGXWRAPWRAPDIR=
+AMGXWRAPWRAPDIR=../amgx-c-wrapper/amgx-c-wrapper
 AMGXWRAPDIR=
 AMGXDIR=
 CUDADIR=

diff --git a/fem3d/Makefile b/fem3d/Makefile
@@ -69,14 +69,8 @@ ifeq ($(SOLVER),SPARSKIT)
   simsys_solver=simsys_spk.o 
 endif
 
-ifeq ($(FORTRAN_COMPILER),INTEL)
-  PREPROC_FLAG = -fpp
-else
-  PREPROC_FLAG = -cpp
-endif
 
 FFLAGS += -Iinclude/ 
-FFLAGS += $(PREPROC_FLAG) 
 LIBG_PETSC =
 INCL_PETSC = 
 LIBG_AmgX =
@@ -92,7 +86,7 @@ ifeq ($(SOLVER),PETSC)
 endif
 ifeq (${SOLVER},PETSC_AmgX)
   LIBG_PETSC =  -lstdc++ \
-	-L$(PETSCDIR)/lib  -lpetsc $(PREPROC_FLAG)
+	-L$(PETSCDIR)/lib  -lpetsc
   LIBG_AmgX = -L$(AMGXWRAPDIR)/lib -L$(AMGXWRAPDIR)/lib64 -lAmgXWrapper \
 	-L$(AMGXDIR)/lib -L$(AMGXDIR)/lib64 -lamgx \
 	-L$(CUDADIR)/lib -L$(CUDADIR)/lib64 -lcusolver -lcusparse -lcublas  -lcuda -lcudart  
@@ -851,7 +845,7 @@ gotm3d.o: gotm3d.f gotmturb.i
 
 
 #------------------------------------------------
-amgx_c_wrapper.o: ../amgx-c-wrapper/amgx-c-wrapper/src/amgx_c_wrapper.cpp ../amgx-c-wrapper/amgx-c-wrapper/src/amgx_c_wrapper.h
+amgx_c_wrapper.o: $(AMGXWRAPWRAPDIR)/src/amgx_c_wrapper.cpp ../amgx-c-wrapper/amgx-c-wrapper/src/amgx_c_wrapper.h
 	$(CXX) -c $(CXXFLAGS) $<  -o $@
 #------------------------------------------------
 

diff --git a/fem3d/include/pragma_directives.h b/fem3d/include/pragma_directives.h
diff --git a/fem3d/mod_shympi_node.f b/fem3d/mod_shympi_node.f
@@ -42,8 +42,6 @@
 ! 06.07.2018	ggu	changed VERS_7_5_48
 ! 16.02.2019	ggu	changed VERS_7_5_60
 ! 07.06.2020	ggu	new routines, 3d exchange array still missing
-! 13.03.2021	ggu	in shympi_exchange_array_3d_r() dimensions inverted
-! 13.03.2021	ggu	shympi_exchange_array_3d_r() has been re-defined
 ! 09.04.2021	clr	bug fix in shympi_bcast_array_r() -> real arg
 ! 17.04.2021	ggu	new shympi_exchange_array_3(), check_external_numbers()
 ! 22.04.2021	ggu	allocation of some arrays for bounds check
@@ -1555,10 +1553,6 @@ subroutine shympi_reduce_r(what,vals,val)
 
 	end subroutine shympi_reduce_r
 
-!******************************************************************
-!******************************************************************
-!******************************************************************
-! 2d get
 !******************************************************************
 !******************************************************************
 !******************************************************************
@@ -1597,10 +1591,54 @@ end subroutine shympi_get_array_2d_i
 !******************************************************************
 !******************************************************************
 !******************************************************************
-! exchange arrays
-!******************************************************************
-!******************************************************************
-!******************************************************************
+
+	subroutine shympi_exchange_array_3d_r_aux(vals,val_out)
+
+	real vals(:,:)
+	real val_out(:,:)
+
+	integer ni1,no1,ni2,no2
+
+	ni1 = size(vals,1)
+	no1 = size(val_out,1)
+	ni2 = size(vals,2)
+	no2 = size(val_out,2)
+
+	if( ni1 > no1 ) then
+	  write(6,*) 'ni1,no1: ',ni1,no1
+	  stop 'error stop exchange: first dimension'
+	end if
+
+	call shympi_exchange_array_internal_r(ni1,no1,ni2,no2
+     +                                    ,vals,val_out)
+
+	end subroutine shympi_exchange_array_3d_r_aux
+
+!*******************************
+
+	subroutine shympi_exchange_array_3d_i(vals,val_out)
+
+	integer vals(:,:)
+	integer val_out(:,:)
+
+	integer ni1,no1,ni2,no2
+
+	ni1 = size(vals,1)
+	no1 = size(val_out,1)
+	ni2 = size(vals,2)
+	no2 = size(val_out,2)
+
+	if( ni1 > no1 ) then
+	  write(6,*) 'ni1,no1: ',ni1,no1
+	  stop 'error stop exchange: first dimension'
+	end if
+
+	call shympi_exchange_array_internal_i(ni1,no1,ni2,no2
+     +                                    ,vals,val_out)
+
+	end subroutine shympi_exchange_array_3d_i
+
+!*******************************
 
 	subroutine shympi_exchange_array_2d_r(vals,val_out)
 
@@ -1667,30 +1705,6 @@ subroutine shympi_exchange_array_3d_r(vals,val_out)
 	real vals(:,:)
 	real val_out(:,:)
 
-	integer ni1,no1,ni2,no2
-
-	ni1 = size(vals,1)
-	no1 = size(val_out,1)
-	ni2 = size(vals,2)
-	no2 = size(val_out,2)
-
-	if( ni1 > no1 ) then
-	  write(6,*) 'ni1,no1: ',ni1,no1
-	  stop 'error stop exchange: first dimension'
-	end if
-
-	call shympi_exchange_array_internal_r(ni1,no1,ni2,no2
-     +                                    ,vals,val_out)
-
-	end subroutine shympi_exchange_array_3d_r
-
-!*******************************
-
-	subroutine shympi_exchange_array_3d_rr(vals,val_out)
-
-	real vals(:,:)
-	real val_out(:,:)
-
 	integer noh,nov
 	real, allocatable :: val_domain(:,:,:)
 
@@ -1716,30 +1730,7 @@ subroutine shympi_exchange_array_3d_rr(vals,val_out)
 	  stop 'error stop shympi_exchange_array_3d_r: (1)'
 	end if
 
-	end subroutine shympi_exchange_array_3d_rr
-
-!*******************************
-
-	subroutine shympi_exchange_array_3d_i(vals,val_out)
-
-	integer vals(:,:)
-	integer val_out(:,:)
-
-	integer ni1,no1,ni2,no2
-
-	ni1 = size(vals,1)
-	no1 = size(val_out,1)
-	ni2 = size(vals,2)
-	no2 = size(val_out,2)
-
-	if( ni1 > no1 ) then
-	  write(6,*) 'ni1,no1: ',ni1,no1
-	  stop 'error stop exchange: first dimension'
-	end if
-	call shympi_exchange_array_internal_i(ni1,no1,ni2,no2
-     +                                    ,vals,val_out)
-
-	end subroutine shympi_exchange_array_3d_i
+	end subroutine shympi_exchange_array_3d_r
 
 !*******************************
 

diff --git a/fem3d/mod_test_zeta.f b/fem3d/mod_test_zeta.f
@@ -3,7 +3,6 @@
         module mod_test_zeta
 !==================================================================
 
-#include "pragma_directives.h"
         implicit none
 
 	integer, private, save :: nn_step = 0

diff --git a/fem3d/shyfem.f b/fem3d/shyfem.f
@@ -165,9 +165,9 @@
 c 09.04.2020    ggu     run bfm through bfm_run()
 c 21.05.2020    ggu     better handle copyright notice
 c 04.06.2020    ggu     debug_output() substituted with shympi_debug_output()
-c 13.03.2021    clr&ggu adapted for petsc solver
 c 30.03.2021    ggu     more on debug, call sp111(2) outside time loop
 c 01.04.2021    ggu     turbulence cleaned
+c 23.04.2021    clr&ggu adapted for petsc solver
 c
 c*****************************************************************
 c
@@ -234,7 +234,7 @@ program shyfem
 	real dt
 	double precision timer
 	double precision mpi_t_start,mpi_t_end,parallel_start
-	double precision mpi_t_solve,mpi_t_run,mpi_t_init_solver
+	double precision mpi_t_solve
 	double precision dtime,dtanf,dtend
         double precision atime_start,atime_end
         character*20 aline_start,aline_end
@@ -461,7 +461,6 @@ program shyfem
         !call test_forcing(dtime,dtend)
 
 	call test_zeta_init
-        mpi_t_run = shympi_wtime()
 
 c%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 c%%%%%%%%%%%%%%%%%%%%%%%%% time loop %%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -582,11 +581,6 @@ program shyfem
         write(6,*)'Parallel_TIME =',mpi_t_end-parallel_start,my_id
 	call shympi_time_get(1,mpi_t_solve)
         write(6,*)'MPI_SOLVE_TIME =',mpi_t_solve,my_id
-	call shympi_time_get(2,mpi_t_init_solver)
-        write(6,*)'MPI_INI_TIME=',
-     +      mpi_t_run-mpi_t_start+mpi_t_init_solver,my_id
-        write(6,*)'MPI_RUN_TIME =',
-     +      mpi_t_end-mpi_t_run-mpi_t_init_solver,my_id
 
         call get_real_time(atime_end,aline_end)
 
@@ -604,8 +598,7 @@ program shyfem
 	!call prifnm(15)
 
 	!call shympi_finalize
-        write(6,*)'shyfem program exiting normally'
-	call shympi_exit(0)
+	call shympi_exit(99)
 	call exit(99)
 
 	stop
@@ -661,7 +654,6 @@ subroutine shyfem_init(strfile,bdebug,bdebout,bmpirun)
         call clo_add_option('debout',.false.
      +			,'writes debugging information to file')
 
-
         call clo_parse_options
 
         call clo_get_option('quiet',bquiet)