Updated build scripts, config files, and stack yaml files (#518)

* update configuration files for cheyenne, hera, jet, noaacloud * updated build_nceplib.sh for crtm and crtm_fix file installation, and higher version ip build typo correction * Update S4 config and installer info. #518 Co-authored-by: Natalie Perlin <Natalie@MacBook-Pro.local> Co-authored-by: David Huber <david.huber@noaa.gov>
NOAA-EMC · Apr 26, 2023 · ef2d9ae · ef2d9ae
1 parent d2acee7
commit ef2d9ae
Show file tree

Hide file tree

Showing 20 changed files with 256 additions and 156 deletions.
diff --git a/README.md b/README.md
@@ -28,11 +28,12 @@ Machine     | Programmer
 ------------|------------------------------------------
 Hera        | Jong Kim, Natalie Perlin, Cameron Book
 Jet         | Jong Kim, Natalie Perlin, Cameron Book
-Orion       | Hang-Lei
+Orion       | Natalie Perlin, Cameron Book
 WCOSS-Dell  | Hang-Lei
 WCOSS-Cray  | Hang-Lei
-Cheyenne    | Jong Kim
+Cheyenne    | Jong Kim, Natalie Perlin
 Gaea        | Jong Kim, Natalie Perlin, Cameron Book
+S4          | David Huber
 
 ## Contributors
 
@@ -45,7 +46,7 @@ The prerequisites of building hpc-stack are:
 - [Lmod](https://lmod.readthedocs.io/en/latest/) - An Environment Module System
 - CMake and make
 - wget and curl
-- git
+- git, git-lfs
 
 Building the software stack is a **Three-Step process**, as described in the documentation:
 

diff --git a/config/config_cheyenne_gnu.sh b/config/config_cheyenne_gnu.sh
@@ -2,7 +2,7 @@
 
 # Compiler/MPI combination
 export HPC_COMPILER="gnu/10.1.0"
-export HPC_MPI="mpt/2.22"
+export HPC_MPI="mpt/2.25"
 export HPC_PYTHON="python/dummy"
 
 # Build options
@@ -19,17 +19,11 @@ export STACK_EXIT_ON_FAIL=Y
 export WGET="wget -nv"
 
 module purge
-module unuse /glade/u/apps/ch/modulefiles/default/compilers
-module use   /glade/p/ral/jntp/GMTB/tools/compiler_mpi_modules/compilers
-export MODULEPATH_ROOT=/glade/p/ral/jntp/GMTB/tools/compiler_mpi_modules
-
-module load gnu/10.1.0
-module load mpt/2.22
-module load ncarcompilers/0.5.0
 module load ncarenv/1.3
-
-# Load these basic modules for Cheyenne
-module load cmake/3.18.2
+module load gnu/10.1.0
+module load mpt/2.25
+module load cmake/3.22.0
+module load sqlite/3.36.0
 
 # gfortran-10 compatibility flags for incompatible software
 export STACK_esmf_FFLAGS="-fallow-argument-mismatch -fallow-invalid-boz"
@@ -39,7 +33,6 @@ export STACK_madis_FFLAGS="-fallow-argument-mismatch -fallow-invalid-boz"
 # Build FMS with AVX2 flags
 export STACK_fms_CFLAGS="-march=core-avx2"
 export STACK_fms_FFLAGS="-march=core-avx2 -fallow-argument-mismatch"
-
 # Patch FMS
 export STACK_fms_PATCH="cheyenne_gnu_fms_mpp_util_mpi_inc.patch"
 
diff --git a/config/config_cheyenne_intel.sh b/config/config_cheyenne_intel.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/2021.2"
-export HPC_MPI="mpt/2.22"
+export HPC_COMPILER="intel/2022.1"
+export HPC_MPI="mpt/2.25"
 export HPC_PYTHON="python/dummy"
 
 # Build options
@@ -19,18 +19,16 @@ export STACK_EXIT_ON_FAIL=Y
 export WGET="wget -nv"
 
 module purge
-module unuse /glade/u/apps/ch/modulefiles/default/compilers
-module use   /glade/p/ral/jntp/GMTB/tools/compiler_mpi_modules/compilers
-export MODULEPATH_ROOT=/glade/p/ral/jntp/GMTB/tools/compiler_mpi_modules
-
-module load intel/2021.2
-module load mpt/2.22
-module load ncarcompilers/0.5.0
 module load ncarenv/1.3
-
-# Load these basic modules for Cheyenne
-module load cmake/3.18.2
+module load intel/2022.1
+module load mpt/2.25
+module load cmake/3.22.0
+module load sqlite/3.36.0
 
 # Build FMS with AVX2 flags
 export STACK_fms_CFLAGS="-march=core-avx2"
 export STACK_fms_FFLAGS="-march=core-avx2"
+
+export SERIAL_CC=icc
+export SERIAL_FC=ifort
+export SERIAL_CXX=icpc
diff --git a/config/config_gaea.sh b/config/config_gaea.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/18.0.6.288"
+export HPC_COMPILER="intel/2021.3.0"
 export HPC_MPI="cray-mpich/7.7.11"
-export HPC_PYTHON="cray-python/3.7.3.2"
+export HPC_PYTHON="miniconda3/4.12.0"
 
 # Build options
 export USE_SUDO=N
@@ -18,18 +18,22 @@ export DOWNLOAD_ONLY=N
 export STACK_EXIT_ON_FAIL=Y
 export WGET="wget -nv"
 
-# Load these basic modules for Gaea
-module load git/2.26.0
+# Load lmod environment
+source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
+#
 module load cmake/3.20.1
-module switch intel/18.0.6.288
+module load intel/2021.3.0
+module load cray-mpich/7.7.11
+module use /lustre/f2/dev/role.epic/contrib/modulefiles
+module load miniconda3/4.12.0
+module load git/2.31.1
+module load git-lfs
 
-export SERIAL_CC=cc
-export SERIAL_CXX=CC
-export SERIAL_FC=ftn
+export SERIAL_CC=icc
+export SERIAL_CXX=icpc
+export SERIAL_FC=ifort
 
 export MPI_CC=cc
 export MPI_CXX=CC
 export MPI_FC=ftn
 
-# Load lmod environment
-source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh
diff --git a/config/config_hera.sh b/config/config_hera.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/18.0.5.274"
-export HPC_MPI="impi/2018.0.4"
-export HPC_PYTHON="miniconda3/4.6.14"
+export HPC_COMPILER="intel/2022.1.2"
+export HPC_MPI="impi/2022.1.2"
+export HPC_PYTHON="miniconda3/4.12.0"
 
 # Build options
 export USE_SUDO=N

diff --git a/config/config_jet.sh b/config/config_jet.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/18.0.5.274"
-export HPC_MPI="impi/2018.4.274"
-export HPC_PYTHON="intelpython/3.6.5"
+export HPC_COMPILER="intel/2022.1.2"
+export HPC_MPI="impi/2022.1.2"
+export HPC_PYTHON="miniconda3/4.12.0"
 
 # Build options
 export USE_SUDO=N
@@ -22,3 +22,6 @@ export WGET="wget -nv"
 export __lmod_vx=""
 
 module load cmake/3.20.1
+
+module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles
+module load miniconda3/4.12.0
diff --git a/config/config_noaacloud.sh b/config/config_noaacloud.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Compiler/MPI combination
+export HPC_COMPILER="intel/2021.3.0"
+export HPC_MPI="impi/2021.3.0"
+export HPC_PYTHON="miniconda3/4.12.0"
+
+# Build options
+export USE_SUDO=N
+export PKGDIR=pkg
+export LOGDIR=log
+export OVERWRITE=N
+export NTHREADS=8
+export   MAKE_CHECK=N
+export MAKE_VERBOSE=Y
+export   MAKE_CLEAN=N
+export DOWNLOAD_ONLY=N
+export STACK_EXIT_ON_FAIL=Y
+export WGET="wget -nv"
+export VENVTYPE="condaenv"
+
+# Load these basic modules for NOAA Cloud
+module purge
+module load cmake/3.20.1
+module load intel/2021.3.0
+module load impi/2021.3.0
+module use /contrib/EPIC/miniconda3/modulefiles
+module load miniconda3/4.12.0
+
+export SERIAL_CC=icc
+export SERIAL_CXX=icpc
+export SERIAL_FC=ifort
diff --git a/config/config_orion.sh b/config/config_orion.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/2018.4"
-export HPC_MPI="impi/2018.4"
-export HPC_PYTHON="miniconda3/4.6.14"
+export HPC_COMPILER="intel/2022.1.2"
+export HPC_MPI="impi/2022.1.2"
+export HPC_PYTHON="miniconda3/4.12.0"
 
 # Build options
 export USE_SUDO=N
@@ -21,9 +21,12 @@ export VENVTYPE="condaenv"
 
 # Load these basic modules for Orion
 module purge
-module load cmake
+module load cmake/3.22.1
 module load git
 
+module use /work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles
+module load miniconda3/4.12.0
+
 # Build FMS with AVX2 flags
 export STACK_fms_CFLAGS="-march=core-avx2"
 export STACK_fms_FFLAGS="-march=core-avx2"
diff --git a/config/config_s4.sh b/config/config_s4.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # Compiler/MPI combination
-export HPC_COMPILER="intel/18.0.4"
-export HPC_MPI="impi/18.0.4"
+export HPC_COMPILER="intel/2022.1"
+export HPC_MPI="impi/2022.1"
 export HPC_PYTHON="miniconda/3.8-s4"
 
 # Build options

diff --git a/docs/source/hpc-install.rst b/docs/source/hpc-install.rst
@@ -316,7 +316,7 @@ The compiler and MPI modules can be handled separately from the rest of the buil
       module load hpc-intel/2020
       module load hpc-impi/2020
 
-These ``hpc-`` modules are really meta-modules that load the compiler/MPI library and modify the MODULEPATH so that the user has access to the software packages that will be built in :numref:`Step %s <NonConHPCBuild>`. On HPC systems, these meta-modules load the native modules provided by the system administrators. 
+These ``hpc-`` modules are really meta-modules that load the compiler/MPI library and modify the MODULEPATH so that the user has access to the software packages that will be built in :numref:`Step %s <NonConHPCBuild>`. On HPC systems, these meta-modules load the modules provided by the system administrators. 
 
 In short, you may prefer not to load the compiler or MPI modules directly. Instead, loading the hpc- meta-modules as demonstrated above will provide everything needed to load software libraries.
 

diff --git a/docs/source/mac-install.rst b/docs/source/mac-install.rst
@@ -317,7 +317,7 @@ Set up the modules and environment:
 
     ./setup_modules.sh -c config/config_macos_gnu.sh -p $HPC_INSTALL_DIR | tee setup_modules.log
 
-where the ``$HPC_INSTALL_DIR`` is the *absolute* path of the HPC-stack installation directory. The $HPC_INSTALL_DIR needs to be different from the source directory, where you build and compile the software stack. When asked whether to use "native" Python or compilers, choose "YES" if using those already installed on your system, or "NO" if they will be built during the HPC-stack installation. The likely response is to answer "YES" to python, "YES" to compilers, and "NO" for MPI/openmpi. 
+where the ``$HPC_INSTALL_DIR`` is the *absolute* path of the HPC-stack installation directory. The $HPC_INSTALL_DIR needs to be different from the source directory, where you build and compile the software stack. When asked whether to use "native" Python or compilers, choose "YES" if python or compiler paths are already listed in the $PATH variable, and "NO" if the modules need to be loaded. The likely response is to answer "YES" to python, "NO" to compilers, and "NO" for MPI/openmpi. 
 
 Building HPC-Stack
 ^^^^^^^^^^^^^^^^^^^^^

diff --git a/libs/build_hdf5.sh b/libs/build_hdf5.sh
@@ -68,7 +68,7 @@ mkdir -p build && cd build
 
 [[ -z $mpi ]] || extra_conf="--enable-parallel --enable-unsupported"
 
-[[ $enable_shared =~ [yYtT] ]] || shared_flags="--disable-shared --enable-static --enable-static-exec"
+[[ $enable_shared =~ [yYtT] ]] || shared_flags="--disable-shared --enable-static"
 [[ $enable_szip =~ [yYtT] ]] && szip_flags="--with-szlib=$SZIP_ROOT"
 [[ $enable_zlib =~ [yYtT] ]] && zlib_flags="--with-zlib=$ZLIB_ROOT"
 

diff --git a/libs/build_mapl.sh b/libs/build_mapl.sh
@@ -74,6 +74,7 @@ cmake .. \
       -DBUILD_WITH_PFLOGGER=OFF \
       -DESMA_USE_GFE_NAMESPACE=ON \
       -DBUILD_SHARED_MAPL=OFF \
+      -DBUILD_WITH_FARGPARSE=NO \
       -DUSE_EXTDATA2G=OFF \
       ${CMAKE_OPTS}
 

diff --git a/libs/build_nceplibs.sh b/libs/build_nceplibs.sh
@@ -30,7 +30,7 @@ if $MODULES; then
 
   case $name in
     # The following require MPI
-    nemsiogfs | ncio | ncdiag)
+    crtm | nemsiogfs | ncio | ncdiag)
       module load hpc-$HPC_MPI
       using_mpi=YES
       ;;
@@ -86,10 +86,9 @@ if $MODULES; then
       module load netcdf
       ;;
     crtm)
-      module load hpc-$HPC_MPI
       module load netcdf
       ;;
-    ip2)
+    ip | ip2)
       module load sp
       ;;
     g2)
@@ -247,27 +246,16 @@ fi
 
 cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"}
 
-if [[ "$name" == "crtm" ]]; then
-# Correction to allow for the crtm-2.4.0 serial build:
-  if [[ "${version}" == "v2.4.0" ]]; then
-sed -i'.orig' -e 's/  find_package(MPI REQUIRED)/\#  find_package(MPI REQUIRED)/g' ./cmake/FindNetCDF.cmake
-  fi
 # Download CRTM fix files
+if [[ "$name" == "crtm" ]]; then
   if [[ ${STACK_crtm_install_fix:-} =~ [yYtT] ]]; then
     if [[ ! -d crtm_fix-${install_as} ]]; then
       crtm_tarball=fix_REL-${install_as}_emc.tgz
-     [[ -f $crtm_tarball ]] || ( $WGET ftp://ftp.ucar.edu/pub/cpaess/bjohns/$crtm_tarball )
+      [[ -f $crtm_tarball ]] || ( $WGET http://ftp.ssec.wisc.edu/pub/s4/CRTM/$crtm_tarball )
       tar xzf $crtm_tarball
-      mv fix crtm_fix-${install_as}
-#      rm -f $crtm_tarball
-    fi
-    if [[ "${install_as}" == "2.3.0" ]]; then
-     if [[ ! -f link_crtm_coeffs.sh ]]; then
-       $WGET https://raw.githubusercontent.com/NOAA-EMC/GSI/master/ush/link_crtm_coeffs.sh
-       sed -i'.backup' -e 's/LINK="ln -sf"/LINK="cp"/g' link_crtm_coeffs.sh
-       chmod +x link_crtm_coeffs.sh
-       rm -f link_crtm_coeffs.sh.backup
-     fi
+      [[ "${install_as}" == "2.3.0" ]] && mv fix_REL-${install_as}_emc crtm_fix-${install_as}
+      [[ "${install_as}" == "2.4.0" ]] && mv crtm-internal_REL-${install_as}_emc/fix crtm_fix-${install_as}
+      rm -f $crtm_tarball
     fi
   fi
 fi
@@ -276,11 +264,12 @@ cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"}
 
 [[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0
 [[ -d $software ]] && cd $software || ( echo "$software does not exist, ABORT!"; exit 1 )
+# A block below with the correction may be needed for certain non-Linux systems (e.g., Gaea/Cray) 
 [[ -d build ]] && rm -rf build
 mkdir -p build && cd build
 
 cmake .. \
-  -DCMAKE_INSTALL_PREFIX=$prefix \
+  -DCMAKE_INSTALL_PREFIX=$prefix -DCMAKE_Fortran_COMPILER=${FC} \
   -DENABLE_TESTS=OFF -DOPENMP=${openmp} ${extraCMakeFlags:-}
 
 VERBOSE=$MAKE_VERBOSE make -j${NTHREADS:-4}
@@ -294,23 +283,21 @@ cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"}
 if [[ "$name" == "crtm" ]]; then
   if [[ ${STACK_crtm_install_fix:-} =~ [yYtT] ]]; then
     if [[ -d crtm_fix-${install_as} ]]; then
-     if [[ "${install_as}" == "2.3.0" ]]; then
-       ./link_crtm_coeffs.sh ./crtm_fix-${install_as} $prefix/fix
-     else
-       mkdir -p $prefix/fix
-       cp ./crtm_fix-${install_as}/ACCoeff/netcdf/* $prefix/fix
-       cp ./crtm_fix-${install_as}/AerosolCoeff/Big_Endian/* $prefix/fix
-       cp ./crtm_fix-${install_as}/AerosolCoeff/netCDF/* $prefix/fix
-       cp ./crtm_fix-${install_as}/CloudCoeff/Big_Endian/* $prefix/fix
-       cp ./crtm_fix-${install_as}/CloudCoeff/netCDF/* $prefix/fix
-       cp ./crtm_fix-${install_as}/EmisCoeff/*/Big_Endian/* $prefix/fix
-       cp ./crtm_fix-${install_as}/EmisCoeff/*/*/Big_Endian/* $prefix/fix
-       cp ./crtm_fix-${install_as}/SpcCoeff/Big_Endian/* $prefix/fix
-       cp ./crtm_fix-${install_as}/SpcCoeff/netcdf/* $prefix/fix
-       cp ./crtm_fix-${install_as}/TauCoeff/ODPS/Big_Endian/* $prefix/fix
-       mv $prefix/fix/amsua_metop-c.SpcCoeff.bin $prefix/fix/amsua_metop-c.SpcCoeff.noACC.bin
-       cp ./crtm_fix-${install_as}/SpcCoeff/Little_Endian/amsua_metop-c_v2.SpcCoeff.bin $prefix/fix/amsua_metop-c.SpcCoeff.bin
-     fi
+      mkdir -p $prefix/fix
+      cp -p ./crtm_fix-${install_as}/AerosolCoeff/Big_Endian/* $prefix/fix
+      cp -p ./crtm_fix-${install_as}/CloudCoeff/Big_Endian/* $prefix/fix
+      cp -p ./crtm_fix-${install_as}/EmisCoeff/*/Big_Endian/* $prefix/fix
+      cp -p ./crtm_fix-${install_as}/EmisCoeff/*/*/Big_Endian/* $prefix/fix
+      cp -p ./crtm_fix-${install_as}/SpcCoeff/Big_Endian/* $prefix/fix
+      cp -p ./crtm_fix-${install_as}/TauCoeff/ODPS/Big_Endian/* $prefix/fix
+      if [[ "${install_as}" == "2.4.0" ]]; then
+         cp -p ./crtm_fix-${install_as}/ACCoeff/netcdf/* $prefix/fix
+         cp -p ./crtm_fix-${install_as}/AerosolCoeff/netCDF/* $prefix/fix
+         cp -p ./crtm_fix-${install_as}/CloudCoeff/netCDF/* $prefix/fix
+         cp -p ./crtm_fix-${install_as}/SpcCoeff/netcdf/* $prefix/fix
+         mv $prefix/fix/amsua_metop-c.SpcCoeff.bin $prefix/fix/amsua_metop-c.SpcCoeff.noACC.bin
+         cp -p ./crtm_fix-${install_as}/SpcCoeff/Little_Endian/amsua_metop-c_v2.SpcCoeff.bin $prefix/fix/amsua_metop-c.SpcCoeff.bin
+      fi
     fi
   fi
 fi