Skip to content

Commit

Permalink
Updates for MacOS
Browse files Browse the repository at this point in the history
  • Loading branch information
@natalie-perlin
natalie-perlin committed Nov 10, 2022
2 parents 8ba0750 + c680449 commit f0afba1
Show file tree
Hide file tree
Showing 36 changed files with 744 additions and 115 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build_intel.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
export HPC_PYTHON="python/${python_ver}"
yes | ./setup_modules.sh -c config/config_custom.sh -p $prefix
./build_stack.sh -p $prefix -c config/config_custom.sh -y stack/stack_custom.yaml -m
- name: Upload logs
if: ${{ always() }}
uses: actions/upload-artifact@v2
Expand Down
12 changes: 7 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,27 @@ Data assimilation Integration
(JEDI)](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/)
framework.

Documentation is available at https://hpc-stack.readthedocs.io/en/latest/.

This is part of the [NCEPLIBS](https://github.com/NOAA-EMC/NCEPLIBS) project.

## Authors

Rahul Mahajan, Kyle Gerheiser, Dusan Jovic, Hang-Lei, Dom Heinzeller

Code Manager: Kyle Gerheiser
Code Manager: Alex Richert

Installers:

Machine | Programmer
------------|------------------
Hera | Kyle Gerheiser
Jet | Kyle Gerheiser
------------|------------------------------------------
Hera | Jong Kim, Natalie Perlin, Cameron Book
Jet | Jong Kim, Natalie Perlin, Cameron Book
Orion | Hang-Lei
WCOSS-Dell | Hang-Lei
WCOSS-Cray | Hang-Lei
Cheyenne | Jong Kim
Gaea | Kyle Gerheiser
Gaea | Jong Kim, Natalie Perlin, Cameron Book

## Contributors

Expand Down
8 changes: 8 additions & 0 deletions build_stack.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ build_lib zlib
build_lib libpng
build_lib szip
build_lib jasper
build_lib gsl
build_lib sqlite
build_lib libtiff
build_lib proj
Expand Down Expand Up @@ -207,6 +208,7 @@ build_lib wgrib2
build_lib prod_util
build_lib grib_util
build_lib ncio
build_lib ncdiag

if $MODULES; then

Expand Down Expand Up @@ -251,6 +253,12 @@ build_lib eckit
build_lib fckit
build_lib atlas

# MET and METplus

build_lib met
build_lib metplus


# UFS 3rd party dependencies

build_lib esmf
Expand Down
84 changes: 69 additions & 15 deletions cmake/wgrib2/wgrib2-config.cmake
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,75 @@
find_path (WGRIB2_INCLUDES
wgrib2api.mod
HINTS $ENV{wgrib2_ROOT}/include)
# Find the wgrib2 headers, library and executable
#
# This module defines:
#
# - wgrib2::wgrib2 - library and include directory, all in a single target.
# - WGRIB2_INCLUDE_DIR - include directory
# - WGRIB2_LIBRARIES - wgrib2 library
# - WGRIB2_EXE - wgrib2 executable
#
# The following paths will be searched in order:
#
# - WGRIB2_INCLUDE_DIRS - folders containing wgrib2.
# - WGRIB2_LIBRARY_DIRS - folders containing libwgrib2.a
# - wgrib2_ROOT - root of wgrib2 installation
# - wgrib2_PATH - root of wgrib2 installation

find_library (WGRIB2_LIBRARIES
names libwgrib2.a
HINTS $ENV{wgrib2_ROOT}/lib)
find_path(
WGRIB2_INCLUDE_DIR
NAMES wgrib2.h wgrib2api.mod
HINTS ${WGRIB2_INCLUDE_DIRS}
${wgrib2_ROOT} $ENV{wgrib2_ROOT}
${wgrib2_PATH} $ENV{wgrib2_PATH}
PATH_SUFFIXES include include/wgrib2
DOC "Path to wgrib2.h, wgrib2api.mod"
)

if(EXISTS ${WGRIB2_INCLUDES} AND EXISTS ${WGRIB2_LIBRARIES})
message(STATUS "Found WGRIB2: include directory ${WGRIB2_INCLUDES}, library ${WGRIB2_LIBRARIES}")
find_library(
WGRIB2_LIBRARIES
NAMES libwgrib2.a
HINTS ${WGRIB2_LIBRARY_DIRS}
${wgrib2_ROOT} $ENV{wgrib2_ROOT}
${wgrib2_PATH} $ENV{wgrib2_PATH}
PATH_SUFFIXES lib lib64
DOC "Path to libwgrib2.a"
)

find_program(WGRIB2_EXE
NAMES wgrib2
HINTS ${wgrib2_ROOT} $ENV{wgrib2_ROOT}
${wgrib2_PATH} $ENV{wgrib2_PATH}
PATH_SUFFIXES bin
DOC "Path to wgrib2 executable"
)

mark_as_advanced(WGRIB2_INCLUDE_DIR WGRIB2_LIBRARIES WGRIB2_EXE)

add_library(wgrib2::wgrib2 UNKNOWN IMPORTED)
set_target_properties(wgrib2::wgrib2 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${WGRIB2_INCLUDE_DIR})
set_target_properties(wgrib2::wgrib2 PROPERTIES IMPORTED_LOCATION ${WGRIB2_LIBRARIES})
set_target_properties(wgrib2::wgrib2 PROPERTIES INTERFACE_LINK_LIBRARIES ${WGRIB2_LIBRARIES})

# wgrib2 changed how it outputs --version from "v0.x.y.z" to "vx.y.z" starting in wgrib2 3.0
execute_process(COMMAND ${WGRIB2_EXE} --version OUTPUT_VARIABLE version)
if(version MATCHES "^v0.*")
string(SUBSTRING "${version}" 3 5 WGRIB2_VERSION)
else()
message(STATUS "Unable to locate WGRIB2 library and/or Fortran modules")
string(SUBSTRING "${version}" 1 5 WGRIB2_VERSION)
endif()
unset(version)

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(wgrib2
REQUIRED_VARS WGRIB2_LIBRARIES WGRIB2_INCLUDE_DIR WGRIB2_EXE
VERSION_VAR WGRIB2_VERSION
)

if(wgrib2_FOUND AND NOT wgrib2_FIND_QUIETLY)
message(STATUS "Findwgrib2:")
message(STATUS " - WGRIB2_INCLUDE_DIR: ${WGRIB2_INCLUDE_DIR}")
message(STATUS " - WGRIB2_LIBRARIES: ${WGRIB2_LIBRARIES}")
message(STATUS " - WGRIB2_EXE: ${WGRIB2_EXE}")
message(STATUS " - WGRIB2_VERSION: ${WGRIB2_VERSION}")
endif()

mark_as_advanced (WGRIB2_INCLUDES WGRIB2_LIBRARIES)

add_library(wgrib2::wgrib2 UNKNOWN IMPORTED)
set_target_properties(wgrib2::wgrib2 PROPERTIES
IMPORTED_LOCATION "${WGRIB2_LIBRARIES}"
INTERFACE_INCLUDE_DIRECTORIES "${WGRIB2_INCLUDES}"
INTERFACE_LINK_LIBRARIES "${WGRIB2_LIBRARIES}")
5 changes: 5 additions & 0 deletions config/config_wcoss_dell_p3.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,8 @@ module load git/2.14.3
# Build FMS with AVX2 flags
export STACK_fms_CFLAGS="-march=core-avx2"
export STACK_fms_FFLAGS="-march=core-avx2"

# python3-config on WCOSS points to non-existant directories
export MET_PYTHON="/usrx/local/prod/packages/python/3.6.3"
export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.6m"
export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.6m -lpthread -ldl -lutil -lm -Xlinker -export-dynamic"
7 changes: 4 additions & 3 deletions docs/source/hpc-components.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
HPC-Stack Components
=====================

The HPC-Stack packages are built in :numref:`Step %s <NonConHPCBuild>` using the ``build_stack.sh`` script. The following software can optionally be built with the scripts under ``libs``.
The HPC-Stack packages are built in :numref:`Step %s <NonConHPCBuild>` using the ``build_stack.sh`` script. The following software can optionally be built with the scripts under ``libs``.

* **Compilers and MPI libraries**

Expand All @@ -21,6 +21,7 @@ The HPC-Stack packages are built in :numref:`Step %s <NonConHPCBuild>` using the
* `CMake <https://cmake.org/>`__
* `Udunits <https://www.unidata.ucar.edu/software/udunits/>`__
* `PNG <http://www.libpng.org/pub/png/>`__
* `TIFF <https://gitlab.com/libtiff/libtiff.git>`__
* `JPEG <https://jpeg.org/>`__
* `Jasper <https://github.com/jasper-software/jasper>`__
* `SZip <https://support.hdfgroup.org/doc_resource/SZIP/>`__
Expand All @@ -34,7 +35,6 @@ The HPC-Stack packages are built in :numref:`Step %s <NonConHPCBuild>` using the
* `CDO <https://code.mpimet.mpg.de/projects/cdo>`__
* `FFTW <http://www.fftw.org/>`__
* `GPTL <https://jmrosinski.github.io/GPTL/>`__
* Tau2
* `Boost <https://beta.boost.org/>`__
* `Eigen <http://eigen.tuxfamily.org/>`__
* `GSL-Lite <http://github.com/gsl-lite/gsl-lite>`__
Expand Down Expand Up @@ -77,7 +77,8 @@ The HPC-Stack packages are built in :numref:`Step %s <NonConHPCBuild>` using the
* `NCEPLIBS-ncio <https://github.com/noaa-emc/nceplibs-ncio.git>`__
* `NCEPLIBS-wrf_io <https://github.com/noaa-emc/nceplibs-wrf_io.git>`__
* `EMC_crtm <https://github.com/noaa-emc/EMC_crtm.git>`__
* `EMC_post <https://github.com/noaa-emc/EMC_post.git>`__
* `UPP <https://github.com/NOAA-EMC/UPP>`__
* `GSI-ncdiag <https://github.com/noaa-emc/GSI-ncdiag.git>`__


* **JEDI Dependencies**
Expand Down
116 changes: 101 additions & 15 deletions docs/source/hpc-install.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ Build the HPC-Stack
sed -i "10 a export PATH=/usr/local/sbin:/usr/local/bin:$PATH" ./build_stack.sh
sed -i "10 a export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH" ./build_stack.sh
#. Build the environment. This may take several hours to complete.
#. Build the environment. This may take up to a couple of hours to complete.

.. code-block:: console
Expand Down Expand Up @@ -119,7 +119,7 @@ To install the HPC-Stack locally, the following pre-requisites must be installed
* **Python 3:** Can be obtained either from the `main distributor <https://www.python.org/>`_ or from `Anaconda <https://www.anaconda.com/>`_.
* **Compilers:** Distributions of Fortran, C, and C++ compilers that work for your system.
* **Message Passing Interface (MPI)** libraries for multi-processor and multi-core communications, configured to work with your corresponding Fortran, C, and C++ compilers.
* **Programs and software packages:** `Lmod <https://lmod.readthedocs.io/en/latest/030_installing.html>`_, `CMake <https://cmake.org/install/>`_, `make <https://www.gnu.org/software/make/>`_, `wget <https://www.gnu.org/software/wget/>`_, `curl <https://curl.se/>`_, `git <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`_, and the `TIFF library <https://gitlab.com/libtiff/libtiff.git>`_.
* **Programs and software packages:** `Lmod <https://lmod.readthedocs.io/en/latest/030_installing.html>`_, `CMake <https://cmake.org/install/>`_, `make <https://www.gnu.org/software/make/>`_, `wget <https://www.gnu.org/software/wget/>`_, `curl <https://curl.se/>`_, `git <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`_.

.. note::
For detailed instructions on how to build the HPC-Stack on two particular configurations of MacOS, see :numref:`Chapter %s <MacInstall>`
Expand All @@ -132,18 +132,105 @@ To determine whether these prerequisites are installed, query the environment va
which cmake
cmake --version
Methods for determining whether ``libtiff`` is installed vary between systems. Users can try the following approaches:
If compilers or MPI's need to be installed, consult the :ref:`HPC-Stack Prerequisites <Prerequisites>` document for further guidance. MPI could also be built bundled with the HPC-stack, if not pre-installed in the system.

.. code-block:: console
.. _UbuntuLinux:

Ubuntu 20.04 Linux Example
--------------------------

The example for Ubuntu 20.4 is for a user with sudo privileges to install system-wide. First, install GNU 10 compilers:

.. code-block:: console
sudo apt install gcc-10 g++-10
sudo apt install gfortran-10
which gcc
The location of default compilers is likely be in ``/usr/bin/`` (e.g. /usr/bin/gcc), and other versions are installed with the same location with the version tag. Check all the versions installed and configure the alternatives to specify which version is to be default. Below is the example when two versions are available, e.g., gcc-9 and gcc-10.

.. code-block:: console
ls -la /usr/bin/gcc-*
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 2
sudo update-alternatives --config gcc
Make selection (1) for gcc-10 to be the first priority (default).

Repeat similar steps to configure the alternatives for g++ and gfortran. No need to configure for gfortran if you have only a single version.

.. code-block:: console
ls -la /usr/bin/g++-*
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 1
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 2
sudo update-alternatives --config g++
Install ``lua``, ``luac`` and ``tcl`` needed to support the ``Lmod`` module management system. Verify the installation location.

.. code-block:: console
sudo apt-get install lua5.3 liblua5.3-0 liblua5.3-dev lua-filesystem-dev lua-posix-dev
sudo apt-get install tcl tcl8.6-dev libtcl8.6
which lua
which luac
Standard installation paths for lua/luac are ``/usr/bin/lua`` and ``/usr/bin/luac``.
Download and install Lmod, the module management environment, if not installed in the system. When installed, the environmental variable $LMOD_ROOT is defined:

.. code-block:: console
echo $LMOD_ROOT
git clone https://github.com/TACC/Lmod.git
cd Lmod
whereis libtiff
locate libtiff
ldconfig -p | grep libtiff
ls /usr/lib64/libtiff*
ls /usr/lib/libtiff*
See the INSTALL file for instructions; configure and install. Use the paths for ``lua`` and ``luac`` installed in the previous step. ``Lmod`` could be configured to be installed system-wide, or in user space as in the example below, e.g., in ``${HOME}/apps``.

.. code-block:: console
./configure --with-lua=/usr/bin/lua --with-luac=/usr/bin/luac --prefix=${HOME}/apps
make install
Add the Lmod environment initialization to your shell profile, i.e. to $HOME/.bashrc or $HOME/.bash_profile:

.. code-block:: console
export BASH_ENV=$HOME/apps/lmod/lmod/init/profile
source $BASH_ENV
Install wget, git, make and cmake; cmake could also be built bundled with the hpc-stack.

.. code-block:: console
sudo apt install git
sudo apt install wget
sudo apt install make
sudo apt install cmake
Install openssl, libssl-dev packages:

.. code-block:: console
sudo apt install openssl
sudo apt-get install libssl-dev
Both ``python2`` and ``python3`` are needed; python2 version higher than 2.7.x; python3 version higher than 3.6. You could update ``python3`` if higher versions are available (python3.9 in the example below at the time of writing), and set the alternatives when multiple versions exist. The highest version among the existing python3.8 and python3.9 is set as a default in the example below (choose selection 1 for python3.9 when prompt).

.. code-block:: console
which python3
python3 --version
apt list | grep python
sudo apt install python3.9
sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2
sudo update-alternatives --config python3
python3 --version
Verify the version (python 3.9) set as default after the configuration.

If compilers or MPI's need to be installed, consult the :ref:`HPC-Stack Prerequisites <Prerequisites>` document for further guidance.

.. _NonConConfigure:

Expand Down Expand Up @@ -202,22 +289,21 @@ These ``hpc-`` modules are really meta-modules that load the compiler/MPI librar

In short, you may prefer not to load the compiler or MPI modules directly. Instead, loading the hpc- meta-modules as demonstrated above will provide everything needed to load software libraries.

It may be necessary to set certain source and path variables in the ``build_stack.sh`` script. For example:
It may be necessary to set few environment variables in the ``build_stack.sh`` script. For example:
r

.. code-block:: console
source /usr/share/lmod/6.6/init/bash
source /usr/share/lmod/lmod/init/bash
export PATH=/usr/local/sbin:/usr/local/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
It may also be necessary to initialize ``Lmod`` when using a user-specific ``Lmod`` installation:
``Lmod`` needs to be initialized based on the installation directory:

.. code-block:: console
module purge
export BASH_ENV=$HOME/<Lmod-installation-dir>/lmod/lmod/init/bash
export BASH_ENV=<Lmod-installation-dir>/lmod/lmod/init/profile
source $BASH_ENV
export LMOD_SYSTEM_DEFAULT_MODULES=<module1>:<module2>:<module3>
module --initial_load --no_redirect restore
Expand Down
6 changes: 3 additions & 3 deletions docs/source/hpc-notes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ HPC-Stack Additional Notes
Setting Compiler Flags and Other Options
-----------------------------------------

Often it is necessary to specify compiler flags (e.g., ``gfortran-10 -fallow-argument-mismatch`` for the packages via ``FFLAGS``. There are 2 ways this can be achieved:
Often it is necessary to specify compiler flags (e.g., gfortran-10 requires ``-fallow-argument-mismatch``to be used with ``FFLAGS``. There are 2 ways this can be achieved:

#. **For all packages:** One can define variable e.g., ``STACK_FFLAGS=-fallow-argument-mismatch`` in the config file ``config_custom.sh``. This will append ``STACK_FFLAGS`` to ``FFLAGS`` in every build script under libs.
#. **For all packages:** One can define variable e.g., ``STACK_FFLAGS=-fallow-argument-mismatch`` in the config file ``config_custom.sh``. This will append ``STACK_FFLAGS`` to ``FFLAGS`` in every build script under ./libs/ directory.

#. **Package specific flags:** To compile only the specific package under ``libs`` with the above compiler flag, one can define variable ``FFLAGS=-fallow-argument-mismatch`` in the ``<package>`` section of the YAML file ``stack_custom.yaml``. This will append ``STACK_<package>_FFLAGS`` to ``FFLAGS`` in the build script for that package only.
#. **Package-specific flags:** To compile only the specific package under ``libs`` with the above compiler flag, one can define variable ``FFLAGS=-fallow-argument-mismatch`` in the ``<package>`` section of the YAML file ``stack_custom.yaml``. This will append ``STACK_<package>_FFLAGS`` to ``FFLAGS`` in the build script for that package only.

Adding a New Library or Package
--------------------------------
Expand Down
4 changes: 3 additions & 1 deletion docs/source/hpc-parameters.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ Compiler & MPI
``HPC_COMPILER``:
This defines the vendor and version of the compiler you wish to use for this build. The format is the same as what you would typically use in a module load command. For example, ``HPC_COMPILER=intel/2020``. Options include:

* ``gnu/6.5.0``
* ``gnu/9.2.0``
* ``gnu/10.1.0``
* ``gnu/11.2.0``
* ``intel/18.0.5.274``
* ``intel/19.0.5.281``
* ``intel/2020``
Expand All @@ -31,6 +32,7 @@ Compiler & MPI
* ``impi/2020.2``
* ``impi/2021.3.0``
* ``mvapich2/2.3``
* ``mpich/3.3.2``
* ``openmpi/4.1.2``

.. note::
Expand Down
Loading

0 comments on commit f0afba1

Please sign in to comment.