Merge pull request #82 from cropsinsilico/topic/hpc

Topic/hpc

Author: langmm

.github/workflows/test-install.yml

@@ -39,6 +39,7 @@ prof
 
 # Coverage
 .coveragerc
+.coveragerc_parallel
 .coverage
 htmlcov
 .coverage.*

.gitignore

@@ -2,6 +2,21 @@
 History
 =======
 
+1.7.0 (2021-08-26) Support for MPI communicators, MPI execution, and pika >= 1.0.0
+------------------
+
+* Allow models to be run on distributed processes via MPI
+* Added support for MPI based comms
+* Update the required version of pika to be >=1.0.0 and update the RMQComm/RMQAsyncComm code to use the updated API
+* Added C, C++, Fortran, Matlab, R versions of server in rpc_lesson1 example
+* Added C, C++, Fortran, Matlab, R versions of server in rpc_lesson2 example
+* Added C, C++, Fortran, Matlab, R versions of server in rpc_lesson2b example
+* Added C, C++, Fortran, Matlab, R versions of server in rpc_lesson3 example
+* Added C, C++, Fortran, Matlab, R versions of server in rpc_lesson3b example
+* Added C++, Fortran, and Python versions of client in rpc_lesson3b example (still need to thread the Python version and add R & Matlab versions)
+* Fixed bug in yggdevup CLI for missing language directories
+* Enhance debug information w/ task status
+
 1.6.4 (2021-08-10) More minor bug fixes & Automated iteration
 ------------------
 

HISTORY.rst

@@ -0,0 +1,57 @@
+.. _hpc_rst:
+
+
+High Performance Computing (HPC)
+================================
+
+As of version 1.7, |yggdrasil| includes support for running integrations (networks of models) on High Performance Computing (HPC) clusters via MPI (either `OpenMPI <https://www.open-mpi.org/>`_ or `MPICH <https://www.mpich.org/>`_) and `mpi4py <https://mpi4py.readthedocs.io/en/stable/>`_. |yggdrasil| will distribute models in an integration evenly amongst the available processes and coordinate message passing between them.
+
+You can run a |yggdrasil| integration using MPI, by passing the usual ``yggrun`` command to ``mpiexec/mpirun/srun`` e.g.::
+
+  $ mpiexec -n 2 yggrun model1.yml model2.yml
+
+Setup
+-----
+
+To run an integration using |yggdrasil|, you must be able to install |yggdrasil| on the cluster. The existing packages and permissions of the cluster will determine how |yggdrasil| should be installed.
+
+* If conda is already installed on the target cluster, you can install |yggdrasil| directly into an environment of your choosing. If the target cluster uses a package manager, you may need to activate/load the conda installation.::
+
+    $ conda install -c conda-forge yggdrasil
+
+* If conda is not installed, you can install miniconda into your home directory (which you should have permission for) via a login node and then install |yggdrasil| via conda (you may need to log out and back in before using conda for the first time). e.g.::
+
+    $ wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+    $ bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH"
+
+* If conda is not installed and you cannot install conda due to limitations on packages for cluster users, but Python and pip are installed, you can install |yggdrasil| via pip as a user package. If the target cluster uses a package manager, you may need to activate/load a Python installation (version 3.6 or higher). You will need to complete additional installation steps for non-Python languages as described for the normal :ref:`pip-based installation <manual_install_rst>`::
+
+    $ python -m pip install yggdrasil-framework --user
+
+
+Writing the Job Script
+----------------------
+
+Many of the specifics of how you would set up your job script will depend on the scheduler used by the cluster you are targeting (e.g. SLURM, Torque), but there are some general tips:
+
+* Make sure that the YAML files defining the integration and the model codes are located in a directory accessible on compute nodes (usually your home directory or a dedicated drive).
+* Activate/load packages you will need that are provided by the cluster package manager (e.g. conda, Python, MPI).
+* If you used conda to install |yggdrasil|, activate the environment that |yggdrasil| was installed in before calling ``mpiexec/mpirun/srun``.
+
+Example SLURM Job Script
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: slurm
+
+   #!/bin/bash
+   #SBATCH --partition=general
+   #SBATCH --job-name=example_name
+   #SBATCH --cpus-per-task 4
+   #SBATCH --ntasks=4
+   #SBATCH --mem-per-cpu=500MB
+
+   module load miniconda
+
+   conda activate env_name
+
+   mpiexec -np $SLURM_NTASKS yggrun model1.yml model2.yml > output.txt

docs/source/hpc.rst

@@ -22,6 +22,7 @@ Welcome to yggdrasil's documentation!
    yaml
    config
    units
+   hpc
    c_format_strings
    debugging
    threading

docs/source/index.rst

@@ -1,5 +1,5 @@
 {% set name = "yggdrasil" %}
-{% set version = "1.6.3" %}
+{% set version = "1.7.0" %}
 
 package:
   name: {{ name|lower }}

recipe/meta.yaml

@@ -1,6 +1,7 @@
 astropy       # [astropy]
-pika<1.0.0b1  # [rmq]
+pika>=1.0.0   # [rmq]
 pygments      # [pygments]
 trimesh       # [trimesh]
 libroadrunner  # [pip,sbml]
 ccache; platform_system != 'Windows'  # [conda]
+mpi4py        # [mpi]

requirements_optional.txt

@@ -9,3 +9,4 @@ pytest-timeout
 pytest-drop-dup-tests  # [pip]
 pytest-rerunfailures
 pytest-repeat
+pytest-mpi

requirements_testing.txt

@@ -29,6 +29,8 @@ omit =
      */yggdrasil/doctools.py
      */yggdrasil/demos/*
      */yggdrasil/__main__.py
+     */yggdrasil/communication/tests/conftest.py
+     */yggdrasil/examples/tests/conftest.py
 
 [coverage:report]
 sort = Cover

setup.cfg

@@ -105,34 +105,50 @@ def call_script(lines, force_bash=False):
     """
     if not lines:
         return
-    if _is_win and (not force_bash):  # pragma: windows
-        script_ext = '.bat'
-        error_check = 'if %errorlevel% neq 0 exit /b %errorlevel%'
-        for i in range(len(lines), 0, -1):
-            lines.insert(i, error_check)
-    else:
-        script_ext = '.sh'
-        if lines[0] != '#!/bin/bash':
-            lines.insert(0, '#!/bin/bash')
-        error_check = 'set -e'
-        if error_check not in lines:
-            lines.insert(1, error_check)
-    fname = 'ci_script_%s%s' % (str(uuid.uuid4()), script_ext)
-    try:
-        pprint.pprint(lines)
-        with open(fname, 'w') as fd:
-            fd.write('\n'.join(lines))
-            
-        call_kws = {}
-        if _is_win:  # pragma: windows
-            call_cmd = [fname]
+    # Split lines that should be allowed to fail
+    line_sets = []
+    idx = 0
+    for i, line in enumerate(lines):
+        if line.endswith('# [ALLOW FAIL]'):
+            line_sets.append(lines[idx:i])
+            line_sets.append([lines[i]])
+            idx = i + 1
+    line_sets.append(lines[idx:])
+    for lines in line_sets:
+        allow_failure = lines[0].endswith('# [ALLOW FAIL]')
+        if allow_failure:
+            lines = [lines[0].split('#')[0].strip()]
+        if _is_win and (not force_bash):  # pragma: windows
+            script_ext = '.bat'
+            error_check = 'if %errorlevel% neq 0 exit /b %errorlevel%'
+            for i in range(len(lines), 0, -1):
+                lines.insert(i, error_check)
         else:
-            call_cmd = ['./%s' % fname]
-            os.chmod(fname, 0o755)
-        subprocess.check_call(call_cmd, **call_kws)
-    finally:
-        if os.path.isfile(fname):
-            os.remove(fname)
+            script_ext = '.sh'
+            if lines[0] != '#!/bin/bash':
+                lines.insert(0, '#!/bin/bash')
+            error_check = 'set -e'
+            if error_check not in lines:
+                lines.insert(1, error_check)
+        fname = 'ci_script_%s%s' % (str(uuid.uuid4()), script_ext)
+        try:
+            pprint.pprint(lines)
+            with open(fname, 'w') as fd:
+                fd.write('\n'.join(lines))
+                
+            call_kws = {}
+            if _is_win:  # pragma: windows
+                call_cmd = [os.environ['COMSPEC'], '/c', 'call', fname]
+            else:
+                call_cmd = ['./%s' % fname]
+                os.chmod(fname, 0o755)
+            subprocess.check_call(call_cmd, **call_kws)
+        except subprocess.CalledProcessError:
+            if not allow_failure:
+                raise
+        finally:
+            if os.path.isfile(fname):
+                os.remove(fname)
 
 
 def conda_env_exists(name):
@@ -217,6 +233,7 @@ def get_install_opts(old=None):
             'omp': (os.environ.get('INSTALLOMP', '0') == '1'),
             'docs': (os.environ.get('BUILDDOCS', '0') == '1'),
             'no_sudo': False,
+            'mpi': (os.environ.get('INSTALLMPI', '0') == '1'),
         }
         if not _is_win:
             new['c'] = True  # c compiler usually installed by default
@@ -235,6 +252,7 @@ def get_install_opts(old=None):
             'omp': False,
             'docs': False,
             'no_sudo': False,
+            'mpi': False,
         }
     if _is_win:
         new['os'] = 'win'
@@ -585,6 +603,18 @@ def itemize_deps(method, for_development=False,
             out['os'] += ['libomp', 'llvm']
         elif install_opts['os'] == 'win':
             pass
+    if install_opts['mpi'] and (not fallback_to_conda):
+        if install_opts['os'] == 'linux':
+            out['os'] += ['openmpi-bin', 'libopenmpi-dev']
+        elif install_opts['os'] == 'osx':
+            out['os'].append('openmpi')
+        elif install_opts['os'] == 'win':
+            pass
+    elif ((install_opts['os'] == 'win') and install_opts['mpi']
+          and (method == 'conda')):
+        # Force mpi4py to be installed last on Windows to avoid
+        # conflicts
+        out['skip'].append('mpi4py')
     if install_opts['fortran'] and (not fallback_to_conda):
         # Fortran is not installed via conda on linux/macos
         if install_opts['os'] == 'linux':
@@ -851,6 +881,18 @@ def install_deps(method, return_commands=False, verbose=False,
             raise RuntimeError("Could not detect conda environment. "
                                "Cannot proceed with a conda deployment "
                                "(required for LPy).")
+    # if _is_win and install_opts['mpi'] and fallback_to_conda:
+    #     if verbose:
+    #         install_flags = '-vvv'
+    #     else:
+    #         install_flags = '-q'
+    #     install_flags = '-vv'
+    #     install_flags += conda_flags
+    #     # This is required as the install script for mpi4py aborts without
+    #     # an error message when called inside a Python subprocess. This seems
+    #     # to occur during cleanup for the installation process as the mpi4py
+    #     # installation is functional. Possibly triggered by the activation script?
+    #     cmds += ["%s install %s mpi4py # [ALLOW FAIL]" % (CONDA_CMD, install_flags)]
     if return_commands:
         return cmds
     cmds = SUMMARY_CMDS + cmds + SUMMARY_CMDS
@@ -962,6 +1004,8 @@ def install_pkg(method, python=None, without_build=False,
             # "%s install %s --update-deps -c %s yggdrasil \"blas=*=openblas\"" % (
             #     CONDA_CMD, install_flags, index_channel)
         ]
+        if _is_win and install_opts['mpi']:
+            cmds[-1] = cmds[-1] + ' mpi4py # [ALLOW FAIL]'
     elif method == 'pip':
         if verbose:
             install_flags = '--verbose'