Merge branch 'develop' of https://github.com/easybuilders/easybuild-easyblocks into empty_dir

Author: smoors

easybuild/easyblocks/a/amber.py

@@ -88,35 +88,37 @@ def extract_step(self):
     def patch_step(self, *args, **kwargs):
         """Patch Amber using 'update_amber' tool, prior to applying listed patch files (if any)."""
 
-        # figure out which Python command to use to run the update_amber script;
-        # by default it uses 'python', but this may not be available (on CentOS 8 for example);
-        # note that the dependencies are not loaded yet at this point, so we're at the mercy of the OS here...
-        python_cmd = None
-        for cand_python_cmd in ['python', 'python3', 'python2']:
-            if which(cand_python_cmd):
-                python_cmd = cand_python_cmd
-                break
-
-        if python_cmd is None:
-            raise EasyBuildError("No suitable Python command found to run update_amber script!")
-
-        if self.cfg['patchlevels'] == "latest":
-            cmd = "%s ./update_amber --update" % python_cmd
-            # Run as many times as specified. It is the responsibility
-            # of the easyconfig author to get this right, especially if
-            # he or she selects "latest". (Note: "latest" is not
-            # recommended for this reason and others.)
-            for _ in range(self.cfg['patchruns']):
-                run_shell_cmd(cmd)
-        else:
-            for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']):
-                if patch_level == 0:
-                    continue
-                cmd = "%s ./update_amber --update-to %s/%s" % (python_cmd, tree, patch_level)
+        # Use the update_amber script if patchlevels is defined - if not then the easyconfig should apply the patches
+        if self.cfg['patchlevels']:
+            # figure out which Python command to use to run the update_amber script;
+            # by default it uses 'python', but this may not be available (on CentOS 8 for example);
+            # note that the dependencies are not loaded yet at this point, so we're at the mercy of the OS here...
+            python_cmd = None
+            for cand_python_cmd in ['python', 'python3', 'python2']:
+                if which(cand_python_cmd):
+                    python_cmd = cand_python_cmd
+                    break
+
+            if python_cmd is None:
+                raise EasyBuildError("No suitable Python command found to run update_amber script!")
+
+            if self.cfg['patchlevels'] == "latest":
+                cmd = "%s ./update_amber --update" % python_cmd
                 # Run as many times as specified. It is the responsibility
-                # of the easyconfig author to get this right.
+                # of the easyconfig author to get this right, especially if
+                # he or she selects "latest". (Note: "latest" is not
+                # recommended for this reason and others.)
                 for _ in range(self.cfg['patchruns']):
                     run_shell_cmd(cmd)
+            else:
+                for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']):
+                    if patch_level == 0:
+                        continue
+                    cmd = "%s ./update_amber --update-to %s/%s" % (python_cmd, tree, patch_level)
+                    # Run as many times as specified. It is the responsibility
+                    # of the easyconfig author to get this right.
+                    for _ in range(self.cfg['patchruns']):
+                        run_shell_cmd(cmd)
 
         super().patch_step(*args, **kwargs)
 

easybuild/easyblocks/e/elpa.py

@@ -206,7 +206,8 @@ def configure_step(self):
             self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string)
             self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string)
             # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards
-            if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'):
+            # Trying to use these kernels for GPUs newer than sm80 will fail ELPA configure
+            if float(cuda_cc) == 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'):
                 self.cfg.update('configopts', '--enable-nvidia-sm80-gpu')
 
         # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure

easybuild/easyblocks/g/ghc.py

@@ -37,7 +37,7 @@ class EB_GHC(ConfigureMake):
     Support for building and installing applications with configure/make/make install
     """
 
-    def build_step(self, verbose=False):
+    def build_step(self, *args, **kwargs):
         """
         Support for a binary 6.12.x installation. Starting there,
         later GHC versions are build from source and thus require
@@ -46,4 +46,4 @@ def build_step(self, verbose=False):
         if LooseVersion(self.version) < LooseVersion("7.0"):
             pass
         else:
-            super().build_step(verbose=verbose)
+            super().build_step(*args, **kwargs)

easybuild/easyblocks/generic/bundle.py

@@ -35,6 +35,7 @@
 """
 import copy
 import os
+from datetime import datetime
 
 import easybuild.tools.environment as env
 from easybuild.framework.easyblock import EasyBlock
@@ -45,7 +46,17 @@
 from easybuild.tools.config import build_option
 from easybuild.tools.hooks import TEST_STEP
 from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.utilities import nub
+from easybuild.tools.utilities import nub, time2str
+
+
+# Description and step name run during component installation
+COMPONENT_INSTALL_STEPS = [
+    ('patching', 'patch'),
+    ('configuring', 'configure'),
+    ('building', 'build'),
+    ('testing', 'test'),
+    ('installing', 'install'),
+]
 
 
 class Bundle(EasyBlock):
@@ -265,14 +276,39 @@ def build_step(self):
         """Do nothing."""
         pass
 
+    def _install_component(self, comp):
+        """Run the installation steps for a single component"""
+        # run relevant steps
+        for descr, step_name in COMPONENT_INSTALL_STEPS:
+            if step_name in comp.cfg['skipsteps']:
+                comp.log.info("Skipping '%s' step for component %s v%s", step_name, comp.name, comp.version)
+            elif build_option('skip_test_step') and step_name == TEST_STEP:
+                comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
+                              comp.name, comp.version)
+            else:
+                msg = f'   {descr} component {comp.name}...'
+                if self.dry_run:
+                    self.dry_run_msg("%s [DRY RUN]\n", msg)
+                else:
+                    print_msg(msg, log=self.log, silent=self.silent)
+                start_time = datetime.now()
+                try:
+                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+                finally:
+                    if not self.dry_run:
+                        step_duration = datetime.now() - start_time
+                        if step_duration.total_seconds() >= 1:
+                            print_msg("   ... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                        elif self.logdebug or build_option('trace'):
+                            print_msg("   ... (took < 1 sec)", log=self.log, silent=self.silent)
+
     def install_step(self):
         """Install components, if specified."""
         comp_cnt = len(self.cfg['components'])
         for idx, (cfg, comp) in enumerate(self.comp_instances):
-
             print_msg("installing bundle component %s v%s (%d/%d)..." %
-                      (cfg['name'], cfg['version'], idx + 1, comp_cnt))
-            self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock)
+                      (comp.name, comp.version, idx + 1, comp_cnt))
+            self.log.info("Installing component %s v%s using easyblock %s", comp.name, comp.version, cfg.easyblock)
 
             # correct build/install dirs
             comp.builddir = self.builddir
@@ -317,18 +353,10 @@ def install_step(self):
                 comp.src[-1]['finalpath'] = comp.cfg['start_dir']
 
             # check if sanity checks are enabled for the component
-            if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']:
+            if self.cfg['sanity_check_all_components'] or comp.name in self.cfg['sanity_check_components']:
                 self.comp_cfgs_sanity_check.append(comp)
 
-            # run relevant steps
-            for step_name in ['patch', 'configure', 'build', 'test', 'install']:
-                if step_name in cfg['skipsteps']:
-                    comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version'])
-                elif build_option('skip_test_step') and step_name == TEST_STEP:
-                    comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
-                                  cfg['name'], cfg['version'])
-                else:
-                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+            self._install_component(comp)
 
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
                 depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
@@ -381,13 +409,13 @@ def make_module_step(self, *args, **kwargs):
         as this is done in the generic EasyBlock while creating
         the module file already.
         """
-        for cfg, comp in self.comp_instances:
-            self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version'])
+        for _, comp in self.comp_instances:
+            self.log.info("Gathering module paths for component %s v%s", comp.name, comp.version)
 
             # take into account that easyblock used for component may not be migrated yet to module_load_environment
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
 
-                depr_msg = f"Easyblock used to install component {cfg['name']} still uses make_module_req_guess"
+                depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
                 self.log.deprecated(depr_msg, '6.0')
 
                 reqs = comp.make_module_req_guess()
@@ -403,7 +431,7 @@ def make_module_step(self, *args, **kwargs):
                             setattr(self.module_load_environment, key, value)
                 except AttributeError:
                     raise EasyBuildError("Cannot process module requirements of bundle component %s v%s",
-                                         cfg['name'], cfg['version'])
+                                         comp.name, comp.version)
             else:
                 # Explicit call required as adding step to 'install_step' is not sufficient
                 # for module-only build. Set fake arg to True, as module components should
@@ -445,8 +473,7 @@ def sanity_check_step(self, *args, **kwargs):
         # run sanity checks for specific components
         cnt = len(self.comp_cfgs_sanity_check)
         for idx, comp in enumerate(self.comp_cfgs_sanity_check):
-            comp_name, comp_ver = comp.cfg['name'], comp.cfg['version']
-            print_msg("sanity checking bundle component %s v%s (%i/%i)...", comp_name, comp_ver, idx + 1, cnt)
-            self.log.info("Starting sanity check step for component %s v%s", comp_name, comp_ver)
+            print_msg("sanity checking bundle component %s v%s (%i/%i)...", comp.name, comp.version, idx + 1, cnt)
+            self.log.info("Starting sanity check step for component %s v%s", comp.name, comp.version)
 
             comp.run_step('sanity_check', [lambda x: x.sanity_check_step])

easybuild/easyblocks/generic/cmdcp.py

@@ -67,7 +67,9 @@ def build_step(self):
             # determine command to use
             # find (first) regex match, then complete matching command template
             cmd = None
-            for pattern, regex_cmd in self.cfg['cmds_map']:
+            for pattern, regex_cmd in self.cfg.get_ref('cmds_map'):
+                pattern = pattern % self.cfg.template_values
+                regex_cmd = regex_cmd % self.cfg.template_values
                 try:
                     regex = re.compile(pattern)
                 except re.error as err:
@@ -78,6 +80,6 @@ def build_step(self):
                     break
             if cmd is None:
                 raise EasyBuildError("No match for %s in %s, don't know which command to use.",
-                                     src, self.cfg['cmds_map'])
+                                     src, self.cfg.get_ref('cmds_map'))
 
             run_shell_cmd(cmd)

easybuild/easyblocks/generic/mesonninja.py

@@ -130,7 +130,7 @@ def configure_step(self, cmd_prefix=''):
         res = run_shell_cmd(cmd)
         return res.output
 
-    def build_step(self, verbose=False, path=None):
+    def build_step(self, *args, **kwargs):
         """
         Build with Ninja.
         """

easybuild/easyblocks/generic/pythonpackage.py

@@ -870,7 +870,6 @@ def test_step(self, return_output_ec=False):
                     res = run_shell_cmd(cmd, fail_on_error=False)
                     # need to retrieve ec by not failing on error
                     (out, ec) = (res.output, res.exit_code)
-                    self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out)
                 else:
                     run_shell_cmd(cmd)
 

easybuild/easyblocks/generic/scons.py

@@ -51,7 +51,7 @@ def configure_step(self):
         else:
             self.prefix = ''
 
-    def build_step(self, verbose=False):
+    def build_step(self, *args, **kwargs):
         """
         Build with SCons
         """

easybuild/easyblocks/generic/waf.py

@@ -52,7 +52,7 @@ def configure_step(self, cmd_prefix=''):
 
         return res.output
 
-    def build_step(self, verbose=False, path=None):
+    def build_step(self, *args, **kwargs):
         """
         Build with ./waf build
         """

easybuild/easyblocks/h/hpl.py

@@ -30,12 +30,16 @@
 @author: Kenneth Hoste (Ghent University)
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
+@author: Davide Grassano (CECAM - EPFL)
 """
 
+import re
 import os
 
+import easybuild.tools.toolchain as toolchain
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import build_option
 from easybuild.tools.filetools import change_dir, copy_file, mkdir, remove_file, symlink
 from easybuild.tools.run import run_shell_cmd
 
@@ -104,10 +108,74 @@ def build_step(self, topdir=None):
         # C compilers flags
         extra_makeopts += "CCFLAGS='$(HPL_DEFS) %s' " % os.getenv('CFLAGS')
 
+        comp_fam = self.toolchain.comp_family()
+        if comp_fam in [toolchain.INTELCOMP]:
+            # Explicitly disable optimization, since Intel compilers apply some default
+            # level not shown on the command line.
+            # This breaks the result comparison, resulting in all tests failing residual checks.
+            # See https://github.com/easybuilders/easybuild-easyconfigs/pull/23704#issuecomment-3202392904
+            extra_makeopts += 'CCNOOPT=\'$(HPL_DEFS) -O0\' '
+
         # set options and build
         self.cfg.update('buildopts', extra_makeopts)
         super().build_step()
 
+    def test_step(self):
+        """Test by running xhpl"""
+        srcdir = os.path.join(self.cfg['start_dir'], 'bin', 'UNKNOWN')
+        change_dir(srcdir)
+
+        pre_cmd = ""
+        post_cmd = ""
+
+        # xhpl needs atleast 4 processes to run the test suite
+        req_cpus = 4
+
+        mpi_fam = self.toolchain.mpi_family()
+        if mpi_fam is None:
+            self.report_test_failure("Toolchain does not include an MPI implementation, cannot run tests")
+
+        parallel = self.cfg.parallel
+        if not build_option('mpi_tests'):
+            self.log.info("MPI tests disabled from buildoption. Setting parallel to 1")
+            parallel = 1
+
+        if parallel < req_cpus:
+            self.log.info("Running tests with 1 oversubscribed process")
+
+            pin_str = ','.join(["0"] * req_cpus)
+            if mpi_fam in [toolchain.INTELMPI]:
+                pre_cmd = f"I_MPI_PIN_PROCESSOR_LIST=\"{pin_str}\" I_MPI_PIN=on "
+            elif mpi_fam in [toolchain.OPENMPI]:
+                post_cmd = f"--cpu-set {pin_str}"
+            elif mpi_fam in [toolchain.MPICH]:
+                post_cmd = f"-bind-to user:{pin_str}"
+            else:
+                self.report_test_failure("Don't know how to oversubscribe for `%s` MPI family" % mpi_fam)
+
+        cmd = self.toolchain.mpi_cmd_for(f'{post_cmd} ./xhpl', req_cpus)
+        cmd = f'{pre_cmd} {cmd}'
+        res = run_shell_cmd(cmd)
+        out = res.output
+
+        passed_rgx = re.compile(r'(\d+) tests completed and passed')
+        failed_rgx = re.compile(r'(\d+) tests completed and failed')
+
+        nfailed = 0
+        passed_mch = passed_rgx.search(out)
+        failed_mch = failed_rgx.search(out)
+        if passed_mch:
+            npassed = int(passed_mch.group(1))
+            self.log.info("%d tests passed residual checks in xhpl output" % npassed)
+        else:
+            self.report_test_failure("Could not find test results in output of xhpl")
+
+        if failed_mch:
+            nfailed = int(failed_mch.group(1))
+
+        if nfailed > 0:
+            self.report_test_failure("%d tests failed residual checks in xhpl output" % nfailed)
+
     def install_step(self):
         """
         Install by copying files to install dir
@@ -119,14 +187,15 @@ def install_step(self):
             srcfile = os.path.join(srcdir, filename)
             copy_file(srcfile, destdir)
 
-    def sanity_check_step(self):
+    def sanity_check_step(self, **kwargs):
         """
         Custom sanity check for HPL
         """
 
-        custom_paths = {
+        # Allow subclasses to set own custom paths
+        kwargs.setdefault('custom_paths', {
             'files': ["bin/xhpl"],
             'dirs': []
-        }
+        })
 
-        super().sanity_check_step(custom_paths)
+        super().sanity_check_step(**kwargs)