Skip to content

Commit 6f04ed0

Browse files
committed
Merge branch 'jgfouca/build_on_compute' (PR #8493)
Update CIME submodule Update CIME submodule from 189fe3b7c3019b5f0e2564cd69b410e92426102f to 240989d6ae3d236894cda07993e0f64a20af1623 Fixes: 1) Remove cdash build description upload, it's wrong for all but latest build 2) Fix a bug in cime_bisect using a non-string in expect arg 3) Fix get_ts_synopsis catch-all triggered by bless trailer after PASS 4) Do NOT skip prereq check for first job Changes: 1) Remove Perl XML dependency (replacing with Lite) 2) Add ability to do builds on compute nodes! 3) For share-exe test suites, allow specification of which case to use 4) Add fused shared/model builds when batched builds is on 5) For batched builds, use the whole node 6) Better support for ninja [BFB]
2 parents b50b964 + 0b4cad6 commit 6f04ed0

11 files changed

Lines changed: 125 additions & 10 deletions

File tree

cime_config/machines/config_machines.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1868,6 +1868,7 @@
18681868
<BASELINE_ROOT>/sems-data-store/ACME/baselines/mappy/$COMPILER</BASELINE_ROOT>
18691869
<CCSM_CPRNC>/sems-data-store/ACME/mappy/cprnc/cprnc</CCSM_CPRNC>
18701870
<GMAKE_J>64</GMAKE_J>
1871+
<BATCHED_BUILD>FALSE</BATCHED_BUILD>
18711872
<TESTS>e3sm_developer</TESTS>
18721873
<BATCH_SYSTEM>slurm_single_node</BATCH_SYSTEM>
18731874
<SUPPORTED_BY>jgfouca at sandia dot gov</SUPPORTED_BY>

cime_config/machines/config_workflow.xml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,17 @@
2929
-->
3030
<workflow_jobs id="default">
3131
<!-- order matters, with no-batch jobs will be run in the order listed here -->
32+
<job name="case.build">
33+
<template>template.case.build</template>
34+
<!-- Not part of the default submit workflow; submitted explicitly by case.build
35+
when BATCHED_BUILD is TRUE on the machine. -->
36+
<prereq>False</prereq>
37+
<runtime_parameters>
38+
<task_count>1</task_count>
39+
<tasks_per_node>1</tasks_per_node>
40+
<walltime>02:00:00</walltime>
41+
</runtime_parameters>
42+
</job>
3243
<job name="case.run">
3344
<template>template.case.run</template>
3445
<prereq>$BUILD_COMPLETE and not $TEST</prereq>
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env python3
2+
{{ batchdirectives }}
3+
4+
"""
5+
Batch script template for building the case when BATCHED_BUILD is TRUE.
6+
This script is submitted as a batch job by case.build and should not be
7+
run directly.
8+
9+
DO NOT RUN THIS SCRIPT MANUALLY
10+
Use ./case.build from the case directory instead.
11+
"""
12+
13+
import os, sys
14+
os.chdir( '{{ caseroot }}')
15+
16+
_LIBDIR = os.path.join("{{ cimeroot }}", "CIME", "Tools")
17+
sys.path.append(_LIBDIR)
18+
19+
from standard_script_setup import *
20+
21+
import CIME.build as build
22+
from CIME.case import Case
23+
24+
logger = logging.getLogger(__name__)
25+
26+
###############################################################################
27+
def _main_func(description):
28+
###############################################################################
29+
# Expand any additional args injected via ARGS_FOR_SCRIPT by the submitter.
30+
sys.argv.extend(
31+
[] if "ARGS_FOR_SCRIPT" not in os.environ
32+
else os.environ["ARGS_FOR_SCRIPT"].split()
33+
)
34+
35+
import argparse
36+
parser = argparse.ArgumentParser(description=description, prog=".case.build")
37+
CIME.utils.setup_standard_logging_options(parser)
38+
39+
parser.add_argument("--caseroot", default=os.getcwd(),
40+
help="Case directory to build")
41+
parser.add_argument("--no-batch-build", action="store_true",
42+
help="Ignore, this is always True")
43+
parser.add_argument("--sharedlib-only", action="store_true",
44+
help="Only build shared libraries")
45+
parser.add_argument("--model-only", "-m", action="store_true",
46+
help="Assume shared libraries are already built")
47+
parser.add_argument("--separate-builds", action="store_true",
48+
help="Build each component separately")
49+
parser.add_argument("--ninja", action="store_true",
50+
help="Use ninja backend for CMake")
51+
parser.add_argument("--skip-provenance-check", action="store_true",
52+
help="Do not check and save build provenance")
53+
parser.add_argument("--build", nargs="+", dest="buildlist",
54+
help="Specific libraries/components to build")
55+
56+
args = CIME.utils.parse_args_and_handle_standard_logging_options(args=sys.argv, parser=parser)
57+
58+
if args.caseroot is not None:
59+
os.chdir(args.caseroot)
60+
caseroot = args.caseroot
61+
else:
62+
caseroot = os.getcwd()
63+
64+
with Case(caseroot, read_only=False, record=True) as case:
65+
success = build.case_build(
66+
caseroot,
67+
case=case,
68+
sharedlib_only=args.sharedlib_only,
69+
model_only=args.model_only,
70+
buildlist=args.buildlist,
71+
save_build_provenance=not args.skip_provenance_check,
72+
separate_builds=args.separate_builds,
73+
ninja=args.ninja,
74+
batched_build_active=True, # This is a batch script
75+
)
76+
77+
sys.exit(0 if success else 1)
78+
79+
80+
if __name__ == "__main__":
81+
_main_func(__doc__)

cime_config/machines/template.case.run

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ OR
3535
\033[1;32m# case.run SMS\033[0m
3636
> {0}
3737
""".format(os.path.basename(args[0])),
38-
description=description,
38+
description=description, prog=".case.run",
3939
formatter_class=argparse.ArgumentDefaultsHelpFormatter
4040
)
4141

cime_config/machines/template.case.test

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,9 @@ OR
3131
\033[1;32m# case.test SMS\033[0m
3232
> {0}
3333
""".format(os.path.basename(args[0])),
34-
35-
description=description,
36-
37-
formatter_class=argparse.ArgumentDefaultsHelpFormatter
38-
)
34+
description=description, prog=".case.test",
35+
formatter_class=argparse.ArgumentDefaultsHelpFormatter
36+
)
3937

4038
CIME.utils.setup_standard_logging_options(parser)
4139

cime_config/machines/template.post_run_io

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ OR
3838
\033[1;32m# case.post_run_io SMS\033[0m
3939
> {0}
4040
""".format(os.path.basename(args[0])),
41-
description=description,
41+
description=description, prog=".case.post_run_io",
4242
formatter_class=argparse.ArgumentDefaultsHelpFormatter
4343
)
4444

cime_config/machines/template.st_archive

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
2424
def parse_command_line(args, description):
2525
###############################################################################
2626

27-
parser = argparse.ArgumentParser(description=description,
27+
parser = argparse.ArgumentParser(description=description, prog=".st_archive",
2828
formatter_class=argparse.RawTextHelpFormatter)
2929

3030
CIME.utils.setup_standard_logging_options(parser)

cime_config/tests.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# suite_name : {
55
# "inherit" : (suite1, suite2, ...), # Optional. Suites to inherit tests from. Default is None. Tuple, list, or str.
66
# "time" : "HH:MM:SS", # Optional. Recommended upper-limit on test time.
7-
# "share" : True|False, # Optional. If True, all tests in this suite share a build. Default is False.
7+
# "share" : True|False|"testname", # Optional. If True, all tests share a build (first in suite order leads). If a test name string, that test's build is shared. Default is False.
88
# "tests" : (test1, test2, ...) # Optional. The list of tests for this suite. See above for format. Tuple, list, or str.
99
# }
1010

driver-mct/cime_config/config_component.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,6 +798,18 @@
798798
<desc>Number of processors for gmake</desc>
799799
</entry>
800800

801+
<entry id="BATCHED_BUILD">
802+
<type>logical</type>
803+
<valid_values>TRUE,FALSE</valid_values>
804+
<default_value>FALSE</default_value>
805+
<group>build_def</group>
806+
<file>env_build.xml</file>
807+
<desc>If TRUE, case.build submits the build as a batch job instead of
808+
building interactively on the login node. The machine must define a
809+
supported batch system (BATCH_SYSTEM != none). When BATCH_SYSTEM is
810+
none, CIME falls back to an interactive build with a warning.</desc>
811+
</entry>
812+
801813
<entry id="BUILD_COMPLETE">
802814
<type>logical</type>
803815
<valid_values>TRUE,FALSE</valid_values>

0 commit comments

Comments
 (0)