Skip to content

Experimenting with Cluster in a box #17966

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ryon-jensen wants to merge 3 commits into
base: master
Choose a base branch
from
+278 −186
Draft

Experimenting with Cluster in a box #17966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ecac279
SRE-3775 build: Introduce Cluster Box Stages
ryon-jensen Jun 9, 2026
d255482
SRE-3775 build: Introduce Cluster Box Stages
ryon-jensen Jun 12, 2026
829ffcc
Build-description: Targeting hdr-233
ryon-jensen Jun 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 70 additions & 50 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

// To use a test branch (i.e. PR) until it lands to master
// I.e. for testing library changes
//@Library(value='pipeline-lib@your_branch') _
@Library(value='pipeline-lib@ryon-jensen/speed_of_ci') _

/* groovylint-disable-next-line CompileStatic */
job_status_internal = [:]
Expand Down Expand Up @@ -346,6 +346,15 @@ pipeline {
booleanParam(name: 'CI_large_md_on_ssd_TEST',
defaultValue: true,
description: 'Run the Functional Hardware Large MD on SSD test stage')
booleanParam(name: 'CI_md_on_ssd_daos_test_TEST',
defaultValue: true,
description: 'Run the Functional Cluster Box daos_test Other test stage')
booleanParam(name: 'CI_md_on_ssd_ftest_TEST',
defaultValue: true,
description: 'Run the Functional Cluster Box FTest test stage')
booleanParam(name: 'CI_md_on_ssd_daos_test_rebuild_TEST',
defaultValue: true,
description: 'Run the Functional Cluster Box daos_test_rebuild test stage')
string(name: 'CI_UNIT_VM1_LABEL',
defaultValue: 'ci_vm1',
description: 'Label to use for 1 VM node unit and RPM tests')
Expand Down Expand Up @@ -376,6 +385,9 @@ pipeline {
string(name: 'FUNCTIONAL_HARDWARE_LARGE_LABEL',
defaultValue: 'ci_nvme9',
description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages')
string(name: 'FUNCTIONAL_CLUSTER_BOX_LABEL',
defaultValue: 'cluster_box',
description: 'Label to use for the Functional Cluster Box stages')
string(name: 'CI_STORAGE_PREP_LABEL',
defaultValue: '',
description: 'Label for cluster to do a DAOS Storage Preparation')
Expand All @@ -391,8 +403,10 @@ pipeline {
stage('Set Description') {
steps {
script {
if (params.CI_BUILD_DESCRIPTION) {
buildDescription params.CI_BUILD_DESCRIPTION
String description = params.CI_BUILD_DESCRIPTION ?:
cachedCommitPragma('Build-description', '')
if (description) {
buildDescription description
}
}
}
Expand Down Expand Up @@ -612,7 +626,7 @@ pipeline {
deps_build: false) +
' --build-arg DAOS_PACKAGES_BUILD=no ' +
' --build-arg DAOS_KEEP_SRC=yes ' +
" -t ${sanitized_JOB_NAME()}-leap15-gcc" +
" -t ${sanitized_JOB_NAME()}-leap15-gcc" +
" -t ${sanitized_JOB_NAME()}-leap15" +
' --build-arg POINT_RELEASE=.6' +
" --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}"
Expand Down Expand Up @@ -652,7 +666,7 @@ pipeline {
}
}
}
stage('Unit Tests') {
stage('Tests') {
when {
beforeAgent true
expression { !skipStage() }
Expand Down Expand Up @@ -811,16 +825,6 @@ pipeline {
}
}
} // stage('Unit Test bdev with memcheck')
}
}
stage('Test') {
when {
beforeAgent true
//expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !skipStage() }
// Above not working, always skipping functional VM tests.
expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) }
}
parallel {
stage('Functional on EL 8.8 with Valgrind') {
when {
beforeAgent true
Expand Down Expand Up @@ -1106,9 +1110,10 @@ pipeline {
expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false) && !skipStage() }
}
steps {
script {
parallel(
'Functional Hardware Medium': getFunctionalTestStage(
script {
Map hwStages = [:]

hwStages['Functional Hardware Medium'] = getFunctionalTestStage(
name: 'Functional Hardware Medium',
pragma_suffix: '-hw-medium',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL,
Expand All @@ -1119,9 +1124,10 @@ pipeline {
run_if_pr: false,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium MD on SSD': getFunctionalTestStage(
image_version: 'el9.7',
node_count: 5
)
hwStages['Functional Hardware Medium MD on SSD'] = getFunctionalTestStage(
name: 'Functional Hardware Medium MD on SSD',
pragma_suffix: '-hw-medium-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL,
Expand All @@ -1132,9 +1138,10 @@ pipeline {
run_if_pr: true,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium VMD': getFunctionalTestStage(
image_version: 'el9.7',
node_count: 5
)
hwStages['Functional Hardware Medium VMD'] = getFunctionalTestStage(
name: 'Functional Hardware Medium VMD',
pragma_suffix: '-hw-medium-vmd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VMD_LABEL,
Expand All @@ -1146,9 +1153,10 @@ pipeline {
run_if_pr: false,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium Verbs Provider': getFunctionalTestStage(
image_version: 'el9.7',
node_count: 5
)
hwStages['Functional Hardware Medium Verbs Provider'] = getFunctionalTestStage(
name: 'Functional Hardware Medium Verbs Provider',
pragma_suffix: '-hw-medium-verbs-provider',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL,
Expand All @@ -1161,8 +1169,8 @@ pipeline {
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium Verbs Provider MD on SSD': getFunctionalTestStage(
)
hwStages['Functional Hardware Medium Verbs Provider MD on SSD'] = getFunctionalTestStage(
name: 'Functional Hardware Medium Verbs Provider MD on SSD',
pragma_suffix: '-hw-medium-verbs-provider-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL,
Expand All @@ -1174,23 +1182,10 @@ pipeline {
run_if_pr: true,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium UCX Provider': getFunctionalTestStage(
name: 'Functional Hardware Medium UCX Provider',
pragma_suffix: '-hw-medium-ucx-provider',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_UCX_PROVIDER_LABEL,
next_version: next_version(),
stage_tags: 'hw,medium,provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto',
provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'),
run_if_pr: false,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Large': getFunctionalTestStage(
image_version: 'el9.7',
node_count: 5
)
hwStages['Functional Hardware Large'] = getFunctionalTestStage(
name: 'Functional Hardware Large',
pragma_suffix: '-hw-large',
label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL,
Expand All @@ -1202,8 +1197,8 @@ pipeline {
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Large MD on SSD': getFunctionalTestStage(
)
hwStages['Functional Hardware Large MD on SSD'] = getFunctionalTestStage(
name: 'Functional Hardware Large MD on SSD',
pragma_suffix: '-hw-large-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL,
Expand All @@ -1215,8 +1210,33 @@ pipeline {
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
)
)

List<Map> clusterBoxStages = [
[name: 'daos_test Rebuild', tag: 'stage_daos_test_rebuild', pragma_suffix: '-cb-md-on-ssd-rebuild'],
[name: 'daos_test Other', tag: 'stage_daos_test', pragma_suffix: '-cb-md-on-ssd-daos-test'],
[name: 'FTest', tag: 'stage_ftest', pragma_suffix: '-cb-md-on-ssd-ftest'],
]

clusterBoxStages.each { cfg ->
String stageKey = "Functional Cluster Box ${cfg.name}"
hwStages[stageKey] = getFunctionalTestStage(
name: stageKey,
pragma_suffix: cfg.pragma_suffix,
label: params.FUNCTIONAL_CLUSTER_BOX_LABEL,
next_version: next_version(),
stage_tags: "cb,medium,${cfg.tag}",
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
nvme: 'auto_md_on_ssd',
image_version: 'el9.7',
node_count: 5,
run_if_pr: true,
run_if_landing: false,
job_status: job_status_internal
)
}

parallel(hwStages)
}
}
} // stage('Test Hardware')
Expand Down
83 changes: 76 additions & 7 deletions ci/gen_commit_pragmas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
Generate default commit pragmas.
"""

import ast
import importlib.util
import os
import re
Expand Down Expand Up @@ -145,12 +146,75 @@
return config


def __get_test_tag(commit_pragma_mapping, paths, default='pr'):
def _strip_docstrings(tree):
"""Remove docstring nodes from an AST tree.

This modifies the tree in place, replacing docstring nodes with ``ast.Pass()``.

Args:
tree (ast.AST): the AST tree to modify

Returns:
ast.AST: the modified tree
"""
for node in ast.walk(tree):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Module)):
if (node.body
and isinstance(node.body[0], ast.Expr)
and isinstance(node.body[0].value, ast.Constant)
and isinstance(node.body[0].value.value, str)):
node.body[0] = ast.Pass()
return tree


def _has_code_changes(path, target):
"""Check if a Python file has changes beyond comments and docstrings.

Check warning on line 171 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

wrong-spelling-in-docstring, Wrong spelling of a word 'docstrings' in a docstring:

Uses AST comparison to determine if actual executable code was modified.
Comments are inherently ignored by the AST parser, and docstrings are

Check warning on line 174 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

wrong-spelling-in-docstring, Wrong spelling of a word 'docstrings' in a docstring:
explicitly stripped before comparison.

Args:
path (str): absolute path to the file
target (str): git commit to compare against

Returns:
bool: True if code changes exist, False if only comments/docstrings changed

Check warning on line 182 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

wrong-spelling-in-docstring, Wrong spelling of a word 'docstrings' in a docstring:
"""
git_root = git_root_dir()
rel_path = os.path.relpath(path, git_root)

# Get old content from git
result = subprocess.run(

Check warning on line 188 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

subprocess-run-check, 'subprocess.run' used without explicitly defining the value for 'check'.
['git', 'show', f'{target}:{rel_path}'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=git_root)
if result.returncode != 0:
return True # New file or not in git

old_content = result.stdout.decode()

# Get new content from disk
try:
with open(path) as f:
new_content = f.read()
except FileNotFoundError:
return True # Deleted file

try:
old_tree = _strip_docstrings(ast.parse(old_content))
new_tree = _strip_docstrings(ast.parse(new_content))
return ast.dump(old_tree) != ast.dump(new_tree)
except SyntaxError:
return True # Can't parse, assume code changed


def __get_test_tag(commit_pragma_mapping, paths, target=None, default='pr'):
"""Get the Test-tag pragma.

Args:
commit_pragma_mapping (dict): full commit pragma mapping config
paths (list): paths to get tags for
target (str): git commit to compare against for detecting code vs comment changes
default (str): default tag to use if a path does not have a test-tag config

Returns:
Expand All @@ -161,7 +225,7 @@

all_tags = set()

for path in paths:

Check warning on line 228 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

too-many-nested-blocks, Too many nested blocks (6/5)
for _pattern, config in commit_pragma_mapping.items():
if re.search(rf'{_pattern}', path):
test_tag_config = config.get('test-tag', default)
Expand All @@ -172,12 +236,17 @@
}
_handler = test_tag_config['handler']
if _handler == 'FtestTagMap':
# Special ftest handling
try:
all_tags.update(ftest_tag_map.minimal_tags(path))
except KeyError:
# Use default from config
# Special ftest handling - only use class-level tags if
# actual code was changed, not just comments or docstrings

Check warning on line 240 in ci/gen_commit_pragmas.py

View workflow job for this annotation

GitHub Actions / Pylint check 

wrong-spelling-in-comment, Wrong spelling of a word 'docstrings' in a comment:
# (e.g. avocado tag changes)
if target and not _has_code_changes(path, target):
all_tags.update(test_tag_config['tags'].split(' '))
else:
try:
all_tags.update(ftest_tag_map.minimal_tags(path))
except KeyError:
# Use default from config
all_tags.update(test_tag_config['tags'].split(' '))
elif _handler == 'direct':
# Use direct tags from config
all_tags.update(test_tag_config['tags'].split(' '))
Expand Down Expand Up @@ -251,7 +320,7 @@

commit_pragma_mapping = read_commit_pragma_mapping()

test_tag = __get_test_tag(commit_pragma_mapping, modified_files)
test_tag = __get_test_tag(commit_pragma_mapping, modified_files, target=target)
if test_tag:
pragmas['Test-tag'] = test_tag

Expand Down
4 changes: 2 additions & 2 deletions src/tests/ftest/control/daos_server_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ def test_daos_server_helper_format(self):
operations while daos_server is run as normal user.

:avocado: tags=all,pr,daily_regression
:avocado: tags=hw,medium
:avocado: tags=cb,medium
:avocado: tags=control,basic,daos_server_helper
:avocado: tags=DaosPrivHelperTest,test_daos_server_helper_format
:avocado: tags=DaosPrivHelperTest,test_daos_server_helper_format,stage_ftest
"""
# Verify that daos_server_helper has the correct permissions
# Get the result remotely with os.stat so the format is compatible with local code
Expand Down
8 changes: 4 additions & 4 deletions src/tests/ftest/control/dmg_telemetry_nvme.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
(C) Copyright 2021-2023 Intel Corporation.

Check failure on line 2 in src/tests/ftest/control/dmg_telemetry_nvme.py

View workflow job for this annotation

GitHub Actions / Copyright check 

Copyright out of date

SPDX-License-Identifier: BSD-2-Clause-Patent
"""
Expand Down Expand Up @@ -32,9 +32,9 @@
Verify the telemetry engine NVMe metrics.

:avocado: tags=all,pr,daily_regression
:avocado: tags=hw,medium
:avocado: tags=cb,medium
:avocado: tags=control,telemetry,nvme
:avocado: tags=TestWithTelemetryNvme,test_nvme_telemetry_metrics
:avocado: tags=TestWithTelemetryNvme,test_nvme_telemetry_metrics,stage_ftest
"""
metrics_data = self.telemetry.get_nvme_metrics()
self.display_nvme_test_metrics(metrics_data)
Expand Down Expand Up @@ -68,8 +68,8 @@
Verify the dmg telemetry list command.

:avocado: tags=all,pr,daily_regression
:avocado: tags=hw,medium
:avocado: tags=cb,medium
:avocado: tags=control,telemetry,nvme
:avocado: tags=TestWithTelemetryNvme,test_telemetry_list_nvme
:avocado: tags=TestWithTelemetryNvme,test_telemetry_list_nvme,stage_ftest
"""
self.verify_telemetry_list()
Loading
Loading