Skip to content

Commit d4a11f8

Browse files
WalterKolczynski-NOAAWalterKolczynski-NOAA
authored
Update remaining bash scripts to be compliant with static bash code analysis checkers (NOAA-EMC#4329)
Updates all the remaining bash scripts to be complaint with the static bash code analysis checkers. With this update, all bash scripts with the global-workflow repo should be compliant with `shfmt` and `shellcheck`. The `setup_ecf_links.sh` scripts was the only one that required substantive refactoring. The existing one didn't work as expected. The refactoring both made the script compliant and restored the correct functionality. One section linking the awips g2 directory has been temporarily commented out as that directory has not yet been copied from v16 to develop. The manual documentation builder script was renamed to have a `.sh` extension. Now that the entire workflow is compliant, the GH action removes the temporary actions and switches to using just the reviewdog actions on the full repository.
1 parent 7f875e8 commit d4a11f8

34 files changed

Lines changed: 1965 additions & 1961 deletions

‎.github/scripts/build_docs.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@ cp -R "${DOCS_DIR}/build/html/." artifact/documentation
2222
# so it will be available in the artifacts
2323
warning_file="${DOCS_DIR}/build/warnings.log"
2424
if [[ -s ${warning_file} ]]; then
25-
cp -r "${DOCS_DIR}/build/warnings.log" artifact/doc_warnings.log
26-
cp artifact/doc_warnings.log artifact/documentation
27-
echo "Warnings were encountered while building documentation."
28-
echo "========== Begin warnings =========="
29-
cat artifact/doc_warnings.log
30-
echo "=========== End warnings ==========="
25+
cp -r "${DOCS_DIR}/build/warnings.log" artifact/doc_warnings.log
26+
cp artifact/doc_warnings.log artifact/documentation
27+
echo "Warnings were encountered while building documentation."
28+
echo "========== Begin warnings =========="
29+
cat artifact/doc_warnings.log
30+
echo "=========== End warnings ==========="
3131
fi

‎.github/workflows/bash_code_analysis.yaml‎

Lines changed: 28 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,70 @@
11
name: bash_code_analysis
22
on:
3-
# push:
4-
# branches:
5-
# - develop
3+
push:
4+
branches:
5+
- develop
66
pull_request:
77

88
workflow_dispatch:
99

10-
permissions:
11-
contents: read
12-
1310
jobs:
1411

1512
shfmt:
1613
runs-on: ubuntu-22.04
1714

1815
permissions:
1916
contents: read # for actions/checkout to fetch code
20-
pull-requests: read # to get PR metadata
17+
pull-requests: write # to get PR metadata
2118
security-events: write
19+
checks: write
2220

2321
steps:
24-
- name: Checkout code
22+
- name: checkout code
2523
uses: actions/checkout@v6
2624
with:
27-
fetch-depth: 0
28-
29-
# Switch to this action once whole repo is shfmt compliant
30-
# - name: shfmt scan
31-
# uses: reviewdog/action-shfmt@v1
32-
# with:
33-
# filter_mode: nofilter
34-
# level: warning
35-
# reporter: github-pr-review
25+
submodules: false
3626

37-
- name: Install shfmt
38-
uses: mfinelli/setup-shfmt@v4
3927
- name: shfmt scan
40-
run: shfmt -d dev/jobs dev/job_cards dev/scripts dev/ush gempak ush
41-
42-
shellcheck:
43-
runs-on: ubuntu-22.04
44-
45-
permissions:
46-
contents: read # for actions/checkout to fetch code
47-
pull-requests: read # to get PR metadata
48-
security-events: write
49-
50-
steps:
51-
- name: Checkout code
52-
uses: actions/checkout@v6
53-
with:
54-
submodules: false
55-
56-
- name: Shellcheck Scan
57-
uses: reviewdog/action-shellcheck@v1
28+
uses: reviewdog/action-shfmt@v1
5829
with:
30+
github_token: ${{ secrets.GITHUB_TOKEN }}
5931
filter_mode: nofilter
60-
level: warning
61-
reporter: github-pr-review
62-
path: |
63-
dev/jobs
64-
dev/job_cards
65-
dev/scripts
66-
dev/ush
67-
gempak
68-
ush
69-
pattern: |
70-
config.*
71-
J*
72-
*.env
73-
*.sh
74-
*.bash
32+
fail_level: any
33+
level: any
34+
reviewdog_flags: '-reporter=github-pr-review'
35+
shfmt_flags: ''
7536

76-
differential_shellcheck:
77-
# Run again on just changes until whole repo is compliant
37+
shellcheck:
7838
runs-on: ubuntu-22.04
7939

8040
permissions:
8141
contents: read # for actions/checkout to fetch code
82-
pull-requests: read # to get PR metadata
42+
pull-requests: write # to get PR metadata
8343
security-events: write
44+
checks: write
8445

8546
steps:
86-
- name: Checkout code
47+
- name: checkout code
8748
uses: actions/checkout@v6
8849
with:
8950
submodules: false
9051

91-
- name: Shellcheck Scan
52+
- name: shellcheck scan
9253
uses: reviewdog/action-shellcheck@v1
9354
with:
94-
filter_mode: added
95-
level: warning
55+
github_token: ${{ secrets.GITHUB_TOKEN }}
9656
reporter: github-pr-review
97-
check_all_files_with_shebangs: true
57+
filter_mode: nofilter
58+
fail_level: any
59+
level: any
9860
pattern: |
9961
config.*
100-
J*
62+
JG*
10163
*.env
10264
*.sh
10365
*.bash
66+
exclude: |
67+
*/.git/*
68+
*/*.yml
69+
*/*.yaml
70+
check_all_files_with_shebangs: true

‎dev/ci/scripts/driver_weekly.sh‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ export REPO_URL="ssh://git@ssh.github.com:443/NOAA-EMC/global-workflow.git"
2727
################################################################
2828
# Setup the relative paths to scripts and PS4 for better logging
2929
################################################################
30-
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
30+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." > /dev/null 2>&1 && pwd)"
3131
scriptname=$(basename "${BASH_SOURCE[0]}")
3232
echo "Begin ${scriptname} at $(date -u)" || true
3333
export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
@@ -38,14 +38,14 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
3838

3939
source "${ROOT_DIR}/ush/detect_machine.sh"
4040
case ${MACHINE_ID} in
41-
hera | orion | hercules | wcoss2 | gaea)
42-
echo "Running Automated Testing on ${MACHINE_ID}"
43-
source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
44-
;;
45-
*)
46-
echo "Unsupported platform. Exiting with error."
47-
exit 1
48-
;;
41+
hera | orion | hercules | wcoss2 | gaea)
42+
echo "Running Automated Testing on ${MACHINE_ID}"
43+
source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
44+
;;
45+
*)
46+
echo "Unsupported platform. Exiting with error."
47+
exit 1
48+
;;
4949
esac
5050

5151
######################################################
@@ -110,9 +110,8 @@ PULL_REQUEST_LABELS=("CI/CD" "CI-Orion-Ready" "CI-Hera-Ready")
110110
"${GH}" pr ready --undo
111111

112112
# Add labels to the pull request
113-
for label in "${PULL_REQUEST_LABELS[@]}"
114-
do
115-
"${GH}" pr edit --add-label "${label}"
113+
for label in "${PULL_REQUEST_LABELS[@]}"; do
114+
"${GH}" pr edit --add-label "${label}"
116115
done
117116
cd "${GFS_BASH_CI_ROOT}"
118117
rm -Rf "${develop_dir}"

‎dev/ci/scripts/run_check_ci.sh‎

Lines changed: 60 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ set -eu
77
# by Rocoto
88
#####################################################################################
99

10-
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
11-
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
12-
SYSTEM_BUILD_DIR=${3:-"global-workflow"} # Name of the system build directory, default is "global-workflow
10+
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
11+
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
12+
SYSTEM_BUILD_DIR=${3:-"global-workflow"} # Name of the system build directory, default is "global-workflow
1313

1414
# TEST_DIR contains 2 directories;
1515
# 1. HOMEgfs: clone of the global-workflow
@@ -34,81 +34,84 @@ source "${HOMEgfs}/dev/ush/gw_setup.sh"
3434

3535
# cd into the experiment directory
3636
echo "cd ${RUNTESTS}/EXPDIR/${pslot}"
37-
cd "${RUNTESTS}/EXPDIR/${pslot}" || (echo "FATAL ERROR: Unable to cd into '${RUNTESTS}/EXPDIR/${pslot}', ABORT!"; exit 1)
37+
cd "${RUNTESTS}/EXPDIR/${pslot}" || (
38+
echo "FATAL ERROR: Unable to cd into '${RUNTESTS}/EXPDIR/${pslot}', ABORT!"
39+
exit 1
40+
)
3841

3942
# Name of the Rocoto XML and database files
4043
xml="${pslot}.xml"
4144
db="${pslot}.db"
4245

4346
# Ensure the XML is present for the experiment
4447
if [[ ! -f "${xml}" ]]; then
45-
echo "FATAL ERROR: XML file ${xml} not found in '${pslot}', experiment ${pslot} failed, ABORT!"
46-
exit 1
48+
echo "FATAL ERROR: XML file ${xml} not found in '${pslot}', experiment ${pslot} failed, ABORT!"
49+
exit 1
4750
fi
4851

4952
# Launch experiment
5053
echo "Launch experiment with Rocoto."
5154
rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
5255
sleep 10
5356
if [[ ! -f "${db}" ]]; then
54-
echo "FATAL ERROR: Rocoto database file ${db} not found, experiment ${pslot} failed, ABORT!"
55-
exit 1
57+
echo "FATAL ERROR: Rocoto database file ${db} not found, experiment ${pslot} failed, ABORT!"
58+
exit 1
5659
fi
5760

5861
# Experiment launched
5962
rc=99
6063
set +e
6164
while true; do
6265

63-
echo "Run rocotorun."
64-
rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
65-
66-
# Wait before running rocotostat
67-
sleep 60
68-
69-
# Get job statistics
70-
echo "Gather Rocoto statistics"
71-
# shellcheck disable=SC2312 # We want to use the exit code of the command
72-
full_state=$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
73-
error_stat=$?
74-
75-
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
76-
declare "${state}"="$(echo "${full_state}" | grep "${state}" | cut -d: -f2)" || true
77-
done
78-
ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
79-
80-
echo -e "(${pslot} on ${MACHINE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
81-
82-
if [[ ${error_stat} -ne 0 ]]; then
83-
{
84-
echo "Experiment ${pslot} Terminated with ${FAIL} tasks failed and ${DEAD} dead at $(date)" || true
85-
echo "Experiment ${pslot} Terminated: *${ROCOTO_STATE}*"
86-
} | tee -a "${run_check_logfile}"
87-
if [[ "${DEAD}" -ne 0 ]]; then
88-
error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
89-
{
90-
echo "Error logs:"
91-
echo "${error_logs}"
92-
} | tee -a "${run_check_logfile}"
93-
rm -f "${RUNTESTS}/${pslot}_error.logs"
94-
for log in ${error_logs}; do
95-
echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/EXPDIR/${pslot}/${pslot}_error.logs"
96-
done
97-
fi
98-
rc=1
99-
break
100-
fi
101-
102-
if [[ "${ROCOTO_STATE}" == "DONE" ]]; then
103-
{
104-
echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles: *SUCCESS* at $(date)" || true
105-
} | tee -a "${run_check_logfile}"
106-
rc=0
107-
break
108-
fi
109-
110-
# Wait before running rocotorun again
111-
sleep 300
66+
echo "Run rocotorun."
67+
rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
68+
69+
# Wait before running rocotostat
70+
sleep 60
71+
72+
# Get job statistics
73+
echo "Gather Rocoto statistics"
74+
# shellcheck disable=SC2312 # We want to use the exit code of the command
75+
full_state=$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
76+
error_stat=$?
77+
78+
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
79+
declare "${state}"="$(echo "${full_state}" | grep "${state}" | cut -d: -f2)" || true
80+
done
81+
ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
82+
83+
echo -e "(${pslot} on ${MACHINE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
84+
85+
if [[ ${error_stat} -ne 0 ]]; then
86+
{
87+
echo "Experiment ${pslot} Terminated with ${FAIL} tasks failed and ${DEAD} dead at $(date)" || true
88+
echo "Experiment ${pslot} Terminated: *${ROCOTO_STATE}*"
89+
} | tee -a "${run_check_logfile}"
90+
if [[ "${DEAD}" -ne 0 ]]; then
91+
error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
92+
{
93+
echo "Error logs:"
94+
echo "${error_logs}"
95+
} | tee -a "${run_check_logfile}"
96+
rm -f "${RUNTESTS}/${pslot}_error.logs"
97+
for log in ${error_logs}; do
98+
echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/EXPDIR/${pslot}/${pslot}_error.logs"
99+
done
100+
fi
101+
rc=1
102+
break
103+
fi
104+
105+
if [[ "${ROCOTO_STATE}" == "DONE" ]]; then
106+
{
107+
echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles: *SUCCESS* at $(date)" || true
108+
} | tee -a "${run_check_logfile}"
109+
rc=0
110+
break
111+
fi
112+
113+
# Wait before running rocotorun again
114+
sleep 300
112115

113116
done
114117

0 commit comments

Comments
 (0)