Main branch integration testing instructions

[forsyth2]

The main branch integration testing process involves many steps. #774 will automate these as much as possible, but until that is completed, here are some functions and instructions. These are relevant to #634, #802.

Helper functions

test_helper_functions.bash:

# Run `source test_helper_functions.bash` to load these functions into your shell.
# These functions are used in the integration tests for zppy.

# Set up environments #########################################################

build_test_env() {
  local repo="$1"
  # Usually the date in YYYYMMDD format followed by a try number, e.g. 20260327-try2:
  local unique_id_suffix="$2" 
  local env_name=""
  local create_cmd=""
  local activate_cmd=""

  if [[ -z "$repo" ]]; then
    echo "Usage: build_test_env <repo>"
    echo "Repos: e3sm_diags | mpas_analysis | livvkit | zppy-interfaces | zppy"
    return 1
  fi

  rm -rf build

  case "$repo" in
    e3sm_diags)
      conda clean --all -y
      env_name="test-diags-main-${unique_id_suffix}"
      create_cmd="conda env create -f conda-env/dev.yml -n $env_name"
      activate_cmd="conda activate $env_name"
      ;;

    mpas-analysis)
      conda clean --all -y
      env_name="test-mpas-analysis-develop-${unique_id_suffix}"
      create_cmd="conda create --name $env_name --file dev-spec.txt -y"
      activate_cmd="conda activate $env_name"
      ;;

    livvkit)
      activate_cmd="source /lcrc/group/e3sm/livvkit/software/load_e3sm_unified_1.12.1_lex.sh"
      ;;

    zppy-interfaces)
      conda clean --all -y
      env_name="test-zi-main-${unique_id_suffix}"
      create_cmd="conda env create -f conda/dev.yml -n $env_name"
      activate_cmd="conda activate $env_name"
      ;;

    zppy)
      conda clean --all -y
      env_name="test-zppy-main-${unique_id_suffix}"
      create_cmd="conda env create -f conda/dev.yml -n $env_name"
      activate_cmd="conda activate $env_name"
      ;;

    *)
      echo "Unknown repo: $repo"
      return 1
      ;;
  esac

  # Make sure conda activation works inside a function
  if command -v conda >/dev/null 2>&1; then
    eval "$(conda shell.bash hook)"
  fi

  if [[ -n "$create_cmd" ]]; then
    echo "Creating environment..."
    eval "$create_cmd" || return 1
  fi

  echo "Activating environment..."
  eval "$activate_cmd" || return 1

  echo "Installing package..."
  python -m pip install . || return 1
}


# Launch jobs ################################################################

launch_zppy_jobs()
{
    zppy -c tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg
    

    zppy -c tests/integration/generated/test_weekly_legacy_3.1.0_comprehensive_v3_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.1.0_bundles_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.1.0_comprehensive_v2_chrysalis.cfg
    
    zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg

    echo "Squeue current job count (subtract 1 to account for header row):"
    squeue -o "%8u %.7a %.4D %.9P %7i %.2t %.10r %.10M %.10l %j" --sort=P,-t,-p -u ac.forsyth2 | wc -l
}

launch_zppy_bundles_part2()
{
    zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.1.0_bundles_chrysalis.cfg
    zppy -c tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg

    echo "Squeue current job count (subtract 1 to account for header row):"
    squeue -o "%8u %.7a %.4D %.9P %7i %.2t %.10r %.10M %.10l %j" --sort=P,-t,-p -u ac.forsyth2 | wc -l
}

# Review status ###############################################################

review_status()
{
    local dir="$1"

    cd ${dir}
    echo "Checking ${dir}"
    # Print any lines that do not contain "OK" in the status files. 
    # If all jobs succeeded, this should print nothing.
    grep -v "OK" *status 
    cd - # Return to previous directory
}

review_bundles_dirs()
{
    local unique_id="$1"

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.1.0_bundles_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}
}

review_v2_dirs()
{
    local unique_id="$1"

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/${unique_id}/v2.LR.historical_0201/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.1.0_comprehensive_v2_output/${unique_id}/v2.LR.historical_0201/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/${unique_id}/v2.LR.historical_0201/post/scripts
    review_status ${dir}
}

review_v3_dirs()
{
    local unique_id="$1"

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.1.0_comprehensive_v3_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}

    dir=/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v3_output/${unique_id}/v3.LR.historical_0051/post/scripts
    review_status ${dir}
}

Instructions

Initial setup

lcrc_conda # Function to activate conda (defined in ~/.bashrc)
lcrc-quota # Check if we have enough space
source test_helper_functions.bash
ENV_UNIQUE_ID_SUFFIX="20260402_try1"
UNIQUE_ID=zppy_main_branch_test_${ENV_UNIQUE_ID_SUFFIX}

Set up environments

# Set up e3sm_diags env
cd ~/ez/e3sm_diags
git status # Check that there are no uncommitted changes
git checkout main
git fetch upstream main
git reset --hard upstream/main
git log --oneline | head -n 1 
# Check that this matches https://github.com/E3SM-Project/e3sm_diags/commits/main
build_test_env e3sm_diags ${ENV_UNIQUE_ID_SUFFIX}

# Set up mpas-analysis env
cd ~/ez/MPAS-Analysis/
git status # Check that there are no uncommitted changes
git checkout develop
git fetch upstream develop
git reset --hard upstream/develop
git log --oneline | head -n 1 
# Check that this matches https://github.com/MPAS-Dev/MPAS-Analysis/commits/develop/
build_test_env mpas-analysis ${ENV_UNIQUE_ID_SUFFIX}

# Set up LIVVkit env
build_test_env livvkit ${ENV_UNIQUE_ID_SUFFIX}

# Set up zppy-interfaces env
cd ~/ez/zppy-interfaces
git status # Check that there are no uncommitted changes
git checkout main
git fetch upstream main
git reset --hard upstream/main
git log --oneline | head -n 1 
# Check that this matches https://github.com/E3SM-Project/zppy-interfaces/commits/main
build_test_env zppy-interfaces ${ENV_UNIQUE_ID_SUFFIX}
pytest tests/unit/global_time_series/test_*.py
pytest tests/unit/pcmdi_diags/test_*.py

zppy run

# zppy itself #################################################################
cd ~/ez/zppy
git status # Check that there are no uncommitted changes
git checkout main
git fetch upstream main
git reset --hard upstream/main
git log --oneline | head -n 1 
# Check that this matches https://github.com/E3SM-Project/zppy/commits/main
build_test_env zppy ${ENV_UNIQUE_ID_SUFFIX}
pytest tests/test_*.py # 38 passed in 0.59s

Edit TEST_SPECIFICS in tests/integration/utils.py. Change all six environment_commands as needed. Be sure to update the unique_id. Example below:

TEST_SPECIFICS: Dict[str, Any] = {
    # These are custom environment_commands for specific tasks.
    # Never set these to "", because they will print the line
    # `environment_commands = ""` for the corresponding task,
    # thus overriding the value set higher up in the cfg.
    # That is, there will be no environment set.
    # (`environment_commands = ""` only redirects to Unified
    # if specified under the [default] task)
    "diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-diags-main-20260327",
    "mpas_analysis_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-mpas-analysis-develop-20260327",
    "global_time_series_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-zi-main-20260327",
    "livvkit_environment_commands": "source /lcrc/group/e3sm/livvkit/software/load_e3sm_unified_1.12.1_lex.sh",
    "pcmdi_diags_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-zi-main-20260327",
    # This is the environment setup for other tasks.
    # Leave as "" to use the latest Unified environment.
    "environment_commands": "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh",
    # For a complete test, run the set of latest cfgs and at least one set of legacy cfgs
    "cfgs_to_run": [
        "weekly_bundles",
        "weekly_comprehensive_v2",
        "weekly_comprehensive_v3",
        "weekly_legacy_3.1.0_bundles",
        "weekly_legacy_3.1.0_comprehensive_v2",
        "weekly_legacy_3.1.0_comprehensive_v3",
        "weekly_legacy_3.0.0_bundles",
        "weekly_legacy_3.0.0_comprehensive_v2",
        "weekly_legacy_3.0.0_comprehensive_v3",
    ],
    "tasks_to_run": [
        "e3sm_diags",
        "mpas_analysis",
        "global_time_series",
        "ilamb",
        "livvkit",
        "pcmdi_diags",
    ],
    "unique_id": "zppy_main_branch_test_20260327",
}

git diff # Check that diff looks as expected
python tests/integration/utils.py
# Check that unique_id and the environments are correct

squeue -o "%8u %.7a %.4D %.9P %7i %.2t %.10r %.10M %.10l %j" --sort=P,-t,-p -u ac.forsyth2
# Check that no other jobs are currently queued
launch_zppy_jobs
# Wait for jobs to finish

Bundles part 2

review_bundles_dirs ${UNIQUE_ID}

# Now, run bundles part 2 for the working cfgs.
cd ~/ez/zppy
git status # Check that branch and conda env are correct
launch_zppy_bundles_part2
# Wait for jobs to finish

Review finished runs

review_v2_dirs ${UNIQUE_ID}
review_v3_dirs ${UNIQUE_ID}
review_bundles_dirs ${UNIQUE_ID}

Python tests

cd ~/ez/zppy
git status # Check that branch and conda env are correct
ls tests/integration/test_*.py
pytest tests/integration/test_bash_generation.py 
pytest tests/integration/test_campaign.py
pytest tests/integration/test_defaults.py
pytest tests/integration/test_last_year.py
pytest tests/integration/test_bundles.py
salloc --nodes=1 --partition=debug --time=02:00:00 --account=e3sm
# We need to reactivate conda now that we're on a compute node
lcrc_conda # Activate conda
conda activate test-zppy-main-${ENV_UNIQUE_ID_SUFFIX}
pytest tests/integration/test_images.py
cat test_images_summary.md