|
| 1 | +#!/bin/bash |
| 2 | +## |
| 3 | +## Copyright contributors to Besu. |
| 4 | +## |
| 5 | +## Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
| 6 | +## the License. You may obtain a copy of the License at |
| 7 | +## |
| 8 | +## http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +## |
| 10 | +## Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
| 11 | +## an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 12 | +## specific language governing permissions and limitations under the License. |
| 13 | +## |
| 14 | +## SPDX-License-Identifier: Apache-2.0 |
| 15 | +## |
| 16 | + |
| 17 | +REPORTS_DIR="$1" |
| 18 | +REPORT_STRIP_PREFIX="$2" |
| 19 | +REPORT_STRIP_SUFFIX="$3" |
| 20 | +SPLIT_COUNT=$4 |
| 21 | +SPLIT_INDEX=$5 |
| 22 | + |
| 23 | +# extract tests time from Junit XML reports |
| 24 | +find "$REPORTS_DIR" -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v 'concat(sum(//testcase/@time), \" \", //testsuite/testcase[1]/@classname)' '{}'; echo '{}' | sed \"s#${REPORT_STRIP_PREFIX}/\(.*\)/${REPORT_STRIP_SUFFIX}.*# \1#\"" > tmp/timing.tsv |
| 25 | + |
| 26 | +# Sort times in descending order |
| 27 | +IFS=$'\n' sorted=($(sort -nr tmp/timing.tsv)) |
| 28 | +unset IFS |
| 29 | + |
| 30 | +sums=() |
| 31 | +tests=() |
| 32 | + |
| 33 | +# Initialize sums |
| 34 | +for ((i=0; i<SPLIT_COUNT; i++)) |
| 35 | +do |
| 36 | + sums[$i]=0 |
| 37 | +done |
| 38 | + |
| 39 | +echo -n '' > tmp/processedTests.list |
| 40 | + |
| 41 | +# add tests to groups trying to balance the sum of execution time of each group |
| 42 | +for line in "${sorted[@]}"; do |
| 43 | + line_parts=( $line ) |
| 44 | + test_time=$( echo "${line_parts[0]} * 1000 / 1" | bc ) # convert to millis without decimals |
| 45 | + test_name=${line_parts[1]} |
| 46 | + module_dir=${line_parts[2]} |
| 47 | + test_with_module="$test_name $module_dir" |
| 48 | + |
| 49 | + # deduplication check to avoid executing a test multiple time |
| 50 | + if grep -F -q --line-regexp "$test_with_module" tmp/processedTests.list |
| 51 | + then |
| 52 | + continue |
| 53 | + fi |
| 54 | + |
| 55 | + # Does the test still exists? |
| 56 | + if grep -F -q --line-regexp "$test_with_module" tmp/currentTests.list |
| 57 | + then |
| 58 | + # Find index of min sum |
| 59 | + idx_min_sum=0 |
| 60 | + min_sum=${sums[0]} |
| 61 | + for ((i=0; i<SPLIT_COUNT; i++)) |
| 62 | + do |
| 63 | + if [[ ${sums[$i]} -lt $min_sum ]] |
| 64 | + then |
| 65 | + idx_min_sum=$i |
| 66 | + min_sum=${sums[$i]} |
| 67 | + fi |
| 68 | + done |
| 69 | + |
| 70 | + # Add the test to the min sum list |
| 71 | + min_sum_tests=${tests[$idx_min_sum]} |
| 72 | + tests[$idx_min_sum]="${min_sum_tests}${test_with_module}," |
| 73 | + |
| 74 | + # Update the sums |
| 75 | + ((sums[idx_min_sum]+=test_time)) |
| 76 | + |
| 77 | + echo "$test_with_module" >> tmp/processedTests.list |
| 78 | + fi |
| 79 | +done |
| 80 | + |
| 81 | +# Any new test? |
| 82 | +grep -F --line-regexp -v -f tmp/processedTests.list tmp/currentTests.list > tmp/newTests.list |
| 83 | +idx_new_test=0 |
| 84 | +while read -r new_test_with_module |
| 85 | +do |
| 86 | + idx_group=$(( idx_new_test % SPLIT_COUNT )) |
| 87 | + group=${tests[$idx_group]} |
| 88 | + tests[$idx_group]="${group}${new_test_with_module}," |
| 89 | + idx_new_test=$(( idx_new_test + 1 )) |
| 90 | +done < tmp/newTests.list |
| 91 | + |
| 92 | +# remove last comma |
| 93 | +for ((i=0; i<SPLIT_COUNT; i++)) |
| 94 | +do |
| 95 | + test_list=${tests[$i]%,} |
| 96 | + tests[$i]="$test_list" |
| 97 | +done |
| 98 | + |
| 99 | + |
| 100 | +# group tests by module |
| 101 | +module_list=( $( echo "${tests[$SPLIT_INDEX]}" | tr "," "\n" | awk '{print $2}' | sort -u ) ) |
| 102 | + |
| 103 | +declare -A group_by_module |
| 104 | +for module_dir in "${module_list[@]}" |
| 105 | +do |
| 106 | + group_by_module[$module_dir]="" |
| 107 | +done |
| 108 | + |
| 109 | +IFS="," test_list=( ${tests[$SPLIT_INDEX]} ) |
| 110 | +unset IFS |
| 111 | + |
| 112 | +for line in "${test_list[@]}" |
| 113 | +do |
| 114 | + line_parts=( $line ) |
| 115 | + test_name=${line_parts[0]} |
| 116 | + module_dir=${line_parts[1]} |
| 117 | + |
| 118 | + module_group=${group_by_module[$module_dir]} |
| 119 | + group_by_module[$module_dir]="$module_group$test_name " |
| 120 | +done |
| 121 | + |
| 122 | +# return the requests index, without quotes to drop the last trailing space |
| 123 | +for module_dir in "${module_list[@]}" |
| 124 | +do |
| 125 | + module_test_task=":${module_dir//\//:}:test" |
| 126 | + module_tests=$( echo "${group_by_module[$module_dir]% }" | sed -e 's/^\| / --tests /g' ) |
| 127 | + echo "$module_test_task $module_tests" |
| 128 | +done |
0 commit comments