Skip to content

Commit 1a2a1e1

Browse files
JeffreyDallasJeffreyDallas
committed
save
Signed-off-by: Jeffrey Tang <jeffrey@swirldslabs.com>
1 parent 10bcee8 commit 1a2a1e1

3 files changed

Lines changed: 177 additions & 2 deletions

File tree

‎.github/workflows/flow-gcs-test.yaml‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ jobs:
142142
run: task build:compile
143143

144144
- name: Run GCS Test Script for type ${{ matrix.storageType }}
145+
id: run-gcs-script
145146
if: ${{ steps.check.outputs.run == 'true' && !cancelled() && !failure() }}
146147
env:
147148
GCS_ACCESS_KEY: ${{ secrets.GCP_S3_ACCESS_KEY }}
@@ -153,13 +154,57 @@ jobs:
153154
GCP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.GCP_SERVICE_ACCOUNT_TOKEN }}
154155
run: |
155156
export PATH=~/.solo/bin:${PATH}
156-
.github/workflows/script/gcs_test.sh || RC=$? || true
157-
if [[ "${RC}" != "0" ]]; then
157+
RC=0
158+
set +e
159+
.github/workflows/script/gcs_test.sh
160+
RC=$?
161+
set -e
162+
if [[ "${RC}" -ne 0 ]]; then
158163
echo "GCS test failed with exit code ${RC}"
159164
npm run solo-test -- deployment diagnostics logs --deployment solo-e2e -q --dev || true
160165
exit ${RC}
161166
fi
162167
168+
- name: Debug Registry Pull Path (GCS Workflow)
169+
if: ${{ always() && steps.check.outputs.run == 'true' && !cancelled() && steps.run-gcs-script.outcome == 'failure' }}
170+
timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
171+
shell: bash
172+
run: |
173+
set +e
174+
export PATH=~/.solo/bin:${PATH}
175+
CLUSTER_NAME="solo-e2e"
176+
TEST_IMAGE="docker.io/library/maven:3-eclipse-temurin-25-alpine"
177+
MIRROR_IMAGE="hub.mirror.docker.lat.ope.eng.hashgraph.io/library/maven:3-eclipse-temurin-25-alpine"
178+
REGISTRY1_IMAGE="registry-1.docker.io/library/maven:3-eclipse-temurin-25-alpine"
179+
180+
echo "::group::kind cluster and node discovery"
181+
kind get clusters || true
182+
kind get nodes --name "${CLUSTER_NAME}" || true
183+
echo "::endgroup::"
184+
185+
NODES="$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null || true)"
186+
if [[ -z "${NODES}" ]]; then
187+
echo "No kind nodes found for cluster ${CLUSTER_NAME}; skipping registry debug."
188+
exit 0
189+
fi
190+
191+
while IFS= read -r NODE; do
192+
[[ -z "${NODE}" ]] && continue
193+
echo "::group::${NODE} containerd mirror config"
194+
docker exec "${NODE}" sh -c "grep -n 'registry.mirrors' -A20 /etc/containerd/config.toml" || true
195+
echo "::endgroup::"
196+
197+
echo "::group::${NODE} explicit ctr pulls"
198+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${TEST_IMAGE}" || true
199+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${MIRROR_IMAGE}" || true
200+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${REGISTRY1_IMAGE}" || true
201+
echo "::endgroup::"
202+
203+
echo "::group::${NODE} recent daemon logs (pull-related)"
204+
docker logs "${NODE}" 2>&1 | grep -E 'maven|hub\\.mirror|registry-1\\.docker\\.io|unexpected EOF|ErrImagePull|ImagePullBackOff' | tail -n 200 || true
205+
echo "::endgroup::"
206+
done <<< "${NODES}"
207+
163208
- name: Upload Logs to GitHub
164209
if: ${{ steps.check.outputs.run == 'true' && !cancelled() }}
165210
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0

‎.github/workflows/script/gcs_test.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,94 @@ check_service_health() {
3232
fi
3333
}
3434

35+
preload_block_node_images_for_kind() {
36+
local cluster_name="$1"
37+
local mirror_registry="hub.mirror.docker.lat.ope.eng.hashgraph.io"
38+
local max_attempts=3
39+
local node source_image image_prefix source_registry short_image dockerhub_qualified_image
40+
local pull_candidate pulled_image attempt
41+
local -a pull_candidates
42+
local kind_nodes
43+
local preload_failed=0
44+
local preload_images=(
45+
"docker.io/library/maven:3-eclipse-temurin-25-alpine"
46+
"docker.io/library/busybox:latest"
47+
)
48+
49+
if ! command -v docker >/dev/null 2>&1 || ! command -v kind >/dev/null 2>&1; then
50+
echo "Skipping block-node image preload: docker and/or kind not available."
51+
return 0
52+
fi
53+
54+
kind_nodes="$(kind get nodes --name "${cluster_name}" 2>/dev/null || true)"
55+
if [ -z "${kind_nodes}" ]; then
56+
echo "Warning: unable to resolve kind nodes for cluster ${cluster_name}; skipping block-node image preload."
57+
return 0
58+
fi
59+
60+
echo "Preloading block-node init images into kind cluster '${cluster_name}' via ${mirror_registry} with Docker Hub fallback"
61+
for source_image in "${preload_images[@]}"; do
62+
image_prefix="${source_image%%/*}"
63+
if [[ "${image_prefix}" == *.* || "${image_prefix}" == *:* || "${image_prefix}" == "localhost" ]]; then
64+
source_registry="${image_prefix}"
65+
short_image="${source_image#*/}"
66+
else
67+
source_registry="docker.io"
68+
short_image="${source_image}"
69+
fi
70+
71+
dockerhub_qualified_image="docker.io/${short_image}"
72+
pull_candidates=()
73+
if [[ "${source_registry}" == "docker.io" || "${source_registry}" == "registry-1.docker.io" ]]; then
74+
pull_candidates=(
75+
"${mirror_registry}/${short_image}"
76+
"${dockerhub_qualified_image}"
77+
"registry-1.docker.io/${short_image}"
78+
)
79+
else
80+
pull_candidates=("${source_image}")
81+
fi
82+
83+
while IFS= read -r node; do
84+
[ -z "${node}" ] && continue
85+
attempt=1
86+
while [ "${attempt}" -le "${max_attempts}" ]; do
87+
pulled_image=""
88+
for pull_candidate in "${pull_candidates[@]}"; do
89+
if docker exec "${node}" ctr --namespace=k8s.io images pull "${pull_candidate}" >/dev/null 2>&1; then
90+
pulled_image="${pull_candidate}"
91+
break
92+
fi
93+
done
94+
95+
if [ -n "${pulled_image}" ]; then
96+
docker exec "${node}" ctr --namespace=k8s.io images tag "${pulled_image}" "${dockerhub_qualified_image}" >/dev/null 2>&1 || true
97+
docker exec "${node}" ctr --namespace=k8s.io images tag "${pulled_image}" "${short_image}" >/dev/null 2>&1 || true
98+
docker exec "${node}" ctr --namespace=k8s.io images tag "${pulled_image}" "${source_image}" >/dev/null 2>&1 || true
99+
echo "Preloaded ${source_image} on node ${node} from ${pulled_image}"
100+
break
101+
fi
102+
103+
if [ "${attempt}" -eq "${max_attempts}" ]; then
104+
echo "ERROR: failed to preload ${source_image} on ${node} after ${max_attempts} attempts."
105+
preload_failed=1
106+
else
107+
echo "Retrying preload ${source_image} on ${node} (attempt ${attempt}/${max_attempts})..."
108+
sleep 3
109+
fi
110+
attempt=$((attempt + 1))
111+
done
112+
done <<< "${kind_nodes}"
113+
done
114+
115+
if [ "${preload_failed}" -ne 0 ]; then
116+
echo "Block-node image preload failed; aborting early to avoid long pod readiness timeout."
117+
return 1
118+
fi
119+
120+
return 0
121+
}
122+
35123
if [ -z "${STORAGE_TYPE}" ]; then
36124
storageType="minio_only"
37125
else
@@ -145,6 +233,8 @@ else
145233

146234
npm run solo-test -- keys consensus generate --gossip-keys --tls-keys -i node1 --deployment "${SOLO_DEPLOYMENT}"
147235

236+
preload_block_node_images_for_kind "${SOLO_CLUSTER_NAME}"
237+
148238
npm run solo-test -- block node add --deployment "${SOLO_DEPLOYMENT}"
149239

150240
npm run solo-test -- consensus network deploy --deployment "${SOLO_DEPLOYMENT}" -i node1 \

‎.github/workflows/zxc-e2e-test.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,46 @@ jobs:
480480
481481
exit $TEST_EXIT_CODE
482482
483+
- name: Debug Registry Pull Path
484+
if: ${{ inputs.runner != 'windows-latest' && !cancelled() && steps.run-e2e-tests.outcome == 'failure' }}
485+
timeout-minutes: ${{ fromJSON(env.STEP_TIMEOUT_MINUTES) }}
486+
shell: bash
487+
run: |
488+
set +e
489+
export PATH=~/.solo/bin:${PATH}
490+
CLUSTER_NAME="${{ inputs.cluster-name }}-c1"
491+
TEST_IMAGE="docker.io/library/maven:3-eclipse-temurin-25-alpine"
492+
MIRROR_IMAGE="hub.mirror.docker.lat.ope.eng.hashgraph.io/library/maven:3-eclipse-temurin-25-alpine"
493+
REGISTRY1_IMAGE="registry-1.docker.io/library/maven:3-eclipse-temurin-25-alpine"
494+
495+
echo "::group::kind cluster and node discovery"
496+
kind get clusters || true
497+
kind get nodes --name "${CLUSTER_NAME}" || true
498+
echo "::endgroup::"
499+
500+
NODES="$(kind get nodes --name "${CLUSTER_NAME}" 2>/dev/null || true)"
501+
if [[ -z "${NODES}" ]]; then
502+
echo "No kind nodes found for cluster ${CLUSTER_NAME}; skipping registry debug."
503+
exit 0
504+
fi
505+
506+
while IFS= read -r NODE; do
507+
[[ -z "${NODE}" ]] && continue
508+
echo "::group::${NODE} containerd mirror config"
509+
docker exec "${NODE}" sh -c "grep -n 'registry.mirrors' -A20 /etc/containerd/config.toml" || true
510+
echo "::endgroup::"
511+
512+
echo "::group::${NODE} explicit ctr pulls"
513+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${TEST_IMAGE}" || true
514+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${MIRROR_IMAGE}" || true
515+
docker exec "${NODE}" ctr --namespace=k8s.io images pull "${REGISTRY1_IMAGE}" || true
516+
echo "::endgroup::"
517+
518+
echo "::group::${NODE} recent daemon logs (pull-related)"
519+
docker logs "${NODE}" 2>&1 | grep -E 'maven|hub\\.mirror|registry-1\\.docker\\.io|unexpected EOF|ErrImagePull|ImagePullBackOff' | tail -n 200 || true
520+
echo "::endgroup::"
521+
done <<< "${NODES}"
522+
483523
- name: Log Metrics
484524
id: log-metrics
485525
if: ${{ !cancelled() }}

0 commit comments

Comments
 (0)