Skip to content

Commit 2a503ce

Browse files
mattkjames7mattkjames7
committed
separate out verification script
1 parent 5307113 commit 2a503ce

2 files changed

Lines changed: 110 additions & 102 deletions

File tree

‎.github/workflows/test-remote-monitoring.yaml‎

Lines changed: 7 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ on:
2020
- "charts/memgraph/**"
2121
- "charts/memgraph-high-availability/**"
2222
- "scripts/remote-monitoring-stack.yaml"
23+
- "scripts/verify-remote-monitoring.sh"
2324
- ".github/workflows/test-remote-monitoring.yaml"
2425

2526
jobs:
@@ -104,57 +105,9 @@ jobs:
104105
105106
- name: Verify metrics and logs reach monitoring cluster
106107
run: |
107-
set -euo pipefail
108-
kubectl wait -n remote-monitoring --for=condition=available deployment/monitoring-gateway --timeout=180s
109-
110-
kubectl port-forward -n remote-monitoring svc/monitoring-gateway 18080:80 >/tmp/pf-gateway.log 2>&1 &
111-
PF_GATEWAY_PID=$!
112-
kubectl port-forward -n remote-monitoring svc/victoria-logs 19428:9428 >/tmp/pf-vlogs.log 2>&1 &
113-
PF_VLOGS_PID=$!
114-
trap 'kill $PF_GATEWAY_PID $PF_VLOGS_PID >/dev/null 2>&1 || true' EXIT
115-
sleep 5
116-
117-
GREEN='\033[0;32m'
118-
YELLOW='\033[1;33m'
119-
RED='\033[0;31m'
120-
BLUE='\033[0;34m'
121-
NC='\033[0m'
122-
123-
echo -e "${BLUE}Checking remote_write metrics ingestion...${NC}"
124-
for i in $(seq 1 40); do
125-
resp="$(curl -s -u ci-monitor:ci-monitor-pass 'http://127.0.0.1:18080/api/v1/query?query=count(up%7Bjob%3D%22memgraph-exporter%22%2Cservice_name%3D%22memgraph-ha%22%7D)')"
126-
val="$(python3 -c 'import json,sys; r=json.loads(sys.argv[1]).get("data",{}).get("result",[]); print("0" if not r else r[0]["value"][1])' "$resp" 2>/dev/null || echo 0)"
127-
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$val"
128-
then
129-
echo -e "${GREEN}Metrics are ingested (matching series: ${val}).${NC}"
130-
break
131-
fi
132-
if [[ "$i" -eq 40 ]]; then
133-
echo -e "${RED}Timed out waiting for remote metrics ingestion.${NC}"
134-
exit 1
135-
fi
136-
echo -e "${YELLOW}Metrics not ingested yet (attempt ${i}/40, value=${val}).${NC}"
137-
sleep 10
138-
done
139-
140-
echo -e "${BLUE}Checking Loki push traffic on VictoriaLogs...${NC}"
141-
for i in $(seq 1 40); do
142-
log_push_total="$(curl -s http://127.0.0.1:19428/metrics | awk '/\/insert\/loki\/api\/v1\/push/ { sum += ($NF+0) } END { print sum+0 }')"
143-
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$log_push_total"; then
144-
echo -e "${GREEN}Logs are ingested (Loki push metric sum: ${log_push_total}).${NC}"
145-
echo -e "${BLUE}Sample gateway access logs (Loki push path):${NC}"
146-
kubectl logs -n remote-monitoring deploy/monitoring-gateway --tail=50 | grep '/loki/api/v1/push' || true
147-
echo -e "${BLUE}Sample Memgraph vector sidecar logs:${NC}"
148-
kubectl logs -n default memgraph-data-0-0 -c memgraph-vector --tail=20 || true
149-
break
150-
fi
151-
if [[ "$i" -eq 40 ]]; then
152-
echo -e "${RED}Timed out waiting for log ingestion.${NC}"
153-
exit 1
154-
fi
155-
echo -e "${YELLOW}Logs not ingested yet (attempt ${i}/40, push sum=${log_push_total}).${NC}"
156-
sleep 10
157-
done
108+
scripts/verify-remote-monitoring.sh \
109+
--service-name memgraph-ha \
110+
--sample-pod memgraph-data-0-0
158111
159112
- name: Collect diagnostics on failure
160113
if: failure()
@@ -249,57 +202,9 @@ jobs:
249202
250203
- name: Verify metrics and logs reach monitoring cluster
251204
run: |
252-
set -euo pipefail
253-
kubectl wait -n remote-monitoring --for=condition=available deployment/monitoring-gateway --timeout=180s
254-
255-
kubectl port-forward -n remote-monitoring svc/monitoring-gateway 18080:80 >/tmp/pf-gateway.log 2>&1 &
256-
PF_GATEWAY_PID=$!
257-
kubectl port-forward -n remote-monitoring svc/victoria-logs 19428:9428 >/tmp/pf-vlogs.log 2>&1 &
258-
PF_VLOGS_PID=$!
259-
trap 'kill $PF_GATEWAY_PID $PF_VLOGS_PID >/dev/null 2>&1 || true' EXIT
260-
sleep 5
261-
262-
GREEN='\033[0;32m'
263-
YELLOW='\033[1;33m'
264-
RED='\033[0;31m'
265-
BLUE='\033[0;34m'
266-
NC='\033[0m'
267-
268-
echo -e "${BLUE}Checking remote_write metrics ingestion...${NC}"
269-
for i in $(seq 1 40); do
270-
resp="$(curl -s -u ci-monitor:ci-monitor-pass 'http://127.0.0.1:18080/api/v1/query?query=count(up%7Bjob%3D%22memgraph-exporter%22%2Cservice_name%3D%22memgraph-standalone%22%7D)')"
271-
val="$(python3 -c 'import json,sys; r=json.loads(sys.argv[1]).get("data",{}).get("result",[]); print("0" if not r else r[0]["value"][1])' "$resp" 2>/dev/null || echo 0)"
272-
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$val"
273-
then
274-
echo -e "${GREEN}Metrics are ingested (matching series: ${val}).${NC}"
275-
break
276-
fi
277-
if [[ "$i" -eq 40 ]]; then
278-
echo -e "${RED}Timed out waiting for remote metrics ingestion.${NC}"
279-
exit 1
280-
fi
281-
echo -e "${YELLOW}Metrics not ingested yet (attempt ${i}/40, value=${val}).${NC}"
282-
sleep 10
283-
done
284-
285-
echo -e "${BLUE}Checking Loki push traffic on VictoriaLogs...${NC}"
286-
for i in $(seq 1 40); do
287-
log_push_total="$(curl -s http://127.0.0.1:19428/metrics | awk '/\/insert\/loki\/api\/v1\/push/ { sum += ($NF+0) } END { print sum+0 }')"
288-
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$log_push_total"; then
289-
echo -e "${GREEN}Logs are ingested (Loki push metric sum: ${log_push_total}).${NC}"
290-
echo -e "${BLUE}Sample gateway access logs (Loki push path):${NC}"
291-
kubectl logs -n remote-monitoring deploy/monitoring-gateway --tail=50 | grep '/loki/api/v1/push' || true
292-
echo -e "${BLUE}Sample Memgraph vector sidecar logs:${NC}"
293-
kubectl logs -n default memgraph-db-0 -c memgraph-vector --tail=20 || true
294-
break
295-
fi
296-
if [[ "$i" -eq 40 ]]; then
297-
echo -e "${RED}Timed out waiting for log ingestion.${NC}"
298-
exit 1
299-
fi
300-
echo -e "${YELLOW}Logs not ingested yet (attempt ${i}/40, push sum=${log_push_total}).${NC}"
301-
sleep 10
302-
done
205+
scripts/verify-remote-monitoring.sh \
206+
--service-name memgraph-standalone \
207+
--sample-pod memgraph-db-0
303208
304209
- name: Collect diagnostics on failure
305210
if: failure()

‎scripts/verify-remote-monitoring.sh‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
SERVICE_NAME=""
5+
SAMPLE_POD=""
6+
SAMPLE_NAMESPACE="default"
7+
SAMPLE_CONTAINER="memgraph-vector"
8+
9+
while [[ $# -gt 0 ]]; do
10+
case "$1" in
11+
--service-name)
12+
SERVICE_NAME="$2"
13+
shift 2
14+
;;
15+
--sample-pod)
16+
SAMPLE_POD="$2"
17+
shift 2
18+
;;
19+
--sample-namespace)
20+
SAMPLE_NAMESPACE="$2"
21+
shift 2
22+
;;
23+
--sample-container)
24+
SAMPLE_CONTAINER="$2"
25+
shift 2
26+
;;
27+
-h|--help)
28+
cat <<'EOF'
29+
Usage: verify-remote-monitoring.sh --service-name <name> --sample-pod <pod> [options]
30+
31+
Verifies remote_write metrics and log push ingestion via the monitoring gateway.
32+
33+
Required:
34+
--service-name Label value used for service_name in the metrics query
35+
--sample-pod Pod name to print sample vector sidecar logs from
36+
37+
Optional:
38+
--sample-namespace Namespace for sample pod logs (default: default)
39+
--sample-container Container for sample pod logs (default: memgraph-vector)
40+
EOF
41+
exit 0
42+
;;
43+
*)
44+
echo "Unknown argument: $1" >&2
45+
exit 1
46+
;;
47+
esac
48+
done
49+
50+
if [[ -z "$SERVICE_NAME" || -z "$SAMPLE_POD" ]]; then
51+
echo "Error: --service-name and --sample-pod are required." >&2
52+
exit 1
53+
fi
54+
55+
GREEN='\033[0;32m'
56+
YELLOW='\033[1;33m'
57+
RED='\033[0;31m'
58+
BLUE='\033[0;34m'
59+
NC='\033[0m'
60+
61+
kubectl wait -n remote-monitoring --for=condition=available deployment/monitoring-gateway --timeout=180s
62+
63+
kubectl port-forward -n remote-monitoring svc/monitoring-gateway 18080:80 >/tmp/pf-gateway.log 2>&1 &
64+
PF_GATEWAY_PID=$!
65+
kubectl port-forward -n remote-monitoring svc/victoria-logs 19428:9428 >/tmp/pf-vlogs.log 2>&1 &
66+
PF_VLOGS_PID=$!
67+
trap 'kill "$PF_GATEWAY_PID" "$PF_VLOGS_PID" >/dev/null 2>&1 || true' EXIT
68+
sleep 5
69+
70+
echo -e "${BLUE}Checking remote_write metrics ingestion...${NC}"
71+
for i in $(seq 1 40); do
72+
resp="$(curl -s -u ci-monitor:ci-monitor-pass "http://127.0.0.1:18080/api/v1/query?query=count(up%7Bjob%3D%22memgraph-exporter%22%2Cservice_name%3D%22${SERVICE_NAME}%22%7D)")"
73+
val="$(python3 -c 'import json,sys; r=json.loads(sys.argv[1]).get("data",{}).get("result",[]); print("0" if not r else r[0]["value"][1])' "$resp" 2>/dev/null || echo 0)"
74+
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$val"; then
75+
echo -e "${GREEN}Metrics are ingested (matching series: ${val}).${NC}"
76+
break
77+
fi
78+
if [[ "$i" -eq 40 ]]; then
79+
echo -e "${RED}Timed out waiting for remote metrics ingestion.${NC}"
80+
exit 1
81+
fi
82+
echo -e "${YELLOW}Metrics not ingested yet (attempt ${i}/40, value=${val}).${NC}"
83+
sleep 10
84+
done
85+
86+
echo -e "${BLUE}Checking Loki push traffic on VictoriaLogs...${NC}"
87+
for i in $(seq 1 40); do
88+
log_push_total="$(curl -s http://127.0.0.1:19428/metrics | awk '/\/insert\/loki\/api\/v1\/push/ { sum += ($NF+0) } END { print sum+0 }')"
89+
if python3 -c 'import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)' "$log_push_total"; then
90+
echo -e "${GREEN}Logs are ingested (Loki push metric sum: ${log_push_total}).${NC}"
91+
echo -e "${BLUE}Sample gateway access logs (Loki push path):${NC}"
92+
kubectl logs -n remote-monitoring deploy/monitoring-gateway --tail=50 | grep '/loki/api/v1/push' || true
93+
echo -e "${BLUE}Sample Memgraph vector sidecar logs:${NC}"
94+
kubectl logs -n "$SAMPLE_NAMESPACE" "$SAMPLE_POD" -c "$SAMPLE_CONTAINER" --tail=20 || true
95+
break
96+
fi
97+
if [[ "$i" -eq 40 ]]; then
98+
echo -e "${RED}Timed out waiting for log ingestion.${NC}"
99+
exit 1
100+
fi
101+
echo -e "${YELLOW}Logs not ingested yet (attempt ${i}/40, push sum=${log_push_total}).${NC}"
102+
sleep 10
103+
done

0 commit comments

Comments
 (0)