|
| 1 | +#!/bin/bash |
| 2 | +# |
| 3 | +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 4 | +# SPDX-License-Identifier: MIT-0 |
| 5 | +# |
| 6 | +# Textfile collector: exports Amazon EFA hardware counters for Prometheus. |
| 7 | +# |
| 8 | +# Why this exists: |
| 9 | +# node_exporter's --collector.infiniband only reads the standard IB |
| 10 | +# counters in /sys/class/infiniband/<dev>/ports/<port>/counters/ |
| 11 | +# (port_rcv_data, port_xmit_data, ...). The EFA-specific counters that |
| 12 | +# matter for HPC/ML fabric health — RDMA read/write bytes, SRD |
| 13 | +# retransmits, work-request errors — live in a SEPARATE directory: |
| 14 | +# /sys/class/infiniband/<dev>/ports/<port>/hw_counters/ |
| 15 | +# which that collector does not read. Rather than ship a forked |
| 16 | +# node_exporter binary (as some solutions do), we surface the same |
| 17 | +# counters via the textfile collector. |
| 18 | +# |
| 19 | +# Metric naming mirrors the upstream awsome-distributed-training EFA node |
| 20 | +# exporter (node_amazonefa_<counter>), so dashboards/queries are portable |
| 21 | +# to/from that exporter. |
| 22 | +# |
| 23 | +# Output: /var/lib/prometheus/node-exporter/efa.prom |
| 24 | +# node_amazonefa_rdma_read_bytes{device="rdmap16s27",port="1"} 12345 |
| 25 | +# node_amazonefa_retrans_pkts{device="rdmap16s27",port="1"} 0 |
| 26 | +# ... |
| 27 | +# |
| 28 | +# The instance_id / instance_type labels are added by Prometheus at scrape |
| 29 | +# time (ec2_sd relabeling on the :9100 node_exporter target), so this |
| 30 | +# script only emits device/port labels — exactly like the other panels. |
| 31 | +# |
| 32 | +# Emits nothing (zero-length file) on instances without EFA, so the panels |
| 33 | +# simply stay empty there. Runs as root via a systemd timer. |
| 34 | +# |
| 35 | +set -euo pipefail |
| 36 | + |
| 37 | +# Paths are overridable for testing (EFA_SYSFS_PATH / EFA_OUTPUT). In |
| 38 | +# production they default to the real sysfs and node_exporter textfile dir. |
| 39 | +IB_PATH="${EFA_SYSFS_PATH:-/sys/class/infiniband}" |
| 40 | +TEXTFILE_DIR="/var/lib/prometheus/node-exporter" |
| 41 | +OUTPUT="${EFA_OUTPUT:-${TEXTFILE_DIR}/efa.prom}" |
| 42 | +TMP="${OUTPUT}.$$" |
| 43 | +mkdir -p "$(dirname "${OUTPUT}")" |
| 44 | + |
| 45 | +# No IB/EFA class at all → write an empty file and exit so any stale |
| 46 | +# metrics are cleared. |
| 47 | +if [[ ! -d "${IB_PATH}" ]]; then |
| 48 | + : > "${TMP}" |
| 49 | + mv -f "${TMP}" "${OUTPUT}" |
| 50 | + exit 0 |
| 51 | +fi |
| 52 | + |
| 53 | +# Decide whether a device is EFA. EFA devices are bound to the 'efa' |
| 54 | +# kernel driver; fall back to the presence of an EFA-specific counter |
| 55 | +# file (rdma_read_bytes) for robustness across kernels. |
| 56 | +is_efa_device() { |
| 57 | + local dev="$1" |
| 58 | + local drv |
| 59 | + drv=$(readlink -f "${IB_PATH}/${dev}/device/driver" 2>/dev/null || true) |
| 60 | + [[ "${drv##*/}" == "efa" ]] && return 0 |
| 61 | + # Fallback: any port exposing the EFA-specific hw_counter. |
| 62 | + compgen -G "${IB_PATH}/${dev}/ports/*/hw_counters/rdma_read_bytes" >/dev/null 2>&1 |
| 63 | +} |
| 64 | + |
| 65 | +# First pass: collect EFA (device, port, hw_counters-dir) tuples and the |
| 66 | +# union of counter file names, so we can group samples by metric name |
| 67 | +# (the Prometheus text format requires each metric's TYPE line to precede |
| 68 | +# its samples and all samples to be contiguous). |
| 69 | +# |
| 70 | +# Uses indexed arrays + a newline-delimited string (deduped via sort -u) |
| 71 | +# instead of associative arrays, so it runs on bash 3.2+ (macOS dev hosts, |
| 72 | +# minimal images) as well as the bash 4+ on the HPC AMIs. |
| 73 | +HWDIRS=() |
| 74 | +DEVS=() |
| 75 | +PORTS=() |
| 76 | +counter_names="" |
| 77 | + |
| 78 | +for devpath in "${IB_PATH}"/*; do |
| 79 | + [[ -d "${devpath}" ]] || continue |
| 80 | + dev="$(basename "${devpath}")" |
| 81 | + is_efa_device "${dev}" || continue |
| 82 | + for portpath in "${devpath}"/ports/*; do |
| 83 | + [[ -d "${portpath}/hw_counters" ]] || continue |
| 84 | + port="$(basename "${portpath}")" |
| 85 | + hwdir="${portpath}/hw_counters" |
| 86 | + HWDIRS+=("${hwdir}") |
| 87 | + DEVS+=("${dev}") |
| 88 | + PORTS+=("${port}") |
| 89 | + for cfile in "${hwdir}"/*; do |
| 90 | + [[ -f "${cfile}" ]] || continue |
| 91 | + counter_names="${counter_names}$(basename "${cfile}") |
| 92 | +" |
| 93 | + done |
| 94 | + done |
| 95 | +done |
| 96 | + |
| 97 | +# Sorted, de-duplicated list of counter names across all EFA ports. |
| 98 | +counters_sorted="$(printf '%s' "${counter_names}" | sort -u | sed '/^$/d')" |
| 99 | + |
| 100 | +# Emit grouped by metric name. |
| 101 | +{ |
| 102 | + while IFS= read -r counter; do |
| 103 | + [[ -n "${counter}" ]] || continue |
| 104 | + metric="node_amazonefa_${counter}" |
| 105 | + echo "# HELP ${metric} EFA hw_counter ${counter} from /sys/class/infiniband/*/ports/*/hw_counters" |
| 106 | + echo "# TYPE ${metric} counter" |
| 107 | + for i in "${!HWDIRS[@]}"; do |
| 108 | + cfile="${HWDIRS[$i]}/${counter}" |
| 109 | + [[ -r "${cfile}" ]] || continue |
| 110 | + value="$(cat "${cfile}" 2>/dev/null || echo "")" |
| 111 | + # Only emit clean non-negative integers; skip N/A or blank. |
| 112 | + [[ "${value}" =~ ^[0-9]+$ ]] || continue |
| 113 | + echo "${metric}{device=\"${DEVS[$i]}\",port=\"${PORTS[$i]}\"} ${value}" |
| 114 | + done |
| 115 | + done <<< "${counters_sorted}" |
| 116 | +} > "${TMP}" |
| 117 | + |
| 118 | +# Atomic rename so node_exporter never reads a partial file. |
| 119 | +mv -f "${TMP}" "${OUTPUT}" |
0 commit comments