feat(oom): report memory limit and usage on OOM kill

Author: sisttri

bpf/dropwatch.c

@@ -205,8 +205,12 @@ int bpf_kfree_skb_prog(struct trace_event_raw_kfree_skb *ctx)
 
 	/* meta */
 	data->meta.ktime_ns = bpf_ktime_get_ns();
+	/* tgid_pid and comm in SoftIRQ Context do not accurately reflect
+	 * the process where the drop occurred, only for reference.
+	 */
 	data->meta.tgid_pid = bpf_get_current_pid_tgid();
 	bpf_get_current_comm(&data->meta.comm, sizeof(data->meta.comm));
+
 	data->meta.kfree_skb_addr = (u64)(unsigned long)ctx->location;
 	data->meta.queue_mapping = BPF_CORE_READ(skb, queue_mapping);
 	data->meta.drop_reason = SKB_DROP_REASON_UNSUPPORT;

bpf/oom.c

@@ -24,6 +24,8 @@ struct oom_info {
 	u32 victim_pid;
 	u64 trigger_memcg_css;
 	u64 victim_memcg_css;
+	u64 mem_limit_pages;
+	u64 mem_usage_pages;
 };
 
 SEC("kprobe/oom_kill_process")
@@ -50,6 +52,13 @@ int BPF_KPROBE(oom_kill_process, struct oom_control *oc, const char *message)
 	info.trigger_memcg_css =
 	    (u64)BPF_CORE_READ(trigger_task, cgroups, subsys[memory_cgrp_id]);
 
+	info.mem_limit_pages = BPF_CORE_READ(oc, totalpages);
+	struct mem_cgroup *memcg = BPF_CORE_READ(oc, memcg);
+	if (memcg) {
+		info.mem_usage_pages =
+		    (u64)BPF_CORE_READ(memcg, memory.usage.counter);
+	}
+
 	bpf_perf_event_output(ctx, &oom_perf_events, COMPAT_BPF_F_CURRENT_CPU,
 			      &info, sizeof(info));
 	return 0;

core/events/oom.go

@@ -17,6 +17,7 @@ package events
 import (
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -38,6 +39,8 @@ type perfEventData struct {
 	VictimPid          int32
 	TriggerMemcgCSS    uint64
 	VictimMemcgCSS     uint64
+	MemLimitPages      uint64
+	MemUsagePages      uint64
 }
 
 type OOMTracingData struct {
@@ -51,6 +54,8 @@ type OOMTracingData struct {
 	VictimContainerHostname  string `json:"victim_container_hostname"`
 	VictimPid                int32  `json:"victim_pid"`
 	VictimProcessName        string `json:"victim_process_name"`
+	CgroupMemoryLimit        uint64 `json:"cgroup_memory_limit"`
+	CgroupMemoryUsage        uint64 `json:"cgroup_memory_usage"`
 }
 
 type oomMetric struct {
@@ -121,6 +126,8 @@ func (c *oomCollector) Start(ctx context.Context) error {
 
 	b.WaitDetachByBreaker(childCtx, cancel)
 
+	pageSize := uint64(os.Getpagesize())
+
 	for {
 		select {
 		case <-childCtx.Done():
@@ -146,6 +153,8 @@ func (c *oomCollector) Start(ctx context.Context) error {
 				VictimPid:          data.VictimPid,
 				VictimProcessName:  bytesutil.ToStr(data.VictimProcessName[:]),
 				VictimContainerID:  cssContainers[data.VictimMemcgCSS],
+				CgroupMemoryLimit:  data.MemLimitPages * pageSize,
+				CgroupMemoryUsage:  data.MemUsagePages * pageSize,
 			}
 
 			// leave the hostname empty if this is not container.