Skip to content

Commit ab4ccfa

Browse files
committed
Refresh percentage memory limits
Problem: percentage memory limits are resolved once at startup, so in-place container resizes leave the memory limiter enforcing stale thresholds. Solution: refresh cgroup-backed percentage thresholds on each memory check while retaining the previous thresholds when a refresh fails. Fixed MiB limits remain unchanged. Validation: go test . in internal/memorylimiter; go test . in processor/memorylimiterprocessor Signed-off-by: ADITYA CHAUHAN <125276621+hooiv@users.noreply.github.com>
1 parent fa69e2c commit ab4ccfa

2 files changed

Lines changed: 64 additions & 0 deletions

File tree

internal/memorylimiter/memorylimiter.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,13 @@ var (
4343
type MemoryLimiter struct {
4444
usageChecker memUsageChecker
4545

46+
// Percentage-based limits are refreshed on each memory check so that
47+
// in-place container resizes are reflected without changing fixed limits.
48+
percentageBased bool
49+
memoryLimitPercentage uint64
50+
memorySpikePercentage uint64
51+
getMemoryFn func() (uint64, error)
52+
4653
memCheckWait time.Duration
4754

4855
// mustRefuse is used to indicate when data should be refused.
@@ -113,6 +120,10 @@ func NewMemoryLimiter(cfg *Config, logger *zap.Logger) (*MemoryLimiter, error) {
113120

114121
return &MemoryLimiter{
115122
usageChecker: *usageChecker,
123+
percentageBased: cfg.MemoryLimitMiB == 0,
124+
memoryLimitPercentage: uint64(cfg.MemoryLimitPercentage),
125+
memorySpikePercentage: uint64(cfg.MemorySpikePercentage),
126+
getMemoryFn: GetMemoryFn,
116127
memCheckWait: cfg.CheckInterval,
117128
ticker: time.NewTicker(cfg.CheckInterval),
118129
minGCIntervalWhenSoftLimited: cfg.MinGCIntervalWhenSoftLimited,
@@ -255,6 +266,7 @@ func (ml *MemoryLimiter) nextBackoffInterval(current, configMin, configMax time.
255266

256267
// CheckMemLimits inspects current memory usage against threshold and toggles mustRefuse when threshold is exceeded
257268
func (ml *MemoryLimiter) CheckMemLimits() {
269+
ml.refreshPercentageMemoryLimit()
258270
ms := ml.readMemStats()
259271

260272
ml.logger.Debug("Currently used memory.", memstatToZapField(ms))
@@ -309,6 +321,20 @@ func (ml *MemoryLimiter) CheckMemLimits() {
309321
ml.mustRefuse.Store(aboveSoftLimit)
310322
}
311323

324+
func (ml *MemoryLimiter) refreshPercentageMemoryLimit() {
325+
if !ml.percentageBased {
326+
return
327+
}
328+
329+
totalMemory, err := ml.getMemoryFn()
330+
if err != nil {
331+
ml.logger.Warn("Unable to refresh percentage memory limiter; retaining previous limits.", zap.Error(err))
332+
return
333+
}
334+
335+
ml.usageChecker = *newPercentageMemUsageChecker(totalMemory, ml.memoryLimitPercentage, ml.memorySpikePercentage)
336+
}
337+
312338
type memUsageChecker struct {
313339
memAllocLimit uint64
314340
memSpikeLimit uint64

internal/memorylimiter/memorylimiter_test.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package memorylimiter
55

66
import (
77
"context"
8+
"errors"
89
"runtime"
910
"testing"
1011
"time"
@@ -107,6 +108,43 @@ func TestGetDecision(t *testing.T) {
107108
})
108109
}
109110

111+
func TestRefreshPercentageMemoryLimit(t *testing.T) {
112+
var totalMemory uint64 = 100 * mibBytes
113+
getMemory := func() (uint64, error) {
114+
return totalMemory, nil
115+
}
116+
originalGetMemory := GetMemoryFn
117+
GetMemoryFn = getMemory
118+
t.Cleanup(func() { GetMemoryFn = originalGetMemory })
119+
120+
ml, err := NewMemoryLimiter(&Config{
121+
CheckInterval: time.Minute,
122+
MemoryLimitPercentage: 80,
123+
MemorySpikePercentage: 20,
124+
}, zap.NewNop())
125+
require.NoError(t, err)
126+
assert.Equal(t, uint64(80*mibBytes), ml.usageChecker.memAllocLimit)
127+
assert.Equal(t, uint64(20*mibBytes), ml.usageChecker.memSpikeLimit)
128+
129+
totalMemory = 200 * mibBytes
130+
ml.refreshPercentageMemoryLimit()
131+
assert.Equal(t, uint64(160*mibBytes), ml.usageChecker.memAllocLimit)
132+
assert.Equal(t, uint64(40*mibBytes), ml.usageChecker.memSpikeLimit)
133+
}
134+
135+
func TestRefreshPercentageMemoryLimitRetainsPreviousLimitOnError(t *testing.T) {
136+
ml, err := NewMemoryLimiter(&Config{
137+
CheckInterval: time.Minute,
138+
MemoryLimitPercentage: 80,
139+
MemorySpikePercentage: 20,
140+
}, zap.NewNop())
141+
require.NoError(t, err)
142+
previous := ml.usageChecker
143+
ml.getMemoryFn = func() (uint64, error) { return 0, errors.New("cgroup unavailable") }
144+
ml.refreshPercentageMemoryLimit()
145+
assert.Equal(t, previous, ml.usageChecker)
146+
}
147+
110148
func TestRefuseDecision(t *testing.T) {
111149
decision1000Limit30Spike30 := newPercentageMemUsageChecker(1000, 60, 30)
112150
decision1000Limit60Spike50 := newPercentageMemUsageChecker(1000, 60, 50)

0 commit comments

Comments
 (0)