[processor/memorylimiter] Fix degenerate GC loop when exporter has problems (#15053)

#### Description

Fixes the degenerate force-GC loop when memory is held by live
references (e.g., exporter queues during a downstream outage). Forced GC
backs off exponentially when ineffective and resets on recovery; the cap
is exposed as `max_gc_interval_when_{soft,hard}_limited` (default `30s`,
`0` disables).

#### Link to tracking issue

Fixes #4981

#### Testing

Added regression test for #4981; existing memorylimiter tests pass.

#### Documentation

README updated; chloggen entry under `.chloggen/`.

---------

Signed-off-by: Kc Balusu <kcbalusu@meta.com>
Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com>
Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com>

Author: 3 people

.chloggen/fix-degenerate-gc-loop-4981.yaml

@@ -0,0 +1,17 @@
+change_type: bug_fix
+
+component: processor/memory_limiter
+
+note: Fix degenerate collector performance when exporter has problems causing permanent CPU-burning GC loop
+
+issues: [4981]
+
+subtext: |
+  Forced GC runs now use exponential backoff when deemed ineffective
+  (still above soft limit and less than 5% reclaimed) to avoid preventing
+  recovery by overloading CPU with excessive GC runs. The cap on the
+  backoff interval is exposed via `max_gc_interval_when_soft_limited` and
+  `max_gc_interval_when_hard_limited` (both default `30s`); set either to
+  `0` to disable backoff on that path.
+
+change_logs: [user]

internal/memorylimiter/config.go

@@ -13,6 +13,11 @@ import (
 var (
 	errCheckIntervalOutOfRange        = errors.New("'check_interval' must be greater than zero")
 	errInconsistentGCMinInterval      = errors.New("'min_gc_interval_when_soft_limited' should be larger than 'min_gc_interval_when_hard_limited'")
+	errInconsistentGCMaxInterval      = errors.New("'max_gc_interval_when_soft_limited' should be larger than or equal to 'max_gc_interval_when_hard_limited' (when both are set)")
+	errInconsistentGCMaxSoftInterval  = errors.New("'max_gc_interval_when_soft_limited' must be greater than or equal to 'min_gc_interval_when_soft_limited' (or 0 to disable)")
+	errInconsistentGCMaxHardInterval  = errors.New("'max_gc_interval_when_hard_limited' must be greater than or equal to 'min_gc_interval_when_hard_limited' (or 0 to disable)")
+	errNegativeGCMaxSoftInterval      = errors.New("'max_gc_interval_when_soft_limited' must be non-negative")
+	errNegativeGCMaxHardInterval      = errors.New("'max_gc_interval_when_hard_limited' must be non-negative")
 	errLimitOutOfRange                = errors.New("'limit_mib' or 'limit_percentage' must be greater than zero")
 	errSpikeLimitOutOfRange           = errors.New("'spike_limit_mib' must be smaller than 'limit_mib'")
 	errSpikeLimitPercentageOutOfRange = errors.New("'spike_limit_percentage' must be smaller than 'limit_percentage'")
@@ -37,6 +42,22 @@ type Config struct {
 	// GCs is a CPU-heavy operation and executing it too frequently may affect the recovery capabilities of the collector.
 	MinGCIntervalWhenHardLimited time.Duration `mapstructure:"min_gc_interval_when_hard_limited"`
 
+	// MaxGCIntervalWhenSoftLimited caps the exponential backoff between forced
+	// GC calls in soft-limited mode. When a forced GC fails to reclaim memory
+	// (e.g., held by live references in exporter queues during a downstream
+	// outage), the interval doubles starting from min_gc_interval_when_soft_limited
+	// (or 95% of check_interval, whichever is larger) up to this cap. It resets
+	// to zero whenever a GC is effective or memory drops on its own.
+	// Set to 0 to disable the exponential backoff on this path; the interval
+	// will not grow beyond the configured min.
+	MaxGCIntervalWhenSoftLimited time.Duration `mapstructure:"max_gc_interval_when_soft_limited"`
+
+	// MaxGCIntervalWhenHardLimited caps the exponential backoff between forced
+	// GC calls in hard-limited mode. Same semantics as
+	// MaxGCIntervalWhenSoftLimited but applies to the hard-limit path.
+	// Set to 0 to disable the exponential backoff on this path.
+	MaxGCIntervalWhenHardLimited time.Duration `mapstructure:"max_gc_interval_when_hard_limited"`
+
 	// MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be
 	// allocated by the process.
 	MemoryLimitMiB uint32 `mapstructure:"limit_mib"`
@@ -59,6 +80,8 @@ var _ component.Config = (*Config)(nil)
 func NewDefaultConfig() *Config {
 	return &Config{
 		MinGCIntervalWhenSoftLimited: 10 * time.Second,
+		MaxGCIntervalWhenSoftLimited: 30 * time.Second,
+		MaxGCIntervalWhenHardLimited: 30 * time.Second,
 	}
 }
 
@@ -70,6 +93,26 @@ func (cfg *Config) Validate() error {
 	if cfg.MinGCIntervalWhenSoftLimited < cfg.MinGCIntervalWhenHardLimited {
 		return errInconsistentGCMinInterval
 	}
+	if cfg.MaxGCIntervalWhenSoftLimited < 0 {
+		return errNegativeGCMaxSoftInterval
+	}
+	if cfg.MaxGCIntervalWhenHardLimited < 0 {
+		return errNegativeGCMaxHardInterval
+	}
+	if cfg.MaxGCIntervalWhenSoftLimited > 0 && cfg.MaxGCIntervalWhenSoftLimited < cfg.MinGCIntervalWhenSoftLimited {
+		return errInconsistentGCMaxSoftInterval
+	}
+	if cfg.MaxGCIntervalWhenHardLimited > 0 && cfg.MaxGCIntervalWhenHardLimited < cfg.MinGCIntervalWhenHardLimited {
+		return errInconsistentGCMaxHardInterval
+	}
+	// Mirror the MinSoft >= MinHard invariant for the max pair: when both
+	// caps are enabled, the soft cap should not be tighter than the hard cap
+	// (it would mean a smaller backoff ceiling for the less-urgent path,
+	// which is almost always a typo).
+	if cfg.MaxGCIntervalWhenSoftLimited > 0 && cfg.MaxGCIntervalWhenHardLimited > 0 &&
+		cfg.MaxGCIntervalWhenSoftLimited < cfg.MaxGCIntervalWhenHardLimited {
+		return errInconsistentGCMaxInterval
+	}
 	if cfg.MemoryLimitMiB == 0 && cfg.MemoryLimitPercentage == 0 {
 		return errLimitOutOfRange
 	}

internal/memorylimiter/config_test.go

@@ -95,6 +95,83 @@ func TestConfigValidate(t *testing.T) {
 			},
 			err: errInconsistentGCMinInterval,
 		},
+		{
+			name: "negative max soft interval",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MaxGCIntervalWhenSoftLimited: -1 * time.Second,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: errNegativeGCMaxSoftInterval,
+		},
+		{
+			name: "negative max hard interval",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MaxGCIntervalWhenHardLimited: -1 * time.Second,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: errNegativeGCMaxHardInterval,
+		},
+		{
+			name: "max soft less than min soft",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MinGCIntervalWhenSoftLimited: 10 * time.Second,
+				MaxGCIntervalWhenSoftLimited: 5 * time.Second,
+				MaxGCIntervalWhenHardLimited: 5 * time.Second,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: errInconsistentGCMaxSoftInterval,
+		},
+		{
+			name: "max hard less than min hard",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MinGCIntervalWhenHardLimited: 20 * time.Second,
+				MinGCIntervalWhenSoftLimited: 20 * time.Second,
+				MaxGCIntervalWhenHardLimited: 5 * time.Second,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: errInconsistentGCMaxHardInterval,
+		},
+		{
+			name: "max soft less than max hard",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MaxGCIntervalWhenSoftLimited: 5 * time.Second,
+				MaxGCIntervalWhenHardLimited: 30 * time.Second,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: errInconsistentGCMaxInterval,
+		},
+		{
+			name: "valid max gc intervals: both zero (disabled)",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MaxGCIntervalWhenSoftLimited: 0,
+				MaxGCIntervalWhenHardLimited: 0,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: nil,
+		},
+		{
+			name: "valid max gc intervals: only hard disabled",
+			cfg: &Config{
+				CheckInterval:                100 * time.Millisecond,
+				MaxGCIntervalWhenSoftLimited: 30 * time.Second,
+				MaxGCIntervalWhenHardLimited: 0,
+				MemoryLimitMiB:               5722,
+				MemorySpikeLimitMiB:          1907,
+			},
+			err: nil,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/memorylimiter/memorylimiter.go

@@ -21,6 +21,12 @@ import (
 
 const (
 	mibBytes = 1024 * 1024
+
+	// gcEffectivenessThreshold is the minimum percentage of memory that must be
+	// reclaimed by a forced GC for it to be considered effective. If GC reclaims
+	// less than this percentage, the GC interval is backed off to avoid burning
+	// CPU on fruitless GC cycles.
+	gcEffectivenessThreshold = 5
 )
 
 var (
@@ -46,13 +52,42 @@ type MemoryLimiter struct {
 
 	minGCIntervalWhenSoftLimited time.Duration
 	minGCIntervalWhenHardLimited time.Duration
+	maxGCIntervalWhenSoftLimited time.Duration
+	maxGCIntervalWhenHardLimited time.Duration
 	lastGCDone                   time.Time
 
 	// The functions to read the mem values and run GC are set as a reference to help with
 	// testing different values.
 	readMemStatsFn func(m *runtime.MemStats)
 	runGCFn        func()
 
+	// currentSoftGCInterval and currentHardGCInterval are per-path dynamic GC
+	// intervals that grow when forced GCs on that path fail to reclaim memory
+	// (held by live references in exporter queues during downstream outages).
+	// Each resets to zero whenever a GC on its path is effective or memory
+	// drops on its own, and doubles after each ineffective GC on its path up
+	// to the corresponding maxGCIntervalWhen(Hard|Soft)Limited cap.
+	//
+	// The floor for doubling is max(minGCIntervalWhen(Hard|Soft)Limited,
+	// memCheckWait*0.95). When the configured min is below memCheckWait the
+	// floor lands just under memCheckWait — small enough that the next-tick
+	// gate (time.Since(lastGCDone) > floor) reliably clears, since lastGCDone
+	// is stamped after the forced GC completes and the next ticker fire may
+	// land slightly under a full memCheckWait afterward (the GC's wall-clock
+	// cost and timer slop can eat into the interval). Using exactly
+	// memCheckWait would let the gate fail on those ticks and reduce forced
+	// GC to every other tick.
+	//
+	// State is kept per-path so that disabling backoff on one path (max=0)
+	// does not influence gating on the other path.
+	currentSoftGCInterval time.Duration
+	currentHardGCInterval time.Duration
+
+	// lastStats is the most recent runtime.MemStats observation. Comparing
+	// current Alloc against lastStats.Alloc is how checkLimitAndBackoff
+	// classifies a forced GC as effective when memory drops by at least 5%.
+	lastStats *runtime.MemStats
+
 	// Fields used for logging.
 	logger *zap.Logger
 
@@ -82,6 +117,9 @@ func NewMemoryLimiter(cfg *Config, logger *zap.Logger) (*MemoryLimiter, error) {
 		ticker:                       time.NewTicker(cfg.CheckInterval),
 		minGCIntervalWhenSoftLimited: cfg.MinGCIntervalWhenSoftLimited,
 		minGCIntervalWhenHardLimited: cfg.MinGCIntervalWhenHardLimited,
+		maxGCIntervalWhenSoftLimited: cfg.MaxGCIntervalWhenSoftLimited,
+		maxGCIntervalWhenHardLimited: cfg.MaxGCIntervalWhenHardLimited,
+		lastStats:                    &runtime.MemStats{},
 		lastGCDone:                   time.Now(),
 		readMemStatsFn:               ReadMemStatsFn,
 		runGCFn:                      runtime.GC,
@@ -170,14 +208,62 @@ func (ml *MemoryLimiter) doGCandReadMemStats() *runtime.MemStats {
 	return ms
 }
 
+// checkLimitAndBackoff returns whether memory is above the soft limit. When
+// didGC is true (called immediately after a forced GC), it also advances both
+// per-path backoff intervals in lockstep so the inactive path is not caught
+// flat-footed when memory pressure shifts between soft and hard. A GC is
+// effective if it drops memory below the soft limit or reclaims at least
+// gcEffectivenessThreshold%; an effective observation resets both intervals
+// to zero because memory really did come down (a global signal).
+func (ml *MemoryLimiter) checkLimitAndBackoff(ms *runtime.MemStats, didGC bool) (aboveSoftLimit bool) {
+	aboveSoftLimit = ml.usageChecker.aboveSoftLimit(ms)
+	gcWasEffective := !aboveSoftLimit ||
+		ms.Alloc <= ml.lastStats.Alloc*(100-gcEffectivenessThreshold)/100
+	ml.lastStats = ms
+	if gcWasEffective {
+		ml.currentSoftGCInterval = 0
+		ml.currentHardGCInterval = 0
+		return aboveSoftLimit
+	}
+	if !didGC {
+		return aboveSoftLimit
+	}
+	ml.currentSoftGCInterval = ml.nextBackoffInterval(ml.currentSoftGCInterval, ml.minGCIntervalWhenSoftLimited, ml.maxGCIntervalWhenSoftLimited)
+	ml.currentHardGCInterval = ml.nextBackoffInterval(ml.currentHardGCInterval, ml.minGCIntervalWhenHardLimited, ml.maxGCIntervalWhenHardLimited)
+	ml.logger.Warn("Forced GC did not reclaim enough memory. Will back off GC frequency to preserve CPU for recovery.",
+		zap.Duration("next_soft_gc_interval", ml.currentSoftGCInterval),
+		zap.Duration("next_hard_gc_interval", ml.currentHardGCInterval),
+		memstatToZapField(ms))
+	return aboveSoftLimit
+}
+
+// nextBackoffInterval returns the new backoff interval for one path after an
+// ineffective forced GC. When configMax is 0 the path's backoff is disabled
+// and the interval is pinned to configMin (so the path keeps GCing every
+// configMin, or every check interval if configMin is also 0). Otherwise the
+// interval starts at max(configMin, 0.95*memCheckWait) — see the comment on
+// currentSoftGCInterval for why 0.95 — doubles on each ineffective GC, and
+// is capped at max(configMax, configMin).
+func (ml *MemoryLimiter) nextBackoffInterval(current, configMin, configMax time.Duration) time.Duration {
+	if configMax == 0 {
+		return configMin
+	}
+	minInterval := max(configMin, time.Duration(float64(ml.memCheckWait)*0.95))
+	maxInterval := max(configMax, configMin)
+	return min(max(current, minInterval)*2, maxInterval)
+}
+
 // CheckMemLimits inspects current memory usage against threshold and toggles mustRefuse when threshold is exceeded
 func (ml *MemoryLimiter) CheckMemLimits() {
 	ms := ml.readMemStats()
 
 	ml.logger.Debug("Currently used memory.", memstatToZapField(ms))
 
-	// Check if we are below the soft limit.
-	aboveSoftLimit := ml.usageChecker.aboveSoftLimit(ms)
+	// Top-of-tick classification. didGC=false because no forced GC has run
+	// yet this tick. If memory dropped on its own (Go runtime GC, queue
+	// drained), this resets both per-path currentGCIntervals so we won't
+	// suppress the next forced GC unnecessarily.
+	aboveSoftLimit := ml.checkLimitAndBackoff(ms, false)
 	if !aboveSoftLimit {
 		if ml.mustRefuse.Load() {
 			// Was previously refusing but enough memory is available now, no need to limit.
@@ -189,22 +275,24 @@ func (ml *MemoryLimiter) CheckMemLimits() {
 	}
 
 	if ml.usageChecker.aboveHardLimit(ms) {
+		gateInterval := max(ml.minGCIntervalWhenHardLimited, ml.currentHardGCInterval)
 		// We are above hard limit, do a GC if it wasn't done recently and see if
 		// it brings memory usage below the soft limit.
-		if time.Since(ml.lastGCDone) > ml.minGCIntervalWhenHardLimited {
+		if time.Since(ml.lastGCDone) > gateInterval {
 			ml.logger.Warn("Memory usage is above hard limit. Forcing a GC.", memstatToZapField(ms))
 			ms = ml.doGCandReadMemStats()
 			// Check the limit again to see if GC helped.
-			aboveSoftLimit = ml.usageChecker.aboveSoftLimit(ms)
+			aboveSoftLimit = ml.checkLimitAndBackoff(ms, true)
 		}
 	} else {
+		gateInterval := max(ml.minGCIntervalWhenSoftLimited, ml.currentSoftGCInterval)
 		// We are above soft limit, do a GC if it wasn't done recently and see if
 		// it brings memory usage below the soft limit.
-		if time.Since(ml.lastGCDone) > ml.minGCIntervalWhenSoftLimited {
+		if time.Since(ml.lastGCDone) > gateInterval {
 			ml.logger.Info("Memory usage is above soft limit. Forcing a GC.", memstatToZapField(ms))
 			ms = ml.doGCandReadMemStats()
 			// Check the limit again to see if GC helped.
-			aboveSoftLimit = ml.usageChecker.aboveSoftLimit(ms)
+			aboveSoftLimit = ml.checkLimitAndBackoff(ms, true)
 		}
 	}