-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhealth.go
More file actions
133 lines (119 loc) · 3.09 KB
/
Copy pathhealth.go
File metadata and controls
133 lines (119 loc) · 3.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
package browserpm
import (
"context"
"time"
)
// startHealthChecker launches a background goroutine that periodically
// checks every page in the pool using both system-level (IsClosed) and
// business-level (PageProvider.Check) checks. Unhealthy pages are
// replaced transparently.
func (p *PagePool) startHealthChecker() {
if p.config.HealthCheckInterval <= 0 {
return
}
p.wg.Add(1)
go func() {
defer p.wg.Done()
ticker := time.NewTicker(p.config.HealthCheckInterval)
defer ticker.Stop()
for {
select {
case <-p.ctx.Done():
return
case <-ticker.C:
p.runHealthCheck()
}
}
}()
}
func (p *PagePool) runHealthCheck() {
p.mu.RLock()
snapshot := make([]*poolPage, len(p.pages))
copy(snapshot, p.pages)
p.mu.RUnlock()
for _, pp := range snapshot {
if pp.getState() != pageIdle {
continue
}
if !p.isPageHealthy(pp) {
p.log.Warn("health check failed, replacing page", String("page_id", pp.id))
p.waitAndReplace(pp)
}
}
}
func (p *PagePool) isPageHealthy(pp *poolPage) bool {
if pp.page.IsClosed() {
return false
}
checkCtx, cancel := context.WithTimeout(p.ctx, 10*time.Second)
defer cancel()
return p.provider.Check(checkCtx, pp.page)
}
// waitAndReplace waits for active ops to drain (up to GracePeriod), then
// replaces the page.
func (p *PagePool) waitAndReplace(pp *poolPage) {
pp.setState(pageClosing)
deadline := time.Now().Add(p.config.GracePeriod)
for time.Now().Before(deadline) && pp.activeOps.Load() > 0 {
time.Sleep(100 * time.Millisecond)
}
p.replacePage(pp)
}
// startReaper launches a background goroutine that recycles pages whose
// TTL has expired. Pages first enter a grace period (no new ops assigned),
// then are closed and replaced if the pool is below MinPages.
func (p *PagePool) startReaper() {
if p.config.TTL <= 0 {
return
}
p.wg.Add(1)
go func() {
defer p.wg.Done()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-p.ctx.Done():
return
case <-ticker.C:
p.runReap()
}
}
}()
}
func (p *PagePool) runReap() {
now := time.Now()
p.mu.RLock()
snapshot := make([]*poolPage, len(p.pages))
copy(snapshot, p.pages)
p.mu.RUnlock()
for _, pp := range snapshot {
age := now.Sub(pp.createdAt)
state := pp.getState()
switch {
case state == pageClosed:
continue
case state == pageIdle && age > p.config.TTL-p.config.GracePeriod:
pp.setState(pageClosing)
p.log.Debug("page entering grace period", String("page_id", pp.id))
case state == pageClosing && age > p.config.TTL:
if pp.activeOps.Load() > 0 {
graceDeadline := pp.createdAt.Add(p.config.TTL + p.config.GracePeriod)
if now.Before(graceDeadline) {
continue
}
p.log.Warn("force-closing page with active ops", String("page_id", pp.id), Int64("active_ops", pp.activeOps.Load()))
}
p.log.Info("reaping expired page", String("page_id", pp.id))
p.mu.Lock()
p.removePageLocked(pp)
needMore := len(p.pages) < p.config.MinPages
p.mu.Unlock()
if needMore {
if err := p.addPage(); err != nil {
p.log.Error("failed to replenish page after reap", err)
}
}
}
}
}