Skip to content

Commit 478a7d8

Browse files
docmerlindocmerlin
authored
Feat: handle Delete messages in join (#2562)
* feat: handle delete messages in joinNode * tests: add tests for join delete messages
1 parent fef0d30 commit 478a7d8

12 files changed

Lines changed: 161 additions & 19 deletions

File tree

‎barrier.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ func (n *idleBarrier) emitBarrier() error {
203203
return err
204204
}
205205
if n.del {
206-
return n.in.Collect(edge.NewDeleteGroupMessage(n.group.ID))
206+
return n.in.Collect(edge.NewDeleteGroupMessage(&n.group))
207207
}
208208
return nil
209209
}
@@ -318,7 +318,7 @@ func (n *periodicBarrier) emitBarrier() error {
318318
}
319319
if n.del {
320320
// Send DeleteGroupMessage into self
321-
return n.in.Collect(edge.NewDeleteGroupMessage(n.group.ID))
321+
return n.in.Collect(edge.NewDeleteGroupMessage(&n.group))
322322
}
323323
return nil
324324
}

‎edge/consumer.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ type MultiReceiver interface {
9999
BufferedBatch(src int, batch BufferedBatchMessage) error
100100
Point(src int, p PointMessage) error
101101
Barrier(src int, b BarrierMessage) error
102+
Delete(src int, d DeleteGroupMessage) error
102103
Finish() error
103104
}
104105

@@ -174,6 +175,10 @@ LOOP:
174175
if err := c.r.Barrier(m.Src, msg); err != nil {
175176
return err
176177
}
178+
case DeleteGroupMessage:
179+
if err := c.r.Delete(m.Src, msg); err != nil {
180+
return err
181+
}
177182
}
178183
}
179184
}

‎edge/messages.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -927,16 +927,16 @@ func (b *barrierMessage) Time() time.Time {
927927

928928
type DeleteGroupMessage interface {
929929
Message
930-
GroupIDGetter
930+
GroupInfoer
931931
}
932932

933933
type deleteGroupMessage struct {
934-
groupID models.GroupID
934+
info *GroupInfo
935935
}
936936

937-
func NewDeleteGroupMessage(id models.GroupID) DeleteGroupMessage {
937+
func NewDeleteGroupMessage(info *GroupInfo) *deleteGroupMessage {
938938
return &deleteGroupMessage{
939-
groupID: id,
939+
info: info,
940940
}
941941
}
942942

@@ -945,5 +945,9 @@ func (d *deleteGroupMessage) Type() MessageType {
945945
}
946946

947947
func (d *deleteGroupMessage) GroupID() models.GroupID {
948-
return d.groupID
948+
return d.info.ID
949+
}
950+
951+
func (d *deleteGroupMessage) GroupInfo() *GroupInfo {
952+
return d.info
949953
}

‎group_by.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,16 @@ func (n *GroupByNode) Barrier(b edge.BarrierMessage) error {
128128
}
129129
return edge.Forward(n.outs, b)
130130
}
131+
131132
func (n *GroupByNode) DeleteGroup(d edge.DeleteGroupMessage) error {
133+
n.timer.Start()
134+
n.mu.Lock()
135+
delete(n.groups, d.GroupID())
136+
n.mu.Unlock()
137+
n.timer.Stop()
132138
return edge.Forward(n.outs, d)
133139
}
140+
134141
func (n *GroupByNode) Done() {}
135142

136143
// emit sends all groups before time t to children nodes.

‎integrations/streamer_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4712,6 +4712,61 @@ errorCounts
47124712
testStreamerWithOutput(t, "TestStream_Join", script, 13*time.Second, er, true, nil)
47134713
}
47144714

4715+
func TestStream_Delete_Join(t *testing.T) {
4716+
var script = `
4717+
var errorCounts = stream
4718+
|from()
4719+
.measurement('cpu')
4720+
.groupBy('host')
4721+
|window()
4722+
.period(10s)
4723+
.every(10s)
4724+
.align()
4725+
4726+
|sum('value')
4727+
|barrier()
4728+
.idle(1s)
4729+
.delete(TRUE)
4730+
4731+
var viewCounts = stream
4732+
|from()
4733+
.measurement('views')
4734+
.groupBy('host')
4735+
|window()
4736+
.period(10s)
4737+
.every(10s)
4738+
.align()
4739+
|sum('value')
4740+
|barrier()
4741+
.idle(1s)
4742+
.delete(TRUE)
4743+
4744+
errorCounts
4745+
|join(viewCounts)
4746+
.as('errors', 'views')
4747+
.streamName('error_view')
4748+
.tolerance(2s)
4749+
.deleteAll(TRUE)
4750+
|eval(lambda: "errors.sum" / "views.sum")
4751+
.as('error_percent')
4752+
.keep()
4753+
|httpOut('TestStream_Delete_Join')
4754+
`
4755+
er := models.Result{
4756+
Series: models.Rows{
4757+
{
4758+
Name: "error_view",
4759+
Tags: map[string]string{"host": "serverA"},
4760+
Columns: []string{"time", "error_percent", "errors.sum", "views.sum"},
4761+
Values: [][]interface{}{
4762+
{time.Date(1971, 1, 1, 0, 0, 10, 0, time.UTC), 1.0, 18.0, 18.0},
4763+
},
4764+
},
4765+
},
4766+
}
4767+
testStreamerWithOutput(t, "TestStream_Delete_Join", script, 30*time.Second, er, true, nil)
4768+
}
4769+
47154770
func TestStream_Join_Delimiter(t *testing.T) {
47164771

47174772
var script = `

‎integrations/testdata/TestBatch_Join.1.brpl‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1-
{"name":"cpu_usage_idle","points":[{"fields":{"mean":93.49999999999409},"time":"2015-10-30T17:14:12Z"},{"fields":{"mean":91.44444444443974},"time":"2015-10-30T17:14:14Z"},{"fields":{"mean":93.44897959187637},"time":"2015-10-30T17:14:16Z"},{"fields":{"mean":95.99999999995998},"time":"2015-10-30T17:14:18Z"},{"fields":{"mean":97.00970097012197},"time":"2015-10-30T17:14:20Z"}]}
2-
{"name":"cpu_usage_idle","points":[{"fields":{"mean":95.98484848485191},"time":"2015-10-30T17:14:22Z"},{"fields":{"mean":92.098039215696},"time":"2015-10-30T17:14:24Z"},{"fields":{"mean":92.99999999998363},"time":"2015-10-30T17:14:26Z"},{"fields":{"mean":86.54015887023496},"time":"2015-10-30T17:14:28Z"},{"fields":{"mean":95.48979591840603},"time":"2015-10-30T17:14:30Z"}]}
3-
{"name":"cpu_usage_idle","points":[{"fields":{"mean":96.49999999996908},"time":"2015-10-30T17:14:32Z"},{"fields":{"mean":93.46464646468584},"time":"2015-10-30T17:14:34Z"},{"fields":{"mean":95.00950095007724},"time":"2015-10-30T17:14:36Z"},{"fields":{"mean":92.99999999998636},"time":"2015-10-30T17:14:38Z"},{"fields":{"mean":90.99999999998545},"time":"2015-10-30T17:14:40Z"}]}
1+
{"name":"cpu_usage_idle","points":[
2+
{"fields":{"mean":93.49999999999409},"time":"2015-10-30T17:14:12Z"},
3+
{"fields":{"mean":91.44444444443974},"time":"2015-10-30T17:14:14Z"},
4+
{"fields":{"mean":93.44897959187637},"time":"2015-10-30T17:14:16Z"},
5+
{"fields":{"mean":95.99999999995998},"time":"2015-10-30T17:14:18Z"},
6+
{"fields":{"mean":97.00970097012197},"time":"2015-10-30T17:14:20Z"}]}
7+
{"name":"cpu_usage_idle","points":[
8+
{"fields":{"mean":95.98484848485191},"time":"2015-10-30T17:14:22Z"},
9+
{"fields":{"mean":92.098039215696},"time":"2015-10-30T17:14:24Z"},
10+
{"fields":{"mean":92.99999999998363},"time":"2015-10-30T17:14:26Z"},
11+
{"fields":{"mean":86.54015887023496},"time":"2015-10-30T17:14:28Z"},
12+
{"fields":{"mean":95.48979591840603},"time":"2015-10-30T17:14:30Z"}]}
13+
{"name":"cpu_usage_idle","points":[
14+
{"fields":{"mean":96.49999999996908},"time":"2015-10-30T17:14:32Z"},
15+
{"fields":{"mean":93.46464646468584},"time":"2015-10-30T17:14:34Z"},
16+
{"fields":{"mean":95.00950095007724},"time":"2015-10-30T17:14:36Z"},
17+
{"fields":{"mean":92.99999999998636},"time":"2015-10-30T17:14:38Z"},
18+
{"fields":{"mean":90.99999999998545},"time":"2015-10-30T17:14:40Z"}]}

‎integrations/testdata/TestStream_Delete_Join.srpl‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
dbname
2+
rpname
3+
cpu,type=idle,host=serverA value=9,anothervalue=4.2 0000000001
4+
dbname
5+
rpname
6+
cpu,type=idle,host=serverA value=9,anothervalue=4.2 0000000002
7+
dbname
8+
rpname
9+
cpu,type=idle,host=serverA value=9,anothervalue=4.2 0000000021
10+
dbname
11+
rpname
12+
cpu,type=idle,host=serverA value=9,anothervalue=4.2 0000000022
13+
dbname
14+
rpname
15+
views,type=idle,host=serverA value=9,anothervalue=4.2 0000000001
16+
dbname
17+
rpname
18+
views,type=idle,host=serverA value=9,anothervalue=4.2 0000000002
19+
dbname
20+
rpname
21+
views,type=idle,host=serverA value=9,anothervalue=4.2 0000000021
22+
dbname
23+
rpname
24+
views,type=idle,host=serverA value=9,anothervalue=4.2 0000000022

‎join.go‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,31 @@ func (n *JoinNode) Point(src int, p edge.PointMessage) error {
9191

9292
func (n *JoinNode) Barrier(src int, b edge.BarrierMessage) error {
9393
g := n.getOrCreateGroup(b.GroupID())
94-
g.Barrier(src, b.Time())
94+
if err := g.Barrier(src, b.Time()); err != nil {
95+
return err
96+
}
9597
return edge.Forward(n.outs, b)
9698
}
9799

100+
// Delete deletes the group from the JoinNode, and resets the Low Marks for from the group from that source.
101+
// if deleteAll is set on the pipeline.Joinnode, then it any delete will delete
102+
func (n *JoinNode) Delete(src int, d edge.DeleteGroupMessage) error {
103+
groupID := d.GroupID()
104+
n.groupsMu.Lock()
105+
delete(n.groups, groupID)
106+
delete(n.matchGroupsBuffer, groupID)
107+
delete(n.specificGroupsBuffer, groupID)
108+
if n.j.DeleteAll {
109+
for x := range n.lowMarks {
110+
delete(n.lowMarks, x)
111+
}
112+
} else {
113+
delete(n.lowMarks, srcGroup{src: src, groupId: groupID})
114+
}
115+
n.groupsMu.Unlock()
116+
return edge.Forward(n.outs, d)
117+
}
118+
98119
func (n *JoinNode) Finish() error {
99120
// No more points are coming signal all groups to finish up.
100121
for _, group := range n.groups {
@@ -109,6 +130,7 @@ type messageMeta interface {
109130
edge.Message
110131
edge.PointMeta
111132
}
133+
112134
type srcPoint struct {
113135
Src int
114136
Msg messageMeta
@@ -123,15 +145,17 @@ func (n *JoinNode) doMessage(src int, m messageMeta) error {
123145
} else {
124146
// Just send point on to group, we are not joining on specific dimensions.
125147
group := n.getOrCreateGroup(m.GroupID())
126-
group.Collect(src, m)
148+
if err := group.Collect(src, m); err != nil {
149+
return err
150+
}
127151
}
128152
return nil
129153
}
130154

131-
// The purpose of this method is to match more specific points
132-
// with the less specific points as they arrive.
155+
// The purpose of this method is to match more-specific points
156+
// with the less-specific points as they arrive.
133157
//
134-
// Where 'more specific' means, that a point has more dimensions than the join.on dimensions.
158+
// Where 'more-specific' means, that a point has more dimensions than the join.on dimensions.
135159
func (n *JoinNode) matchPoints(p srcPoint) {
136160
// Specific points may be sent to the joinset without a matching point, but not the other way around.
137161
// This is because the specific points have the needed specific tag data.
@@ -232,7 +256,6 @@ func (n *JoinNode) matchPoints(p srcPoint) {
232256
n.getOrCreateSpecificGroup(groupId).Enqueue(p)
233257
}
234258
}
235-
236259
} else {
237260
// Cache match point.
238261
n.getOrCreateMatchGroup(groupId).Enqueue(p)

‎models/point.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ func ToGroupID(name string, tags map[string]string, dims Dimensions) GroupID {
113113
buf.WriteString(d)
114114
buf.WriteRune('=')
115115
buf.WriteString(tags[d])
116-
117116
}
118117
return GroupID(buf.String())
119118
}

‎pipeline/join.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ const (
4545
// .fill(0.0)
4646
// // name the resulting stream
4747
// .streamName('error_rate')
48+
// // treat a delete from one side of the join as a delete to all sides
49+
// .deleteAll(TRUE)
4850
// // Both the "value" fields from each parent have been prefixed
4951
// // with the respective names 'errors' and 'requests'.
5052
// |eval(lambda: "errors.value" / "requests.value")
@@ -70,6 +72,10 @@ type JoinNode struct {
7072
// Can be the empty string.
7173
Delimiter string `json:"delimiter"`
7274

75+
// Deletes both sides of the join regardless what
76+
// side receive the delete message.
77+
DeleteAll bool `json:"deleteAll"`
78+
7379
// The name of this new joined data stream.
7480
// If empty the name of the left parent is used.
7581
StreamName string `json:"streamName"`

0 commit comments

Comments
 (0)