Skip to content

Commit 9fff94a

Browse files
committed
supporting heartbeat in cloudevents subscribe stream
Signed-off-by: Wei Liu <liuweixa@redhat.com>
1 parent 7d8041a commit 9fff94a

11 files changed

Lines changed: 435 additions & 49 deletions

File tree

pkg/cloudevents/generic/options/grpc/agentoptions.go

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
cloudevents "github.com/cloudevents/sdk-go/v2"
7+
"k8s.io/klog/v2"
78

89
"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options"
910
"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc/protocol"
@@ -34,11 +35,7 @@ func (o *grpcAgentOptions) WithContext(ctx context.Context, evtCtx cloudevents.E
3435
}
3536

3637
func (o *grpcAgentOptions) Protocol(ctx context.Context, dataType types.CloudEventsDataType) (options.CloudEventsProtocol, error) {
37-
receiver, err := o.GetCloudEventsProtocol(
38-
ctx,
39-
func(err error) {
40-
o.errorChan <- err
41-
},
38+
opts := []protocol.Option{
4239
protocol.WithSubscribeOption(&protocol.SubscribeOption{
4340
// TODO: Update this code to determine the subscription source for the agent client.
4441
// Currently, the grpc agent client is not utilized, and the 'Source' field serves
@@ -47,6 +44,23 @@ func (o *grpcAgentOptions) Protocol(ctx context.Context, dataType types.CloudEve
4744
ClusterName: o.clusterName,
4845
DataType: dataType.String(),
4946
}),
47+
protocol.WithReconnectErrorChan(o.errorChan),
48+
}
49+
50+
if o.ServerHealthinessTimeout != nil {
51+
opts = append(opts, protocol.WithServerHealthinessTimeout(o.ServerHealthinessTimeout))
52+
}
53+
54+
receiver, err := o.GetCloudEventsProtocol(
55+
ctx,
56+
func(err error) {
57+
select {
58+
case o.errorChan <- err:
59+
default:
60+
klog.Errorf("no error channel available to report error: %v", err)
61+
}
62+
},
63+
opts...,
5064
)
5165
if err != nil {
5266
return nil, err

pkg/cloudevents/generic/options/grpc/options.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ func (d *GRPCDialer) Close() error {
114114
// GRPCOptions holds the options that are used to build gRPC client.
115115
type GRPCOptions struct {
116116
Dialer *GRPCDialer
117+
118+
ServerHealthinessTimeout *time.Duration
117119
}
118120

119121
// GRPCConfig holds the information needed to build connect to gRPC server as a given user.
@@ -130,6 +132,10 @@ type GRPCConfig struct {
130132

131133
// keepalive options
132134
KeepAliveConfig KeepAliveConfig `json:"keepAliveConfig,omitempty" yaml:"keepAliveConfig,omitempty"`
135+
136+
// serverHealthinessTimeout is the max duration that client will reconnect if no server healthiness status is
137+
// received in this duration, if it is not set, client will not reconnect when health message is received
138+
ServerHealthinessTimeout *time.Duration `json:"serverHealthinessTimeout,omitempty" yaml:"serverHealthinessTimeout,omitempty"`
133139
}
134140

135141
// KeepAliveConfig holds the keepalive options for the gRPC client.
@@ -241,6 +247,7 @@ func BuildGRPCOptionsFromFlags(configPath string) (*GRPCOptions, error) {
241247
}
242248
}
243249

250+
options.ServerHealthinessTimeout = config.ServerHealthinessTimeout
244251
return options, nil
245252
}
246253

pkg/cloudevents/generic/options/grpc/options_test.go

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/google/go-cmp/cmp"
99
"github.com/google/go-cmp/cmp/cmpopts"
10+
"k8s.io/utils/ptr"
1011
clienttesting "open-cluster-management.io/sdk-go/pkg/testing"
1112
)
1213

@@ -36,7 +37,7 @@ func TestBuildGRPCOptionsFromFlags(t *testing.T) {
3637
name: "customized options",
3738
config: "{\"url\":\"test\"}",
3839
expectedOptions: &GRPCOptions{
39-
&GRPCDialer{
40+
Dialer: &GRPCDialer{
4041
URL: "test",
4142
KeepAliveOptions: KeepAliveOptions{
4243
Enable: false,
@@ -51,7 +52,7 @@ func TestBuildGRPCOptionsFromFlags(t *testing.T) {
5152
name: "customized options with yaml format",
5253
config: "url: test",
5354
expectedOptions: &GRPCOptions{
54-
&GRPCDialer{
55+
Dialer: &GRPCDialer{
5556
URL: "test",
5657
KeepAliveOptions: KeepAliveOptions{
5758
Enable: false,
@@ -66,7 +67,7 @@ func TestBuildGRPCOptionsFromFlags(t *testing.T) {
6667
name: "customized options with keepalive",
6768
config: "{\"url\":\"test\",\"keepAliveConfig\":{\"enable\":true,\"time\":10s,\"timeout\":5s,\"permitWithoutStream\":true}}",
6869
expectedOptions: &GRPCOptions{
69-
&GRPCDialer{
70+
Dialer: &GRPCDialer{
7071
URL: "test",
7172
KeepAliveOptions: KeepAliveOptions{
7273
Enable: true,
@@ -77,6 +78,22 @@ func TestBuildGRPCOptionsFromFlags(t *testing.T) {
7778
},
7879
},
7980
},
81+
{
82+
name: "customized options with ServerHealthinessTimeout",
83+
config: "{\"url\":\"test\",\"serverHealthinessTimeout\":\"10s\"}",
84+
expectedOptions: &GRPCOptions{
85+
Dialer: &GRPCDialer{
86+
URL: "test",
87+
KeepAliveOptions: KeepAliveOptions{
88+
Enable: false,
89+
Time: 30 * time.Second,
90+
Timeout: 10 * time.Second,
91+
PermitWithoutStream: false,
92+
},
93+
},
94+
ServerHealthinessTimeout: ptr.To(10 * time.Second),
95+
},
96+
},
8097
}
8198

8299
for _, c := range cases {

pkg/cloudevents/generic/options/grpc/protocol/message.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ const (
1919
prefix = "ce-"
2020
contenttype = "contenttype"
2121
// dataSchema = "dataschema"
22-
subject = "subject"
23-
time = "time"
22+
subject = "subject"
23+
timestamp = "time"
2424
)
2525

2626
var specs = spec.WithPrefix(prefix)

pkg/cloudevents/generic/options/grpc/protocol/option.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package protocol
22

33
import (
44
"fmt"
5+
"time"
56
)
67

78
// Option is the function signature
@@ -24,3 +25,25 @@ func WithSubscribeOption(subscribeOpt *SubscribeOption) Option {
2425
return nil
2526
}
2627
}
28+
29+
func WithReconnectErrorChan(errorChan chan error) Option {
30+
return func(p *Protocol) error {
31+
if errorChan == nil {
32+
return fmt.Errorf("the error channel must not be nil")
33+
}
34+
p.reconnectErrorChan = errorChan
35+
return nil
36+
}
37+
}
38+
39+
func WithServerHealthinessTimeout(timeout *time.Duration) Option {
40+
return func(p *Protocol) error {
41+
if timeout != nil {
42+
if *timeout <= 0 {
43+
return fmt.Errorf("the server healthiness timeout must be greater than 0")
44+
}
45+
p.serverHealthinessTimeout = timeout
46+
}
47+
return nil
48+
}
49+
}
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
package protocol
2+
3+
import (
4+
"context"
5+
"net"
6+
"testing"
7+
"time"
8+
9+
"github.com/google/uuid"
10+
"google.golang.org/grpc/credentials/insecure"
11+
"k8s.io/utils/ptr"
12+
13+
"google.golang.org/grpc"
14+
"google.golang.org/grpc/test/bufconn"
15+
"google.golang.org/protobuf/types/known/emptypb"
16+
17+
pbv1 "open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc/protobuf/v1"
18+
"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/types"
19+
)
20+
21+
const bufSize = 1024 * 1024
22+
23+
// mockCloudEventService implements a basic mock of CloudEventServiceServer
24+
type mockCloudEventService struct {
25+
pbv1.UnimplementedCloudEventServiceServer
26+
healthEnabled bool
27+
}
28+
29+
func (m *mockCloudEventService) Publish(ctx context.Context, req *pbv1.PublishRequest) (*emptypb.Empty, error) {
30+
return &emptypb.Empty{}, nil
31+
}
32+
33+
func (m *mockCloudEventService) Subscribe(req *pbv1.SubscriptionRequest, stream pbv1.CloudEventService_SubscribeServer) error {
34+
if m.healthEnabled {
35+
go func() {
36+
ticker := time.NewTicker(500 * time.Millisecond)
37+
defer ticker.Stop()
38+
39+
for {
40+
select {
41+
case <-ticker.C:
42+
heartbeat := &pbv1.CloudEvent{
43+
SpecVersion: "1.0",
44+
Id: uuid.New().String(),
45+
Type: types.HeartbeatCloudEventsType,
46+
}
47+
48+
if err := stream.Send(heartbeat); err != nil {
49+
return
50+
}
51+
case <-stream.Context().Done():
52+
return
53+
}
54+
}
55+
}()
56+
}
57+
// Keep the stream open for testing
58+
<-stream.Context().Done()
59+
return stream.Context().Err()
60+
}
61+
62+
func setupMockServer(t *testing.T, healthEnabled bool) (*grpc.ClientConn, func()) {
63+
lis := bufconn.Listen(bufSize)
64+
s := grpc.NewServer()
65+
66+
// Register cloud event service
67+
pbv1.RegisterCloudEventServiceServer(s, &mockCloudEventService{healthEnabled: healthEnabled})
68+
69+
go func() {
70+
if err := s.Serve(lis); err != nil {
71+
t.Logf("Server exited with error: %v", err)
72+
}
73+
}()
74+
75+
// Wait for server to start accepting connections
76+
time.Sleep(50 * time.Millisecond)
77+
78+
bufDialer := func(context.Context, string) (net.Conn, error) {
79+
return lis.Dial()
80+
}
81+
82+
conn, err := grpc.NewClient("passthrough:///bufnet",
83+
grpc.WithTransportCredentials(insecure.NewCredentials()),
84+
grpc.WithContextDialer(bufDialer))
85+
86+
if err != nil {
87+
t.Fatalf("Failed to create client: %v", err)
88+
}
89+
90+
cleanup := func() {
91+
conn.Close()
92+
s.Stop()
93+
}
94+
95+
return conn, cleanup
96+
}
97+
98+
func TestProtocol_Success(t *testing.T) {
99+
conn, cleanup := setupMockServer(t, true)
100+
defer cleanup()
101+
102+
reconnectErrorChan := make(chan error, 1)
103+
p, err := NewProtocol(
104+
conn,
105+
WithSubscribeOption(&SubscribeOption{
106+
Source: "test",
107+
ClusterName: "test-cluster",
108+
DataType: "io.open-cluster-management.test",
109+
}),
110+
WithReconnectErrorChan(reconnectErrorChan),
111+
WithServerHealthinessTimeout(ptr.To(5*time.Second)),
112+
)
113+
if err != nil {
114+
t.Fatal(err)
115+
}
116+
117+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
118+
defer cancel()
119+
120+
go func() {
121+
if err := p.OpenInbound(ctx); err != nil {
122+
p.reconnectErrorChan <- err
123+
}
124+
}()
125+
126+
// Should not receive any error since health check was successful
127+
select {
128+
case err := <-p.reconnectErrorChan:
129+
t.Errorf("Unexpected error from health check: %v", err)
130+
case <-time.After(2 * time.Second):
131+
// Expected - no error
132+
}
133+
}
134+
135+
func TestProtocol_Timeout(t *testing.T) {
136+
conn, cleanup := setupMockServer(t, false) // No health service
137+
defer cleanup()
138+
139+
reconnectErrorChan := make(chan error, 1)
140+
p, err := NewProtocol(
141+
conn,
142+
WithSubscribeOption(&SubscribeOption{
143+
Source: "test",
144+
ClusterName: "test-cluster",
145+
DataType: "io.open-cluster-management.test",
146+
}),
147+
WithReconnectErrorChan(reconnectErrorChan),
148+
WithServerHealthinessTimeout(ptr.To(time.Second)),
149+
)
150+
if err != nil {
151+
t.Fatal(err)
152+
}
153+
154+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
155+
defer cancel()
156+
157+
go func() {
158+
if err := p.OpenInbound(ctx); err != nil {
159+
p.reconnectErrorChan <- err
160+
}
161+
}()
162+
163+
// Should receive an error due to health service not being available
164+
select {
165+
case err := <-p.reconnectErrorChan:
166+
if err == nil {
167+
t.Errorf("Expected health check error, but got nil %v", err)
168+
}
169+
case <-time.After(3 * time.Second):
170+
t.Error("Expected health check error within timeout")
171+
}
172+
}

0 commit comments

Comments
 (0)