Skip to content

Commit 0a9ec3f

Browse files
authored
Merge pull request #188 from PFZheng/master
Pick patchs from the internal stable branch of braft in Baidu.
2 parents 591f0cf + d552db7 commit 0a9ec3f

35 files changed

Lines changed: 3934 additions & 807 deletions

src/braft/builtin_service_impl.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,10 @@ void RaftStatImpl::default_method(::google::protobuf::RpcController* controller,
4141
brpc::Controller* cntl = (brpc::Controller*)controller;
4242
std::string group_id = cntl->http_request().unresolved_path();
4343
std::vector<scoped_refptr<NodeImpl> > nodes;
44-
NodeManager* nm = NodeManager::GetInstance();
4544
if (group_id.empty()) {
46-
nm->get_all_nodes(&nodes);
45+
global_node_manager->get_all_nodes(&nodes);
4746
} else {
48-
nm->get_nodes_by_group_id(group_id, &nodes);
47+
global_node_manager->get_nodes_by_group_id(group_id, &nodes);
4948
}
5049
const bool html = brpc::UseHTML(cntl->http_request());
5150
if (html) {

src/braft/cli_service.cpp

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -198,20 +198,20 @@ void CliServiceImpl::get_leader(::google::protobuf::RpcController* controller,
198198
brpc::Controller* cntl = (brpc::Controller*)controller;
199199
brpc::ClosureGuard done_guard(done);
200200
std::vector<scoped_refptr<NodeImpl> > nodes;
201-
NodeManager* const nm = NodeManager::GetInstance();
202201
if (request->has_peer_id()) {
203202
PeerId peer;
204203
if (peer.parse(request->peer_id()) != 0) {
205204
cntl->SetFailed(EINVAL, "Fail to parse %s",
206205
request->peer_id().c_str());
207206
return;
208207
}
209-
scoped_refptr<NodeImpl> node = nm->get(request->group_id(), peer);
208+
scoped_refptr<NodeImpl> node =
209+
global_node_manager->get(request->group_id(), peer);
210210
if (node) {
211211
nodes.push_back(node);
212212
}
213213
} else {
214-
nm->get_nodes_by_group_id(request->group_id(), &nodes);
214+
global_node_manager->get_nodes_by_group_id(request->group_id(), &nodes);
215215
}
216216
if (nodes.empty()) {
217217
cntl->SetFailed(ENOENT, "No nodes in group %s",
@@ -232,17 +232,16 @@ void CliServiceImpl::get_leader(::google::protobuf::RpcController* controller,
232232
butil::Status CliServiceImpl::get_node(scoped_refptr<NodeImpl>* node,
233233
const GroupId& group_id,
234234
const std::string& peer_id) {
235-
NodeManager* const nm = NodeManager::GetInstance();
236235
if (!peer_id.empty()) {
237-
*node = nm->get(group_id, peer_id);
236+
*node = global_node_manager->get(group_id, peer_id);
238237
if (!(*node)) {
239238
return butil::Status(ENOENT, "Fail to find node %s in group %s",
240239
peer_id.c_str(),
241240
group_id.c_str());
242241
}
243242
} else {
244243
std::vector<scoped_refptr<NodeImpl> > nodes;
245-
nm->get_nodes_by_group_id(group_id, &nodes);
244+
global_node_manager->get_nodes_by_group_id(group_id, &nodes);
246245
if (nodes.empty()) {
247246
return butil::Status(ENOENT, "Fail to find node in group %s",
248247
group_id.c_str());

src/braft/configuration.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
namespace braft {
3232

3333
typedef std::string GroupId;
34+
// GroupId with version, format: {group_id}_{index}
35+
typedef std::string VersionedGroupId;
3436

3537
// Represent a participant in a replicating group.
3638
struct PeerId {

src/braft/fsm_caller.cpp

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ int FSMCaller::run(void* meta, bthread::TaskIterator<ApplyTask>& iter) {
9898
delete iter->status;
9999
break;
100100
case LEADER_START:
101-
caller->do_leader_start(iter->term);
101+
caller->do_leader_start(*(iter->leader_start_context));
102+
delete iter->leader_start_context;
102103
break;
103104
case START_FOLLOWING:
104105
caller->_cur_task = START_FOLLOWING;
@@ -430,19 +431,26 @@ int FSMCaller::on_leader_stop(const butil::Status& status) {
430431
return 0;
431432
}
432433

433-
int FSMCaller::on_leader_start(int64_t term) {
434+
int FSMCaller::on_leader_start(int64_t term, int64_t lease_epoch) {
434435
ApplyTask task;
435436
task.type = LEADER_START;
436-
task.term = term;
437-
return bthread::execution_queue_execute(_queue_id, task);
437+
LeaderStartContext* on_leader_start_context =
438+
new LeaderStartContext(term, lease_epoch);
439+
task.leader_start_context = on_leader_start_context;
440+
if (bthread::execution_queue_execute(_queue_id, task) != 0) {
441+
delete on_leader_start_context;
442+
return -1;
443+
}
444+
return 0;
438445
}
439446

440447
void FSMCaller::do_leader_stop(const butil::Status& status) {
441448
_fsm->on_leader_stop(status);
442449
}
443450

444-
void FSMCaller::do_leader_start(int64_t term) {
445-
_fsm->on_leader_start(term);
451+
void FSMCaller::do_leader_start(const LeaderStartContext& leader_start_context) {
452+
_node->leader_lease_start(leader_start_context.lease_epoch);
453+
_fsm->on_leader_start(leader_start_context.term);
446454
}
447455

448456
int FSMCaller::on_start_following(const LeaderChangeContext& start_following_context) {

src/braft/fsm_caller.h

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "braft/closure_queue.h"
2727
#include "braft/macros.h"
2828
#include "braft/log_entry.h"
29+
#include "braft/lease.h"
2930

3031
namespace braft {
3132

@@ -112,7 +113,7 @@ class BAIDU_CACHELINE_ALIGNMENT FSMCaller {
112113
BRAFT_MOCK int on_snapshot_load(LoadSnapshotClosure* done);
113114
BRAFT_MOCK int on_snapshot_save(SaveSnapshotClosure* done);
114115
int on_leader_stop(const butil::Status& status);
115-
int on_leader_start(int64_t term);
116+
int on_leader_start(int64_t term, int64_t lease_epoch);
116117
int on_start_following(const LeaderChangeContext& start_following_context);
117118
int on_stop_following(const LeaderChangeContext& stop_following_context);
118119
BRAFT_MOCK int on_error(const Error& e);
@@ -138,14 +139,23 @@ friend class IteratorImpl;
138139
ERROR,
139140
};
140141

142+
struct LeaderStartContext {
143+
LeaderStartContext(int64_t term_, int64_t lease_epoch_)
144+
: term(term_), lease_epoch(lease_epoch_)
145+
{}
146+
147+
int64_t term;
148+
int64_t lease_epoch;
149+
};
150+
141151
struct ApplyTask {
142152
TaskType type;
143153
union {
144154
// For applying log entry (including configuration change)
145155
int64_t committed_index;
146156

147157
// For on_leader_start
148-
int64_t term;
158+
LeaderStartContext* leader_start_context;
149159

150160
// For on_leader_stop
151161
butil::Status* status;
@@ -167,7 +177,7 @@ friend class IteratorImpl;
167177
void do_snapshot_load(LoadSnapshotClosure* done);
168178
void do_on_error(OnErrorClousre* done);
169179
void do_leader_stop(const butil::Status& status);
170-
void do_leader_start(int64_t term);
180+
void do_leader_start(const LeaderStartContext& leader_start_context);
171181
void do_start_following(const LeaderChangeContext& start_following_context);
172182
void do_stop_following(const LeaderChangeContext& stop_following_context);
173183
void set_error(const Error& e);

src/braft/lease.cpp

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Authors: Pengfei Zheng (zhengpengfei@baidu.com)
16+
17+
#include <gflags/gflags.h>
18+
#include <brpc/reloadable_flags.h>
19+
#include "braft/lease.h"
20+
21+
namespace braft {
22+
23+
DEFINE_bool(raft_enable_leader_lease, false,
24+
"Enable or disable leader lease. only when all peers in a raft group "
25+
"set this configuration to true, leader lease check and vote are safe.");
26+
BRPC_VALIDATE_GFLAG(raft_enable_leader_lease, ::brpc::PassValidate);
27+
28+
void LeaderLease::init(int64_t election_timeout_ms) {
29+
_election_timeout_ms = election_timeout_ms;
30+
}
31+
32+
void LeaderLease::on_leader_start(int64_t term) {
33+
BAIDU_SCOPED_LOCK(_mutex);
34+
++_lease_epoch;
35+
_term = term;
36+
_last_active_timestamp = 0;
37+
}
38+
39+
void LeaderLease::on_leader_stop() {
40+
BAIDU_SCOPED_LOCK(_mutex);
41+
_last_active_timestamp = 0;
42+
_term = 0;
43+
}
44+
45+
void LeaderLease::on_lease_start(int64_t expect_lease_epoch, int64_t last_active_timestamp) {
46+
BAIDU_SCOPED_LOCK(_mutex);
47+
if (_term == 0 || expect_lease_epoch != _lease_epoch) {
48+
return;
49+
}
50+
_last_active_timestamp = last_active_timestamp;
51+
}
52+
53+
void LeaderLease::renew(int64_t last_active_timestamp) {
54+
BAIDU_SCOPED_LOCK(_mutex);
55+
_last_active_timestamp = last_active_timestamp;
56+
}
57+
58+
void LeaderLease::get_lease_info(LeaseInfo* lease_info) {
59+
lease_info->term = 0;
60+
lease_info->lease_epoch = 0;
61+
if (!FLAGS_raft_enable_leader_lease) {
62+
lease_info->state = LeaderLease::DISABLED;
63+
return;
64+
}
65+
66+
BAIDU_SCOPED_LOCK(_mutex);
67+
if (_term == 0) {
68+
lease_info->state = LeaderLease::EXPIRED;
69+
return;
70+
}
71+
if (_last_active_timestamp == 0) {
72+
lease_info->state = LeaderLease::NOT_READY;
73+
return;
74+
}
75+
if (butil::monotonic_time_ms() < _last_active_timestamp + _election_timeout_ms) {
76+
lease_info->term = _term;
77+
lease_info->lease_epoch = _lease_epoch;
78+
lease_info->state = LeaderLease::VALID;
79+
} else {
80+
lease_info->state = LeaderLease::SUSPECT;
81+
}
82+
}
83+
84+
int64_t LeaderLease::lease_epoch() {
85+
BAIDU_SCOPED_LOCK(_mutex);
86+
return _lease_epoch;
87+
}
88+
89+
void LeaderLease::reset_election_timeout_ms(int64_t election_timeout_ms) {
90+
BAIDU_SCOPED_LOCK(_mutex);
91+
_election_timeout_ms = election_timeout_ms;
92+
}
93+
94+
void FollowerLease::init(int64_t election_timeout_ms, int64_t max_clock_drift_ms) {
95+
_election_timeout_ms = election_timeout_ms;
96+
_max_clock_drift_ms = max_clock_drift_ms;
97+
// When the node restart, we are not sure when the lease will be expired actually,
98+
// so just be conservative.
99+
_last_leader_timestamp = butil::monotonic_time_ms();
100+
}
101+
102+
void FollowerLease::renew(const PeerId& leader_id) {
103+
_last_leader = leader_id;
104+
_last_leader_timestamp = butil::monotonic_time_ms();
105+
}
106+
107+
int64_t FollowerLease::votable_time_from_now() {
108+
if (!FLAGS_raft_enable_leader_lease) {
109+
return 0;
110+
}
111+
112+
int64_t now = butil::monotonic_time_ms();
113+
int64_t votable_timestamp = _last_leader_timestamp + _election_timeout_ms +
114+
_max_clock_drift_ms;
115+
if (now >= votable_timestamp) {
116+
return 0;
117+
}
118+
return votable_timestamp - now;
119+
}
120+
121+
const PeerId& FollowerLease::last_leader() {
122+
return _last_leader;
123+
}
124+
125+
bool FollowerLease::expired() {
126+
return butil::monotonic_time_ms() - _last_leader_timestamp
127+
>= _election_timeout_ms + _max_clock_drift_ms;
128+
}
129+
130+
void FollowerLease::reset() {
131+
_last_leader = PeerId();
132+
_last_leader_timestamp = 0;
133+
}
134+
135+
void FollowerLease::reset_election_timeout_ms(int64_t election_timeout_ms,
136+
int64_t max_clock_drift_ms) {
137+
_election_timeout_ms = election_timeout_ms;
138+
_max_clock_drift_ms = max_clock_drift_ms;
139+
}
140+
141+
} // namespace braft

src/braft/lease.h

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Authors: Pengfei Zheng (zhengpengfei@baidu.com)
16+
17+
#ifndef PUBLIC_RAFT_LEASE_H
18+
#define PUBLIC_RAFT_LEASE_H
19+
20+
#include "braft/util.h"
21+
22+
namespace braft {
23+
24+
class LeaderLease {
25+
public:
26+
enum InternalState {
27+
DISABLED,
28+
EXPIRED,
29+
NOT_READY,
30+
VALID,
31+
SUSPECT,
32+
};
33+
34+
struct LeaseInfo {
35+
InternalState state;
36+
int64_t term;
37+
int64_t lease_epoch;
38+
};
39+
40+
LeaderLease()
41+
: _election_timeout_ms(0)
42+
, _last_active_timestamp(0)
43+
, _term(0)
44+
, _lease_epoch(0)
45+
{}
46+
47+
void init(int64_t election_timeout_ms);
48+
void on_leader_start(int64_t term);
49+
void on_leader_stop();
50+
void on_lease_start(int64_t expect_lease_epoch, int64_t last_active_timestamp);
51+
void get_lease_info(LeaseInfo* lease_info);
52+
void renew(int64_t last_active_timestamp);
53+
int64_t lease_epoch();
54+
void reset_election_timeout_ms(int64_t election_timeout_ms);
55+
56+
private:
57+
raft_mutex_t _mutex;
58+
int64_t _election_timeout_ms;
59+
int64_t _last_active_timestamp;
60+
int64_t _term;
61+
int64_t _lease_epoch;
62+
};
63+
64+
class FollowerLease {
65+
public:
66+
FollowerLease()
67+
: _election_timeout_ms(0), _max_clock_drift_ms(0)
68+
, _last_leader_timestamp(0)
69+
{}
70+
71+
void init(int64_t election_timeout_ms, int64_t max_clock_drift_ms);
72+
void renew(const PeerId& leader_id);
73+
int64_t votable_time_from_now();
74+
const PeerId& last_leader();
75+
bool expired();
76+
void reset();
77+
void reset_election_timeout_ms(int64_t election_timeout_ms, int64_t max_clock_drift_ms);
78+
79+
private:
80+
int64_t _election_timeout_ms;
81+
int64_t _max_clock_drift_ms;
82+
PeerId _last_leader;
83+
int64_t _last_leader_timestamp;
84+
};
85+
86+
} // namespace braft
87+
88+
#endif // PUBLIC_RAFT_LEADER_LEASE_H

0 commit comments

Comments
 (0)