Skip to content

Commit 034ef2c

Browse files
Nasf-FanNasf-Fan
committed
DAOS-18891 object: retry if vos_update_end return -DER_AGAIN - b28
On server side, for an update operation, there may be CPU yield between related vos_update_begin() and vos_update_end(). During yield interval, the object that is held via vos_update_begin() maybe evicted by others, such as by another failed modification against the same object shard or evicted under md-on-ssd mode. So vos_update_end() logic will check such case and return -DER_AGAIN instead of -DER_TX_RESTART to the caller for notification. And then related caller needs to retry update instead of fail out. The patch also adds initialization for some local varilables in object module to avoid random corruption when handle some failure cases. Signed-off-by: Fan Yong <fan.yong@hpe.com>
1 parent f0e08e5 commit 034ef2c

6 files changed

Lines changed: 154 additions & 77 deletions

File tree

‎src/object/srv_ec_aggregate.c‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -796,6 +796,7 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry,
796796
struct dcs_iod_csums *iod_csums = NULL;
797797
int err;
798798
int rc = 0;
799+
uint64_t ts = daos_gettime_coarse();
799800

800801
ap = container_of(entry, struct ec_agg_param, ap_agg_entry);
801802

@@ -829,9 +830,14 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry,
829830
}
830831
D_ASSERT(iod_csums != NULL);
831832
}
833+
834+
again1:
832835
rc = vos_obj_update(ap->ap_cont_handle, entry->ae_oid,
833836
entry->ae_cur_stripe.as_hi_epoch, 0, VOS_OF_CRIT,
834837
&entry->ae_dkey, 1, &iod, iod_csums, &sgl);
838+
839+
OBJ_CHECK_EAGAIN(rc, ts, "vos_obj_update", entry->ae_oid, again1);
840+
835841
if (csummer != NULL && iod_csums != NULL)
836842
daos_csummer_free_ic(csummer, &iod_csums);
837843
if (rc) {
@@ -844,8 +850,13 @@ agg_update_vos(struct ec_agg_param *agg_param, struct ec_agg_entry *entry,
844850
recx.rx_nr = ec_age2ss(entry);
845851
epoch_range.epr_lo = ap->ap_epr.epr_lo;
846852
epoch_range.epr_hi = entry->ae_cur_stripe.as_hi_epoch;
853+
854+
again2:
847855
err = vos_obj_array_remove(ap->ap_cont_handle, entry->ae_oid, &epoch_range, &entry->ae_dkey,
848856
&entry->ae_akey, &recx);
857+
858+
OBJ_CHECK_EAGAIN(err, ts, "vos_obj_array_remove", entry->ae_oid, again2);
859+
849860
if (err)
850861
D_ERROR("array_remove fails: " DF_RC "\n", DP_RC(err));
851862
if (!rc && err)
@@ -1817,6 +1828,7 @@ agg_process_holes(struct ec_agg_entry *entry)
18171828
struct ec_agg_param *agg_param;
18181829
int tid, rc = 0;
18191830
int *status;
1831+
uint64_t ts = daos_gettime_coarse();
18201832

18211833
agg_param = container_of(entry, struct ec_agg_param, ap_agg_entry);
18221834
/* If rebuild started, abort it before sending RPC to save conflict window with rebuild
@@ -1868,10 +1880,15 @@ agg_process_holes(struct ec_agg_entry *entry)
18681880
if (iod->iod_nr) {
18691881
/* write the reps to vos */
18701882
entry->ae_sgl.sg_nr = 1;
1883+
1884+
again1:
18711885
rc = vos_obj_update(agg_param->ap_cont_handle, entry->ae_oid,
18721886
entry->ae_cur_stripe.as_hi_epoch, 0, VOS_OF_CRIT,
18731887
&entry->ae_dkey, 1, iod, stripe_ud.asu_iod_csums,
18741888
&entry->ae_sgl);
1889+
1890+
OBJ_CHECK_EAGAIN(rc, ts, "vos_obj_update", entry->ae_oid, again1);
1891+
18751892
if (rc) {
18761893
DL_ERROR(rc, "vos_obj_update failed");
18771894
goto ev_out;
@@ -1884,10 +1901,13 @@ agg_process_holes(struct ec_agg_entry *entry)
18841901
recx.rx_nr = ec_age2cs(entry);
18851902
recx.rx_idx = (entry->ae_cur_stripe.as_stripenum * recx.rx_nr) |
18861903
PARITY_INDICATOR;
1904+
1905+
again2:
18871906
rc = vos_obj_array_remove(agg_param->ap_cont_handle,
18881907
entry->ae_oid, &epoch_range,
18891908
&entry->ae_dkey, &entry->ae_akey,
18901909
&recx);
1910+
OBJ_CHECK_EAGAIN(rc, rc, "vos_obj_array_remove", entry->ae_oid, again2);
18911911
} else {
18921912
D_DEBUG(DB_EPC, "no valid hole, set ae_process_partial flag\n");
18931913
entry->ae_process_partial = 1;

‎src/object/srv_internal.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,22 @@ enum latency_type {
211211
VOS_LATENCY,
212212
};
213213

214+
#define OBJ_CHECK_EAGAIN(result, ts, func, oid, label) \
215+
do { \
216+
if (unlikely((result) == -DER_AGAIN)) { \
217+
uint64_t now = daos_gettime_coarse(); \
218+
\
219+
if (now - ts > 30) { \
220+
D_WARN("%s hit conflict on object " DF_UOID ", will retry.\n", \
221+
func, DP_UOID(oid)); \
222+
ts = now; \
223+
} \
224+
\
225+
ABT_thread_yield(); \
226+
goto label; \
227+
} \
228+
} while (0)
229+
214230
static inline void
215231
obj_update_latency(uint32_t opc, uint32_t type, uint64_t latency, uint64_t io_size)
216232
{

‎src/object/srv_obj.c‎

Lines changed: 82 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -2527,16 +2527,17 @@ obj_ioc_begin(daos_obj_id_t oid, uint32_t rpc_map_ver, uuid_t pool_uuid,
25272527
void
25282528
ds_obj_ec_rep_handler(crt_rpc_t *rpc)
25292529
{
2530-
struct obj_ec_rep_in *oer = crt_req_get(rpc);
2531-
struct obj_ec_rep_out *oero = crt_reply_get(rpc);
2532-
daos_key_t *dkey;
2533-
daos_iod_t *iod;
2534-
struct dcs_iod_csums *iod_csums;
2535-
struct bio_desc *biod;
2536-
daos_recx_t recx = { 0 };
2537-
struct obj_io_context ioc;
2538-
daos_handle_t ioh = DAOS_HDL_INVAL;
2539-
int rc;
2530+
struct obj_ec_rep_in *oer = crt_req_get(rpc);
2531+
struct obj_ec_rep_out *oero = crt_reply_get(rpc);
2532+
daos_key_t *dkey;
2533+
daos_iod_t *iod;
2534+
struct dcs_iod_csums *iod_csums;
2535+
struct bio_desc *biod;
2536+
daos_recx_t recx = {0};
2537+
struct obj_io_context ioc = {0};
2538+
daos_handle_t ioh = DAOS_HDL_INVAL;
2539+
uint64_t ts = daos_gettime_coarse();
2540+
int rc;
25402541

25412542
D_ASSERT(oer != NULL);
25422543
D_ASSERT(oero != NULL);
@@ -2561,6 +2562,8 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc)
25612562
iod = (daos_iod_t *)&oer->er_iod;
25622563
if (iod->iod_nr == 0) /* nothing to replicate, directly remove parity */
25632564
goto remove_parity;
2565+
2566+
again:
25642567
iod_csums = oer->er_iod_csums.ca_arrays;
25652568
rc = vos_update_begin(ioc.ioc_coc->sc_hdl, oer->er_oid, oer->er_epoch_range.epr_hi,
25662569
VOS_OF_REBUILD | VOS_OF_CRIT, dkey, 1, iod, iod_csums, 0, &ioh, NULL);
@@ -2589,6 +2592,8 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc)
25892592
end:
25902593
rc = vos_update_end(ioh, ioc.ioc_map_ver, dkey, rc, &ioc.ioc_io_size, NULL);
25912594
if (rc) {
2595+
OBJ_CHECK_EAGAIN(rc, ts, "vos_update_end", oer->er_oid, again);
2596+
25922597
D_ERROR(DF_UOID " vos_update_end failed: " DF_RC "\n", DP_UOID(oer->er_oid),
25932598
DP_RC(rc));
25942599
goto out_agg;
@@ -2598,6 +2603,8 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc)
25982603
recx.rx_idx = (oer->er_stripenum * recx.rx_nr) | PARITY_INDICATOR;
25992604
rc = vos_obj_array_remove(ioc.ioc_coc->sc_hdl, oer->er_oid, &oer->er_epoch_range, dkey,
26002605
&iod->iod_name, &recx);
2606+
OBJ_CHECK_EAGAIN(rc, ts, "vos_obj_array_remove", oer->er_oid, remove_parity);
2607+
26012608
out_agg:
26022609
ioc.ioc_coc->sc_ec_agg_updates--;
26032610
out:
@@ -2608,19 +2615,19 @@ ds_obj_ec_rep_handler(crt_rpc_t *rpc)
26082615
void
26092616
ds_obj_ec_agg_handler(crt_rpc_t *rpc)
26102617
{
2611-
struct obj_ec_agg_in *oea = crt_req_get(rpc);
2612-
struct obj_ec_agg_out *oeao = crt_reply_get(rpc);
2613-
daos_key_t *dkey;
2614-
struct bio_desc *biod;
2615-
daos_iod_t *iod = &oea->ea_iod;
2616-
struct dcs_iod_csums *iod_csums = oea->ea_iod_csums.ca_arrays;
2617-
2618-
crt_bulk_t parity_bulk = oea->ea_bulk;
2619-
daos_recx_t recx = { 0 };
2620-
struct obj_io_context ioc;
2621-
daos_handle_t ioh = DAOS_HDL_INVAL;
2622-
int rc;
2623-
int rc1;
2618+
struct obj_ec_agg_in *oea = crt_req_get(rpc);
2619+
struct obj_ec_agg_out *oeao = crt_reply_get(rpc);
2620+
daos_key_t *dkey;
2621+
struct bio_desc *biod;
2622+
daos_iod_t *iod = &oea->ea_iod;
2623+
struct dcs_iod_csums *iod_csums = oea->ea_iod_csums.ca_arrays;
2624+
crt_bulk_t parity_bulk = oea->ea_bulk;
2625+
daos_recx_t recx = {0};
2626+
struct obj_io_context ioc = {0};
2627+
daos_handle_t ioh = DAOS_HDL_INVAL;
2628+
uint64_t ts = daos_gettime_coarse();
2629+
int rc;
2630+
int rc1;
26242631

26252632
D_ASSERT(oea != NULL);
26262633
D_ASSERT(oeao != NULL);
@@ -2643,6 +2650,7 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc)
26432650

26442651
dkey = (daos_key_t *)&oea->ea_dkey;
26452652
if (parity_bulk != CRT_BULK_NULL) {
2653+
again1:
26462654
rc = vos_update_begin(ioc.ioc_coc->sc_hdl, oea->ea_oid, oea->ea_epoch_range.epr_hi,
26472655
VOS_OF_REBUILD | VOS_OF_CRIT, dkey, 1, iod, iod_csums, 0,
26482656
&ioh, NULL);
@@ -2672,6 +2680,8 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc)
26722680
end:
26732681
rc = vos_update_end(ioh, ioc.ioc_map_ver, dkey, rc, &ioc.ioc_io_size, NULL);
26742682
if (rc) {
2683+
OBJ_CHECK_EAGAIN(rc, ts, "vos_update_end", oea->ea_oid, again1);
2684+
26752685
if (rc == -DER_NO_PERM) {
26762686
/* Parity already exists, May need a
26772687
* different error code.
@@ -2693,9 +2703,13 @@ ds_obj_ec_agg_handler(crt_rpc_t *rpc)
26932703
*/
26942704
recx.rx_idx = oea->ea_stripenum * obj_ioc2ec_ss(&ioc);
26952705
recx.rx_nr = obj_ioc2ec_ss(&ioc);
2706+
again2:
26962707
rc1 = vos_obj_array_remove(ioc.ioc_coc->sc_hdl, oea->ea_oid,
26972708
&oea->ea_epoch_range, dkey,
26982709
&iod->iod_name, &recx);
2710+
2711+
OBJ_CHECK_EAGAIN(rc1, ts, "vos_obj_array_remove", oea->ea_oid, again2);
2712+
26992713
if (rc1)
27002714
D_ERROR(DF_UOID ": array_remove failed: " DF_RC "\n", DP_UOID(oea->ea_oid),
27012715
DP_RC(rc1));
@@ -2790,18 +2804,18 @@ obj_handle_resend(daos_handle_t coh, struct dtx_id *dti, daos_epoch_t *epoch, ui
27902804
void
27912805
ds_obj_tgt_update_handler(crt_rpc_t *rpc)
27922806
{
2793-
struct obj_rw_in *orw = crt_req_get(rpc);
2794-
struct obj_rw_out *orwo = crt_reply_get(rpc);
2795-
daos_key_t *dkey = &orw->orw_dkey;
2796-
struct obj_io_context ioc;
2797-
struct dtx_handle *dth = NULL;
2798-
struct dtx_memberships *mbs = NULL;
2799-
struct daos_shard_tgt *tgts = NULL;
2800-
uint32_t tgt_cnt;
2801-
uint32_t opc = opc_get(rpc->cr_opc);
2802-
uint32_t dtx_flags = 0;
2803-
struct dtx_epoch epoch;
2804-
int rc;
2807+
struct obj_rw_in *orw = crt_req_get(rpc);
2808+
struct obj_rw_out *orwo = crt_reply_get(rpc);
2809+
daos_key_t *dkey = &orw->orw_dkey;
2810+
struct obj_io_context ioc = {0};
2811+
struct dtx_handle *dth = NULL;
2812+
struct dtx_memberships *mbs = NULL;
2813+
struct daos_shard_tgt *tgts = NULL;
2814+
uint32_t tgt_cnt;
2815+
uint32_t opc = opc_get(rpc->cr_opc);
2816+
uint32_t dtx_flags = 0;
2817+
struct dtx_epoch epoch;
2818+
int rc;
28052819

28062820
D_ASSERT(orw != NULL);
28072821
D_ASSERT(orwo != NULL);
@@ -3251,11 +3265,11 @@ ds_obj_rw_handler(crt_rpc_t *rpc)
32513265
d_tm_inc_counter(opm->opm_update_restart, 1);
32523266
goto again;
32533267
case -DER_AGAIN:
3268+
ABT_thread_yield();
32543269
need_abort = true;
32553270
exec_arg.flags |= ORF_RESEND;
32563271
flags = ORF_RESEND;
32573272
d_tm_inc_counter(opm->opm_update_retry, 1);
3258-
ABT_thread_yield();
32593273
goto again;
32603274
default:
32613275
break;
@@ -3573,14 +3587,14 @@ obj_enum_reply_bulk(crt_rpc_t *rpc)
35733587
void
35743588
ds_obj_enum_handler(crt_rpc_t *rpc)
35753589
{
3576-
struct ds_obj_enum_arg enum_arg = { 0 };
3577-
struct vos_iter_anchors *anchors = NULL;
3578-
struct obj_key_enum_in *oei;
3579-
struct obj_key_enum_out *oeo;
3580-
struct obj_io_context ioc;
3581-
daos_epoch_t epoch = 0;
3582-
int opc = opc_get(rpc->cr_opc);
3583-
int rc = 0;
3590+
struct ds_obj_enum_arg enum_arg = {0};
3591+
struct vos_iter_anchors *anchors = NULL;
3592+
struct obj_key_enum_in *oei;
3593+
struct obj_key_enum_out *oeo;
3594+
struct obj_io_context ioc = {0};
3595+
daos_epoch_t epoch = 0;
3596+
int opc = opc_get(rpc->cr_opc);
3597+
int rc = 0;
35843598

35853599
oei = crt_req_get(rpc);
35863600
D_ASSERT(oei != NULL);
@@ -4163,10 +4177,10 @@ ds_obj_punch_handler(crt_rpc_t *rpc)
41634177
flags = ORF_RESEND;
41644178
goto again;
41654179
case -DER_AGAIN:
4180+
ABT_thread_yield();
41664181
need_abort = true;
41674182
exec_arg.flags |= ORF_RESEND;
41684183
flags = ORF_RESEND;
4169-
ABT_thread_yield();
41704184
goto again;
41714185
default:
41724186
break;
@@ -4457,11 +4471,11 @@ ds_obj_query_key_handler(crt_rpc_t *rpc)
44574471
void
44584472
ds_obj_sync_handler(crt_rpc_t *rpc)
44594473
{
4460-
struct obj_sync_in *osi;
4461-
struct obj_sync_out *oso;
4462-
struct obj_io_context ioc;
4463-
daos_epoch_t epoch = d_hlc_get();
4464-
int rc;
4474+
struct obj_sync_in *osi;
4475+
struct obj_sync_out *oso;
4476+
struct obj_io_context ioc = {0};
4477+
daos_epoch_t epoch = d_hlc_get();
4478+
int rc;
44654479

44664480
osi = crt_req_get(rpc);
44674481
D_ASSERT(osi != NULL);
@@ -5589,18 +5603,18 @@ ds_obj_cpd_body_bulk(crt_rpc_t *rpc, struct obj_io_context *ioc, bool leader,
55895603
void
55905604
ds_obj_cpd_handler(crt_rpc_t *rpc)
55915605
{
5592-
struct obj_cpd_in *oci = crt_req_get(rpc);
5593-
struct obj_cpd_out *oco = crt_reply_get(rpc);
5594-
struct daos_cpd_args *dcas = NULL;
5595-
struct obj_io_context ioc;
5596-
ABT_future future = ABT_FUTURE_NULL;
5597-
struct daos_cpd_bulk **dcbs = NULL;
5598-
uint32_t dcb_nr = 0;
5599-
int tx_count = oci->oci_sub_heads.ca_count;
5600-
int rc = 0;
5601-
int i;
5602-
int j;
5603-
bool leader;
5606+
struct obj_cpd_in *oci = crt_req_get(rpc);
5607+
struct obj_cpd_out *oco = crt_reply_get(rpc);
5608+
struct daos_cpd_args *dcas = NULL;
5609+
struct obj_io_context ioc = {0};
5610+
ABT_future future = ABT_FUTURE_NULL;
5611+
struct daos_cpd_bulk **dcbs = NULL;
5612+
uint32_t dcb_nr = 0;
5613+
int tx_count = oci->oci_sub_heads.ca_count;
5614+
int rc = 0;
5615+
int i;
5616+
int j;
5617+
bool leader;
56045618

56055619
D_ASSERT(oci != NULL);
56065620

@@ -5753,11 +5767,11 @@ ds_obj_cpd_handler(crt_rpc_t *rpc)
57535767
void
57545768
ds_obj_key2anchor_handler(crt_rpc_t *rpc)
57555769
{
5756-
struct obj_key2anchor_in *oki;
5757-
struct obj_key2anchor_out *oko;
5758-
struct obj_io_context ioc;
5759-
daos_key_t *akey = NULL;
5760-
int rc = 0;
5770+
struct obj_key2anchor_in *oki;
5771+
struct obj_key2anchor_out *oko;
5772+
struct obj_io_context ioc = {0};
5773+
daos_key_t *akey = NULL;
5774+
int rc = 0;
57615775

57625776
oki = crt_req_get(rpc);
57635777
D_ASSERT(oki != NULL);
@@ -5935,10 +5949,10 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc)
59355949
flags = ORF_RESEND;
59365950
goto again;
59375951
case -DER_AGAIN:
5952+
ABT_thread_yield();
59385953
need_abort = true;
59395954
exec_arg.flags |= ORF_RESEND;
59405955
flags = ORF_RESEND;
5941-
ABT_thread_yield();
59425956
goto again;
59435957
default:
59445958
break;

0 commit comments

Comments
 (0)