Skip to content

Commit 65db489

Browse files
authored
fix(poll): exponential backoff on 401 Unauthorized (#163)
## Summary When the Hub returns `401 Unauthorized` during a poll round (token expired or revoked), the poll loop kept hammering `/api/tree` every `poll_interval` seconds with the same invalid token, generating noise and unnecessary load. This adds an exponential backoff specific to 401s in `src/virtual_fs/poll.rs`: - Track `auth_backoff_exp`, incremented each round that observes at least one `Error::Hub { status: Some(401), .. }`. - Next sleep becomes `interval * 2^exp`, capped at `2^6` (so ~32 min with a 30 s base interval). - Exponent resets to 0 as soon as a poll round completes without any 401, so recovery (e.g. token refresh) is immediate. - Other failures (504s, network errors, etc.) keep behaving exactly as before, no backoff applied.
1 parent 7c47438 commit 65db489

1 file changed

Lines changed: 24 additions & 1 deletion

File tree

src/virtual_fs/poll.rs

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,17 @@ use std::time::{Duration, Instant, SystemTime};
55
use futures::stream::{self, StreamExt};
66
use tracing::{info, warn};
77

8+
use crate::error::Error;
89
use crate::hub_api::HubOps;
910

1011
use super::Invalidator;
1112
use super::inode::{self, InodeTable};
1213

14+
/// Cap on the exponential-backoff multiplier applied to the poll interval
15+
/// when we keep getting 401s. With `interval = 30s` and `MAX_AUTH_BACKOFF_EXP = 6`,
16+
/// the max delay between polls becomes `30s * 2^6 = 32 min`.
17+
const MAX_AUTH_BACKOFF_EXP: u32 = 6;
18+
1319
impl super::VirtualFs {
1420
/// Background task: polls Hub API tree listing to detect remote changes.
1521
///
@@ -27,8 +33,11 @@ impl super::VirtualFs {
2733
interval: Duration,
2834
listing_concurrency: usize,
2935
) {
36+
// Exponent applied to `interval` when the Hub returns 401 (token expired
37+
// or revoked). Reset to 0 as soon as we see a successful round.
38+
let mut auth_backoff_exp: u32 = 0;
3039
loop {
31-
tokio::time::sleep(interval).await;
40+
tokio::time::sleep(interval.saturating_mul(1u32 << auth_backoff_exp)).await;
3241

3342
// Only poll directories the user has actually visited (children_loaded).
3443
// This avoids fetching the entire tree for large repos where most
@@ -49,18 +58,32 @@ impl super::VirtualFs {
4958
let mut all_entries = Vec::new();
5059
let mut polled_prefixes = HashSet::new();
5160
let mut failed_prefixes = Vec::new();
61+
let mut saw_auth_failure = false;
5262
for (prefix, result) in results {
5363
match result {
5464
Ok(entries) => {
5565
polled_prefixes.insert(prefix);
5666
all_entries.extend(entries);
5767
}
5868
Err(e) => {
69+
if matches!(e, Error::Hub { status: Some(401), .. }) {
70+
saw_auth_failure = true;
71+
}
5972
warn!("Remote poll failed for prefix '{prefix}': {e}");
6073
failed_prefixes.push(prefix);
6174
}
6275
}
6376
}
77+
if saw_auth_failure {
78+
auth_backoff_exp = (auth_backoff_exp + 1).min(MAX_AUTH_BACKOFF_EXP);
79+
warn!(
80+
"Remote poll saw 401 Unauthorized; backing off next poll to {:?}",
81+
interval.saturating_mul(1u32 << auth_backoff_exp)
82+
);
83+
} else if auth_backoff_exp > 0 {
84+
info!("Remote poll auth recovered, resetting backoff");
85+
auth_backoff_exp = 0;
86+
}
6487
// For failed prefixes, check if the parent was polled successfully
6588
// and the dir no longer appears in its listing. If so, the dir was
6689
// deleted remotely — mark it as polled so its files get cleaned up.

0 commit comments

Comments
 (0)