Skip to content

Commit 3537f1f

Browse files
committed
Faster upload token refresh on 5xx errors during uploads
When a server returns a 5xx error (e.g., 503 Service Unavailable) during file upload, the SDK now properly refreshes upload tokens instead of retrying with stale cached upload URLs a bunch of times first. Previously, the HTTP layer would catch ServiceError and retry using the same pre-fetched upload URLs from the token pool. This caused repeated failures because the upload URLs may no longer be valid after a 5xx error and it delayed the recovery. The fix introduces ServiceErrorDuringUpload exception that disables HTTP-level retries (similar to B2RequestTimeoutDuringUpload), forcing the error to propagate to the upload manager. The upload manager then clears cached tokens via clear_bucket_upload_data() and requests fresh upload URLs for retry attempts. Changes: - Added ServiceErrorDuringUpload exception class - Convert ServiceError to ServiceErrorDuringUpload in b2http.py - Fixed RawSimulator to generate unique upload tokens per URL - Added comprehensive test for token refresh on 503 errors Fixes Backblaze/B2_Command_Line_Tool#1118
1 parent 1fff8c2 commit 3537f1f

7 files changed

Lines changed: 148 additions & 3 deletions

File tree

b2sdk/_internal/b2http.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
ClockSkew,
4343
ConnectionReset,
4444
PotentialS3EndpointPassedAsRealm,
45+
ServiceError,
46+
ServiceErrorDuringUpload,
4547
UnknownError,
4648
UnknownHost,
4749
interpret_b2_error,
@@ -363,6 +365,11 @@ def post_content_return_json(
363365
# this forces a token refresh, which is necessary if request is still alive
364366
# on the server but has terminated for some reason on the client. See #79
365367
raise B2RequestTimeoutDuringUpload()
368+
except ServiceError as e:
369+
# Convert ServiceError to ServiceErrorDuringUpload for upload operations.
370+
# This disables HTTP-level retries and forces the upload manager to clear
371+
# cached upload tokens and request fresh ones.
372+
raise ServiceErrorDuringUpload(str(e))
366373

367374
def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER):
368375
"""

b2sdk/_internal/exception.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,23 @@ class ServiceError(TransientErrorMixin, B2Error):
434434
"""
435435

436436

437+
class ServiceErrorDuringUpload(ServiceError):
438+
"""
439+
A :py:class:`b2sdk.v3.exception.ServiceError` that occurred during an upload operation.
440+
441+
This exception disables HTTP-level retries to force the error to propagate
442+
back to the upload manager, which will clear cached upload tokens and request
443+
fresh ones. This is necessary because 5xx errors may indicate the upload URL
444+
is no longer valid (typically '503 Service Unavailable').
445+
446+
Similar to :py:class:`b2sdk.v3.exception.B2RequestTimeoutDuringUpload`, this ensures upload token refresh
447+
happens when the server is experiencing issues.
448+
"""
449+
450+
def should_retry_http(self):
451+
return False
452+
453+
437454
class CapExceeded(B2Error):
438455
def __str__(self):
439456
return 'Cap exceeded.'

b2sdk/_internal/raw_simulator.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,9 @@ def _check_capability(self, account_auth_token, capability):
624624
# fortunately BucketSimulator makes it easy to retrieve the true account_auth_token
625625
# from an upload url
626626
real_auth_token = account_auth_token.split('/')[-1]
627+
# Strip the _upload_<id> suffix if present to get the base account auth token
628+
if '_upload_' in real_auth_token:
629+
real_auth_token = real_auth_token.split('_upload_')[0]
627630
key = self.api.auth_token_to_key[real_auth_token]
628631
capabilities = key.get_allowed()['capabilities']
629632
return capability in capabilities
@@ -773,10 +776,13 @@ def get_file_info_by_name(self, account_auth_token, file_name):
773776

774777
def get_upload_url(self, account_auth_token):
775778
upload_id = next(self.upload_url_counter)
779+
# Generate a unique upload authorization token by appending upload_id to account token
780+
# This ensures each upload URL has a different auth token, matching real B2 API behavior
781+
unique_upload_token = '%s_upload_%d' % (account_auth_token, upload_id)
776782
upload_url = 'https://upload.example.com/%s/%d/%s' % (
777783
self.bucket_id,
778784
upload_id,
779-
account_auth_token,
785+
unique_upload_token,
780786
)
781787
return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url)
782788

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Refresh upload tokens faster when server responds with 5xx during an upload attempt. Fixes [B2_Command_Line_Tool#1118](https://github.com/Backblaze/B2_Command_Line_Tool/issues/1118)

doc/sqlite_account_info_schema.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
# License https://www.backblaze.com/using_b2_code.html
88
#
99
######################################################################
10-
""" generates a dot file with SqliteAccountInfo database structure """
10+
"""generates a dot file with SqliteAccountInfo database structure"""
11+
1112
from __future__ import annotations
1213

1314
import tempfile

test/integration/conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99
######################################################################
1010

1111

12-
pytest_plugins = ["b2sdk.v3.testing"]
12+
pytest_plugins = ['b2sdk.v3.testing']
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
######################################################################
2+
#
3+
# File: test/unit/test_upload_503_token_refresh.py
4+
#
5+
# Copyright 2025 Backblaze Inc. All Rights Reserved.
6+
#
7+
# License https://www.backblaze.com/using_b2_code.html
8+
#
9+
######################################################################
10+
"""
11+
Test to verify that upload tokens are refreshed when a 503 error is returned.
12+
This test simulates the scenario where a server returns a 503 Service Unavailable
13+
error during file upload and verifies that the SDK properly requests new upload
14+
tokens for subsequent retry attempts.
15+
"""
16+
17+
from __future__ import annotations
18+
19+
from unittest.mock import patch
20+
21+
import pytest
22+
23+
from b2sdk._internal.exception import ServiceError
24+
from b2sdk.v3 import RawSimulator
25+
from b2sdk.v3 import B2Api, B2HttpApiConfig, DummyCache, StubAccountInfo
26+
27+
from .test_base import TestBase
28+
29+
30+
class TestUpload503TokenRefresh(TestBase):
31+
"""Test that 503 errors trigger upload token refresh using RawSimulator"""
32+
33+
def setUp(self):
34+
self.bucket_name = 'test-bucket'
35+
self.account_info = StubAccountInfo()
36+
self.api = B2Api(
37+
self.account_info,
38+
cache=DummyCache(),
39+
api_config=B2HttpApiConfig(_raw_api_class=RawSimulator),
40+
)
41+
self.simulator = self.api.session.raw_api
42+
(self.account_id, self.master_key) = self.simulator.create_account()
43+
self.api.authorize_account(
44+
application_key_id=self.account_id,
45+
application_key=self.master_key,
46+
realm='production',
47+
)
48+
self.bucket = self.api.create_bucket(self.bucket_name, 'allPublic')
49+
50+
def test_upload_503_triggers_token_refresh(self):
51+
"""
52+
Test that when a 503 error occurs during upload, the SDK:
53+
1. Retries the upload
54+
2. Uses a different upload token for the retry
55+
"""
56+
# Track upload URLs/tokens used during upload attempts
57+
upload_urls_used = []
58+
59+
# Wrap the upload_file method to track upload URLs
60+
original_upload_file = self.simulator.upload_file
61+
62+
def tracked_upload_file(upload_url, *args, **kwargs):
63+
upload_urls_used.append(upload_url)
64+
return original_upload_file(upload_url, *args, **kwargs)
65+
66+
# Inject a 503 error for the first upload attempt
67+
self.simulator.set_upload_errors(
68+
[ServiceError('503 service_unavailable Service Unavailable')]
69+
)
70+
71+
# Patch upload_file to track URLs
72+
with patch.object(self.simulator, 'upload_file', side_effect=tracked_upload_file):
73+
# Perform upload - should fail once with 503, then retry and succeed
74+
data = b'test data for 503 error scenario'
75+
file_name = 'test_503.txt'
76+
77+
self.bucket.upload_bytes(data, file_name)
78+
79+
# Verify the upload succeeded
80+
file_info = self.bucket.get_file_info_by_name(file_name)
81+
assert file_info is not None
82+
83+
# Verify that at least 2 upload attempts were made
84+
assert (
85+
len(upload_urls_used) >= 2
86+
), f'Expected at least 2 upload attempts, but got {len(upload_urls_used)}'
87+
88+
# Extract auth tokens from the URLs
89+
# URL format: https://upload.example.com/bucket_id/upload_id/auth_token
90+
first_url = upload_urls_used[0]
91+
second_url = upload_urls_used[1]
92+
93+
first_auth_token = first_url.split('/')[-1]
94+
second_auth_token = second_url.split('/')[-1]
95+
96+
print('\n✓ Upload token refresh test results:')
97+
print(f' First URL: {first_url}')
98+
print(f' Second URL: {second_url}')
99+
print(f' First auth token: {first_auth_token}')
100+
print(f' Second auth token: {second_auth_token}')
101+
print(f' Total upload attempts: {len(upload_urls_used)}')
102+
103+
# Verify that auth tokens are different after a 503 error.
104+
# This confirms that the SDK properly clears cached upload tokens
105+
# and requests fresh ones when a 503 Service Error occurs.
106+
assert first_auth_token != second_auth_token, (
107+
f'BUG: Auth tokens are the same after 503 error!\n'
108+
f'Expected different auth tokens, but got:\n'
109+
f' First: {first_auth_token}\n'
110+
f' Second: {second_auth_token}\n'
111+
f'This indicates the SDK is cycling through pre-fetched upload URLs\n'
112+
f'instead of requesting fresh upload tokens after a 503 error.'
113+
)

0 commit comments

Comments
 (0)