Skip to content

Commit 8075ebb

Browse files
authored
Merge pull request #204 from HENNGE/health-monitoring
Basic health monitoring system
2 parents 2c68cf4 + cd0ad6a commit 8075ebb

6 files changed

Lines changed: 315 additions & 15 deletions

File tree

docs/advanced.rst

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,58 @@ which conforms to the :py:class:`aiodynamo.credentials.Credentials` interface.
5656

5757
.. autoclass:: aiodynamo.credentials.Credentials
5858
:members: get_key,invalidate,is_disabled
59+
60+
61+
Health Monitoring
62+
-----------------
63+
64+
.. versionadded:: 25.2
65+
66+
When running code on AWS, you sometimes end up with unhealthy instances that can not communicate
67+
with DynamoDB anymore. Ideally, you can detect this and use AWS health monitoring systems to automatically
68+
remove the unhealthy instance (or perform other actions that are appropriate for your situation).
69+
70+
To facilitate this, aiodynamo has basic support for health monitoring of a client. This feature is disabled
71+
by default. To enable it, pass an instance of a class conforming to the :py:class:`aiodynamo.health.HealthMonitor`
72+
protocol to the client.
73+
74+
Some simple monitoring strategies are provided built in, but you can implement your own strategy to better suit your needs.
75+
76+
.. autoclass:: aiodynamo.health.HealthMonitor
77+
:members: is_healthy, on_exception, on_success
78+
79+
80+
.. autoclass:: aiodynamo.health.CountingHealthMonitor
81+
82+
83+
.. py:class:: aiodynamo.health.OnSuccess
84+
85+
.. py:attribute:: decrement
86+
87+
Decrement the failure count by one on success.
88+
89+
.. py:attribute:: reset
90+
91+
Reset the failure count on success.
92+
93+
.. py:attribute:: noop
94+
95+
Take no action on success.
96+
97+
.. autoclass:: aiodynamo.health.CallbackHealthMonitor
98+
99+
.. py:class:: aiodynamo.health.CallStrategy
100+
101+
.. py:attribute:: edge
102+
103+
Call the callback once when transitioning from healthy to unhealthy.
104+
105+
.. py:attribute:: always
106+
107+
Always call the callback if an event happens and the inner monitor is unhealthy.
108+
109+
.. py:attribute:: once
110+
111+
Call the callback once when the inner monitor goes unhealthy.
112+
113+
.. autoclass:: aiodynamo.health.NoOpHealthMonitor

docs/changelog.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
Changelog
22
=========
33

4+
25.2
5+
----
6+
7+
* Add health monitoring support.
8+
49
24.7
510
----
611

docs/usage.rst

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ Client instantiation
77
You should try to re-use clients, and especially HTTP clients, as much as possible. Don't create a new one for each
88
action.
99

10-
The :py:class:`aiodynamo.client.Client` class takes three required and three optional arguments:
10+
The :py:class:`aiodynamo.client.Client` class takes three required and four optional arguments:
1111

1212
1. An HTTP client adaptor, conforming to the :py:class:`aiodynamo.http.base.HTTP` interface.
1313
2. An instance of :py:class:`aiodynamo.credentials.Credentials` to authenticate with DynamoDB. You may use
@@ -16,6 +16,7 @@ The :py:class:`aiodynamo.client.Client` class takes three required and three opt
1616
4. An optional endpoint URL of your DynamoDB, as a :py:class:`yarl.URL` instance. Useful when using a local DynamoDB implementation such as dynalite or dynamodb-local.
1717
5. Which numeric type to use. This should be a callable which accepts a string as input and returns your numeric type as output. Defaults to ``float``.
1818
6. The throttling configuration to use. An instance of :py:class:`aiodynamo.models.RetryConfig`. By default, if the DynamoDB rate limit is exceeded, aiodynamo will retry up for up to one minute with increasing delays.
19+
7. A :py:class:`aiodynamo.health.HealthMonitor` instance. By default, health monitoring is disabled.
1920

2021
Credentials
2122
-----------

src/aiodynamo/client.py

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
ProjectionExpression,
4646
UpdateExpression,
4747
)
48+
from .health import HealthMonitor, NoOpHealthMonitor
4849
from .http.types import HttpImplementation, Request, RequestFailed
4950
from .models import (
5051
BatchGetRequest,
@@ -368,6 +369,7 @@ class Client:
368369
endpoint: Optional[URL] = None
369370
numeric_type: NumericTypeConverter = float
370371
throttle_config: RetryConfig = RetryConfig.default()
372+
health_monitor: HealthMonitor = NoOpHealthMonitor()
371373

372374
def table(self, name: str) -> Table:
373375
return Table(self, name)
@@ -997,35 +999,45 @@ async def send_request(
997999
except asyncio.TimeoutError as exc:
9981000
logger.debug("http timeout")
9991001
exception = exc
1002+
self.health_monitor.on_exception(exception)
10001003
continue
10011004
except RequestFailed as exc:
10021005
logger.debug("request failed")
10031006
exception = exc.inner
1007+
self.health_monitor.on_exception(exception)
10041008
continue
1009+
except Exception as exc:
1010+
self.health_monitor.on_exception(exc)
1011+
raise
10051012
response_logger.debug("got response %r", response)
10061013
if response.status == 200:
1014+
self.health_monitor.on_success()
10071015
return cast(Dict[str, Any], json.loads(response.body))
10081016
exception = exception_from_response(response.status, response.body)
1009-
if isinstance(exception, Throttled):
1010-
logger.debug("request throttled")
1011-
elif isinstance(exception, ProvisionedThroughputExceeded):
1012-
logger.debug("provisioned throughput exceeded")
1013-
elif isinstance(exception, ExpiredToken):
1014-
logger.debug("token expired")
1015-
if not self.credentials.invalidate():
1016-
raise exception
1017-
elif isinstance(exception, ServiceUnavailable):
1018-
logger.debug("service unavailable")
1019-
elif isinstance(exception, InternalDynamoError):
1020-
logger.debug("internal dynamo error")
1021-
else:
1022-
raise exception
1017+
self.handle_exception(exception)
10231018
except RetryTimeout:
10241019
if exception is not None:
10251020
raise exception
10261021
raise
10271022
raise BrokenThrottleConfig()
10281023

1024+
def handle_exception(self, exception: Exception) -> None:
1025+
self.health_monitor.on_exception(exception)
1026+
if isinstance(exception, Throttled):
1027+
logger.debug("request throttled")
1028+
elif isinstance(exception, ProvisionedThroughputExceeded):
1029+
logger.debug("provisioned throughput exceeded")
1030+
elif isinstance(exception, ExpiredToken):
1031+
logger.debug("token expired")
1032+
if not self.credentials.invalidate():
1033+
raise exception
1034+
elif isinstance(exception, ServiceUnavailable):
1035+
logger.debug("service unavailable")
1036+
elif isinstance(exception, InternalDynamoError):
1037+
logger.debug("internal dynamo error")
1038+
else:
1039+
raise exception
1040+
10291041
async def _depaginate(
10301042
self, action: str, payload: Dict[str, Any], limit: Optional[int] = None
10311043
) -> AsyncIterator[Dict[str, Any]]:

src/aiodynamo/health.py

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
from dataclasses import dataclass, field
2+
from enum import Enum, auto
3+
from typing import Callable, Protocol
4+
5+
from aiodynamo.errors import AIODynamoError
6+
7+
8+
class HealthMonitor(Protocol):
9+
"""
10+
Protocol for health monitors that can be configured on clients.
11+
12+
These can be used to detect unhealthy instances on AWS that can no longer communicate with DynamoDB.
13+
"""
14+
15+
def is_healthy(self) -> bool:
16+
"""
17+
Returns whether the client should be considered healthy or not.
18+
"""
19+
20+
def on_exception(self, exc: Exception) -> None:
21+
"""
22+
A request sent to DynamoDB resulted in an exception.
23+
"""
24+
25+
def on_success(self) -> None:
26+
"""
27+
A request sent to DynamoDB succeeded.
28+
"""
29+
30+
31+
class OnSuccess(Enum):
32+
decrement = auto()
33+
reset = auto()
34+
noop = auto()
35+
36+
37+
@dataclass(kw_only=True)
38+
class CountingHealthMonitor:
39+
"""
40+
A simple health monitor that counts any exception (other than those defined in `ignore_exceptions`).
41+
Your system should monitor the `healthy` property to see if this client is considered healthy.
42+
Adjust the number of failures that need to happen before it gets marked as unhealthy using the
43+
`max_failures` property.
44+
`on_success_action` controls what should happen when a successful request finished.
45+
"""
46+
47+
max_failures: int = 5
48+
ignore_exceptions: tuple[type[Exception]] = (AIODynamoError,)
49+
on_success_action: OnSuccess = OnSuccess.decrement
50+
_count: int = field(init=False, default=0)
51+
52+
def is_healthy(self) -> bool:
53+
return self._count < self.max_failures
54+
55+
def on_exception(self, exc: Exception) -> None:
56+
if isinstance(exc, self.ignore_exceptions):
57+
return
58+
self._count += 1
59+
60+
def on_success(self) -> None:
61+
match self.on_success_action:
62+
case OnSuccess.decrement:
63+
self._count = max(self._count - 1, 0)
64+
case OnSuccess.reset:
65+
self._count = 0
66+
case OnSuccess.noop:
67+
pass
68+
69+
70+
class CallStrategy(Enum):
71+
edge = auto()
72+
always = auto()
73+
once = auto()
74+
75+
76+
@dataclass
77+
class CallbackHealthMonitor:
78+
"""
79+
A health monitor wrapping another health monitor and calling the provided callback according to the
80+
`callback_strategy`. The callback takes no arguments and its return value is ignored.
81+
"""
82+
83+
inner: HealthMonitor
84+
callback: Callable[[], None]
85+
call_strategy: CallStrategy = CallStrategy.edge
86+
_was_healthy: bool = field(init=False, default=True)
87+
_did_call: bool = field(init=False, default=False)
88+
89+
def is_healthy(self) -> bool:
90+
return self.inner.is_healthy()
91+
92+
def _call(self) -> None:
93+
self.callback()
94+
self._did_call = True
95+
96+
def on_exception(self, exc: Exception) -> None:
97+
self.inner.on_exception(exc)
98+
healthy = self.is_healthy()
99+
if not healthy:
100+
match self.call_strategy:
101+
case CallStrategy.edge:
102+
if self._was_healthy:
103+
self._call()
104+
case CallStrategy.always:
105+
self._call()
106+
case CallStrategy.once:
107+
if not self._did_call:
108+
self._call()
109+
self._was_healthy = healthy
110+
111+
def on_success(self) -> None:
112+
self.inner.on_success()
113+
self._was_healthy = self.is_healthy()
114+
115+
116+
class NoOpHealthMonitor:
117+
"""
118+
Noop health monitor. Doesn't do anything and always reports as healthy.
119+
"""
120+
121+
def is_healthy(self) -> bool:
122+
return True
123+
124+
def on_exception(self, exc: Exception) -> None:
125+
pass
126+
127+
def on_success(self) -> None:
128+
pass

tests/unit/test_health_monitor.py

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
from itertools import cycle
2+
from typing import Iterable
3+
4+
import pytest
5+
6+
from aiodynamo.client import Client
7+
from aiodynamo.credentials import Key, StaticCredentials
8+
from aiodynamo.health import (
9+
CallbackHealthMonitor,
10+
CallStrategy,
11+
CountingHealthMonitor,
12+
OnSuccess,
13+
)
14+
from aiodynamo.http.types import HttpImplementation, Request, Response
15+
from aiodynamo.models import RetryConfig
16+
from aiodynamo.types import Seconds
17+
18+
19+
def http(responses: Iterable[Response | Exception]) -> HttpImplementation:
20+
it = iter(responses)
21+
22+
async def impl(_request: Request) -> Response:
23+
response = next(it)
24+
if isinstance(response, Exception):
25+
raise response
26+
return response
27+
28+
return impl
29+
30+
31+
class Once(RetryConfig):
32+
def delays(self) -> Iterable[Seconds]:
33+
yield 0
34+
35+
36+
@pytest.mark.parametrize(
37+
"strategy,first,second",
38+
[
39+
(OnSuccess.noop, False, False),
40+
(OnSuccess.decrement, False, True),
41+
(OnSuccess.reset, True, True),
42+
],
43+
)
44+
async def test_counting_health_monitor(
45+
strategy: OnSuccess, first: bool, second: bool
46+
) -> None:
47+
class Error(Exception):
48+
pass
49+
50+
monitor = CountingHealthMonitor(max_failures=2, on_success_action=strategy)
51+
client = Client(
52+
http(cycle([Error()])),
53+
credentials=StaticCredentials(Key("a", "b")),
54+
region="test",
55+
throttle_config=Once(),
56+
health_monitor=monitor,
57+
)
58+
assert monitor.is_healthy()
59+
with pytest.raises(Error):
60+
await client.describe_table("table")
61+
assert monitor.is_healthy()
62+
with pytest.raises(Error):
63+
await client.describe_table("table")
64+
assert not monitor.is_healthy()
65+
with pytest.raises(Error):
66+
await client.describe_table("table")
67+
monitor.on_success()
68+
assert monitor.is_healthy() is first
69+
monitor.on_success()
70+
assert monitor.is_healthy() is second
71+
72+
73+
@pytest.mark.parametrize(
74+
"strategy,expected",
75+
[
76+
(CallStrategy.edge, 2),
77+
(CallStrategy.always, 3),
78+
(CallStrategy.once, 1),
79+
],
80+
)
81+
async def test_callback_health_monitor(strategy: CallStrategy, expected: int) -> None:
82+
calls = 0
83+
84+
def callback() -> None:
85+
nonlocal calls
86+
calls += 1
87+
88+
monitor = CallbackHealthMonitor(
89+
inner=CountingHealthMonitor(max_failures=1, on_success_action=OnSuccess.reset),
90+
callback=callback,
91+
call_strategy=strategy,
92+
)
93+
error = Exception()
94+
monitor.on_exception(error)
95+
assert calls == 1
96+
monitor.on_exception(error)
97+
monitor.on_success()
98+
monitor.on_exception(error)
99+
assert calls == expected

0 commit comments

Comments
 (0)