Skip to content

Commit 8bb0dd0

Browse files
gmolveau-anssigmolveau-anssi
committed
version 1.8.0
1 parent 384381f commit 8bb0dd0

30 files changed

Lines changed: 1050 additions & 1230 deletions

File tree

‎.devcontainer/.env.devcontainer‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ PATH="/home/abc/workdir/backend/.venv/bin:/home/abc/workdir/frontend/node_module
66
LOCAL_DOCKER_REGISTRY=docker.io
77
REMOTE_DOCKER_REGISTRY=docker.io
88
EURYDICE_VERSION=devcontainer
9-
LIDI_VERSION=1.3.5
10-
ELK_VERSION=7.17.11
9+
LIDI_VERSION=2.1.1
10+
ELK_VERSION=8.18.6
1111

1212
# Backend configuration
1313
GUNICORN_CONFIGURATION=
@@ -41,12 +41,13 @@ BACKEND_HOSTNAMES=localhost
4141
HTTP_SERVICES_BIND_HOST=127.0.0.1
4242

4343
# Lidi configuration
44+
LIDI_LOG_LEVEL=Info
45+
LIDI_BLOCK_SIZE=734928
46+
LIDI_REPAIR_PERCENTAGE=2
4447
LIDIR_PORT=6000
4548
LIDIS_PORT=5000
4649
LIDIR_NB_DECODING_THREADS=2
4750
LIDI_UDP_MTU=1500
48-
LIDI_ENCODING_BLOCK_SIZE=60000
49-
LIDI_REPAIR_BLOCK_SIZE=6000
5051
LIDIS_NB_ENCODING_THREADS=2
5152

5253
# Network configuration
@@ -55,7 +56,6 @@ EURYDICE_IPV4_SUBNET=172.42.0.0/24
5556
# Backend logging configuration
5657
LOG_TO_FILE=true
5758
LOG_LEVEL=INFO
58-
RUST_LOG=INFO
5959

6060
# File remover configuration
6161
FILE_REMOVER_RUN_EVERY=1min

‎.env.dev.example‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ PGID=CHANGE_ME
66
LOCAL_DOCKER_REGISTRY=docker.io
77
REMOTE_DOCKER_REGISTRY=docker.io
88
EURYDICE_VERSION=dev
9-
LIDI_VERSION=1.3.5
9+
LIDI_VERSION=2.1.1
1010
ELK_VERSION=8.18.3
1111

1212
# Backend configuration
@@ -41,14 +41,15 @@ BACKEND_HOSTNAMES=localhost
4141
HTTP_SERVICES_BIND_HOST=127.0.0.1
4242

4343
# Lidi common configuration
44-
LIDI_ENCODING_BLOCK_SIZE=600000
45-
LIDI_REPAIR_BLOCK_SIZE=30000
44+
LIDI_LOG_LEVEL=Info
45+
LIDI_BLOCK_SIZE=734928
46+
LIDI_REPAIR_PERCENTAGE=2
4647
LIDI_UDP_MTU=1500
4748
# Lidi sender configuration
48-
LIDIS_NB_ENCODING_THREADS=2
49+
LIDIS_ENCODE_THREADS=2
4950
LIDIS_PORT=5000
5051
# Lidi receiver configuration
51-
LIDIR_NB_DECODING_THREADS=2
52+
LIDIR_DECODE_THREADS=2
5253
LIDIR_PORT=6000
5354

5455
# Network configuration
@@ -57,7 +58,6 @@ EURYDICE_IPV4_SUBNET=172.42.0.0/24
5758
# Backend logging configuration
5859
LOG_TO_FILE=true
5960
LOG_LEVEL=INFO
60-
RUST_LOG=INFO
6161

6262
# File remover configuration
6363
ORIGIN_FILE_REMOVER_RUN_EVERY=60min

‎.env.prod.example‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
LOCAL_DOCKER_REGISTRY=docker.io
55
REMOTE_DOCKER_REGISTRY=docker.io
66
EURYDICE_VERSION=latest
7-
LIDI_VERSION=1.3.5
7+
LIDI_VERSION=2.1.1
88
ELK_VERSION=8.18.3
99

1010
# User configuration
@@ -43,24 +43,24 @@ BACKEND_HOSTNAMES=localhost
4343
HTTP_SERVICES_BIND_HOST=127.0.0.1
4444

4545
# Lidi common configuration
46-
LIDI_ENCODING_BLOCK_SIZE=600000
47-
LIDI_REPAIR_BLOCK_SIZE=30000
46+
LIDI_LOG_LEVEL=Info
47+
LIDI_BLOCK_SIZE=734928
48+
LIDI_REPAIR_PERCENTAGE=2
4849
LIDI_UDP_MTU=1500
4950
# Lidi sender configuration
50-
LIDIS_NB_ENCODING_THREADS=2
51+
LIDIS_ENCODE_THREADS=2
5152
LIDIS_PORT=33010
5253
# Lidi receiver configuration
53-
LIDIR_NB_DECODING_THREADS=2
54+
LIDIR_DECODE_THREADS=2
5455
LIDIR_PORT=11011
55-
LIDIR_HOST=
56+
LIDIR_HOST=172.16.16.2
5657

5758
# Network configuration
5859
EURYDICE_IPV4_SUBNET=172.18.0.0/24
5960

6061
# Backend logging configuration
6162
LOG_TO_FILE=true
6263
LOG_LEVEL=INFO
63-
RUST_LOG=INFO
6464

6565
# File remover configuration
6666
FILE_REMOVER_RUN_EVERY=10min

‎Makefile‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ COMPOSE := docker compose
99
COMPOSE_DEV := PUID=$(shell id -u) PGID=$(shell id -g) $(COMPOSE) -f compose.yml
1010
COMPOSE_PROD := $(COMPOSE) -f compose.prod.yml
1111

12-
SYSCTL_OPTIONS = net.core.rmem_max net.core.rmem_default net.core.netdev_max_backlog net.ipv4.udp_mem
12+
SYSCTL_OPTIONS = net.ipv4.udp_mem net.core.rmem_max net.ipv4.udp_rmem_min net.core.wmem_max net.ipv4.udp_wmem_min
1313
SYSCTL_BACKUP_FILE = sysctl_backup.conf
1414

1515
# ------------------------------------------------------------------------------
@@ -35,10 +35,11 @@ restore-sysctl: ## restore sysctl config from backup file
3535
config-sysctl: ## sysctl config of LIDI
3636
# https://github.com/ANSSI-FR/lidi/blob/master/doc/tweaking.rst
3737
$(MAKE) backup-sysctl
38-
sudo sysctl -w net.core.rmem_max=67108864 \
39-
-w net.core.rmem_default=67108864 \
40-
-w net.core.netdev_max_backlog=10000 \
41-
-w net.ipv4.udp_mem="12148128 16197504 24296256"
38+
sudo sysctl -w net.ipv4.udp_mem="97536000 97536000 97536000" \
39+
-w net.core.rmem_max=97536000 \
40+
-w net.ipv4.udp_rmem_min=97536000 \
41+
-w net.core.wmem_max=97536000 \
42+
-w net.ipv4.udp_wmem_min=97536000
4243

4344
.PHONY: config-ufw
4445
config-ufw: ## open sender and receiver firewall ports
@@ -261,7 +262,6 @@ prod-stop-destination-elk: ## stop the production destination stack
261262
hadolint: ## Lint the Dockerfiles.
262263
docker run --rm -i hadolint/hadolint:2.8.0-alpine < backend/docker/Dockerfile
263264
docker run --rm -i hadolint/hadolint:2.8.0-alpine < frontend/docker/Dockerfile
264-
docker run --rm -i hadolint/hadolint:2.8.0-alpine < pgadmin/Dockerfile
265265

266266
# ------------------------------------------------------------------------------
267267
# Tests

‎backend/eurydice/destination/api/docs/decorators.py‎

Lines changed: 16 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -207,39 +207,25 @@
207207
summary=_("Retrieve a transferable"),
208208
description=_((settings.DOCS_PATH / "download-transferable.md").read_text()),
209209
responses={
210-
(
211-
status.HTTP_200_OK,
212-
"application/octet-stream",
213-
): spectacular_utils.OpenApiResponse(
210+
status.HTTP_200_OK: spectacular_utils.OpenApiResponse(
214211
response=types.OpenApiTypes.BINARY,
215212
description=_("The transferable was successfully retrieved and is returned in the response body."),
216213
),
217-
(
218-
status.HTTP_401_UNAUTHORIZED,
219-
"application/json",
220-
): docs.NotAuthenticatedResponse,
221-
(
222-
status.HTTP_403_FORBIDDEN,
223-
"application/json",
224-
): docs.create_open_api_response(exceptions.TransferableErroredError),
225-
(
226-
status.HTTP_404_NOT_FOUND,
227-
"application/json",
228-
): docs.NotFoundResponse,
229-
(
230-
status.HTTP_409_CONFLICT,
231-
"application/json",
232-
): docs.create_open_api_response(exceptions.TransferableOngoingError),
233-
(
234-
status.HTTP_410_GONE,
235-
"application/json",
236-
): docs.create_open_api_response(exceptions.TransferableExpiredError),
237-
# Converting the status code to a string with an extra whitespace is a hack
238-
# to be able to have multiple responses for one status code
239-
(
240-
f"{status.HTTP_410_GONE} ",
241-
"application/json",
242-
): docs.create_open_api_response(exceptions.TransferableRevokedError),
214+
status.HTTP_401_UNAUTHORIZED: docs.NotAuthenticatedResponse,
215+
status.HTTP_403_FORBIDDEN: docs.create_open_api_response(exceptions.TransferableErroredError),
216+
status.HTTP_404_NOT_FOUND: docs.NotFoundResponse,
217+
status.HTTP_409_CONFLICT: docs.create_open_api_response(exceptions.TransferableOngoingError),
218+
status.HTTP_410_GONE: spectacular_utils.OpenApiResponse(
219+
response=spectacular_utils.PolymorphicProxySerializer(
220+
component_name="TransferableGoneErrors",
221+
serializers=[
222+
docs.create_api_exception_serializer(exceptions.TransferableExpiredError),
223+
docs.create_api_exception_serializer(exceptions.TransferableRevokedError),
224+
],
225+
resource_type_field_name=None,
226+
),
227+
description=_("The transferable is either expired or revoked."),
228+
),
243229
},
244230
parameters=[
245231
# Hide the format query parameter, user should not be able to use it

‎backend/eurydice/destination/receiver/packet_handler/extractors/transferable_range.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,14 @@ def _extract_transferable_range(transferable_range: protocol.TransferableRange)
255255
transferable_range: the transferable range to process.
256256
257257
"""
258-
logger.info({LOG_KEY: "extract_transferable_range", "transferable_id": str(transferable_range.transferable.id)})
258+
logger.info(
259+
{
260+
LOG_KEY: "extract_transferable_range",
261+
"transferable_id": str(transferable_range.transferable.id),
262+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
263+
"message": "Start extracting transferable range",
264+
}
265+
)
259266

260267
transferable = _get_or_create_transferable(transferable_range)
261268

@@ -264,7 +271,8 @@ def _extract_transferable_range(transferable_range: protocol.TransferableRange)
264271
{
265272
LOG_KEY: "extract_transferable_range",
266273
"transferable_id": str(transferable_range.transferable.id),
267-
"state": models.IncomingTransferableState.ERROR.value,
274+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
275+
"incoming_transferable_state": models.IncomingTransferableState.ERROR.value,
268276
"message": "Ignoring the associated transferable range received.",
269277
}
270278
)
@@ -282,6 +290,7 @@ def _extract_transferable_range(transferable_range: protocol.TransferableRange)
282290
{
283291
LOG_KEY: "extract_transferable_range",
284292
"transferable_id": str(transferable_range.transferable.id),
293+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
285294
"message": "Successfully extracted and ingested TransferableRange",
286295
}
287296
)
@@ -293,6 +302,7 @@ def _extract_transferable_range(transferable_range: protocol.TransferableRange)
293302
{
294303
LOG_KEY: "extract_transferable_range",
295304
"transferable_id": str(transferable_range.transferable.id),
305+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
296306
"state": models.IncomingTransferableState.SUCCESS.value,
297307
"message": "IncomingTransferable fully received",
298308
}
@@ -304,6 +314,7 @@ def _extract_transferable_range(transferable_range: protocol.TransferableRange)
304314
{
305315
LOG_KEY: "extract_transferable_range_failure",
306316
"transferable_id": str(transferable_range.transferable.id),
317+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
307318
"message": "Encountered an error when trying to extract and ingest "
308319
"transferable range from an OnTheWirePacket.",
309320
"error": str(error),

‎backend/eurydice/origin/api/docs/decorators.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,6 @@
1313
from eurydice.common.api.docs import utils as docs
1414
from eurydice.origin.api import exceptions, serializers
1515

16-
17-
def create_one_of_response(ref_names: list[str]) -> spectacular_utils.OpenApiResponse:
18-
return spectacular_utils.OpenApiResponse(
19-
description="One of the following errors occurred.",
20-
response={"oneOf": [{"$ref": f"#/components/schemas/{name}"} for name in ref_names]},
21-
)
22-
23-
2416
outgoing_transferable_example = {
2517
"id": "00002800-0000-1000-8000-00805f9b34fb",
2618
"created_at": "1969-12-28T14:15:22Z",

‎backend/eurydice/origin/sender/packet_generator/fillers/transferable_range.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,10 @@ def _add(
183183
LOG_KEY: "adding_transferable_range",
184184
"transferable_id": str(transferable_range.outgoing_transferable.id),
185185
"transferable_range_id": str(transferable_range.id),
186+
"transferable_range_byte_offset": str(transferable_range.byte_offset),
186187
}
187188
)
189+
188190
packet.transferable_ranges.append(protocol_transferable_range)
189191

190192
transferable_range.mark_as_transferred()

‎backend/pyproject.toml‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[project]
22
name = "eurydice"
3-
version = "1.6.1"
3+
version = "1.8.0"
44
requires-python = "~=3.12.0"
55
dependencies = [
6-
"gunicorn>=23.0",
6+
"gunicorn>=24.1.1",
77
"djangorestframework>=3.16.1",
88
"django-environ>=0.12.0",
99
"drf-spectacular>=0.29.0",
@@ -16,10 +16,9 @@ dependencies = [
1616
"pydantic>=2.12.5",
1717
"django-filter>=25.2",
1818
"jinja2>=3.1.6",
19-
"setuptools>=80.9",
2019
"django-stubs[compatible-mypy]>=5.2.8",
21-
"djangorestframework-stubs>=3.16.6",
22-
"pynacl>=1.6.1",
20+
"djangorestframework-stubs>=3.16.7",
21+
"pynacl>=1.6.2",
2322
]
2423

2524
[tool.uv]
@@ -36,18 +35,18 @@ dev = [
3635
"pytest>=9.0.2",
3736
"pytest-django>=4.11.1",
3837
"factory-boy>=3.3.3",
39-
"hypothesis>=6.148.7",
40-
"coverage>=7.13.0",
38+
"hypothesis>=6.150.2",
39+
"coverage>=7.13.1",
4140
"pytest-cov>=7.0.0",
42-
"ipython>=9.8.0",
41+
"ipython>=9.9.0",
4342
"ipdb>=0.13.13",
4443
"mypy-extensions>=1.1.0",
4544
"bandit>=1.9.2",
46-
"Faker>=38.2.0",
45+
"Faker>=40.1.2",
4746
"radon>=6.0.1",
4847
"freezegun>=1.5.5",
49-
"mypy>=1.19.0",
50-
"ruff>=0.14.8",
48+
"mypy>=1.19.1",
49+
"ruff>=0.14.13",
5150
]
5251

5352
[tool.coverage.run]

‎backend/tests/destination/integration/receiver/packet_handler/extractors/test_transferable_range.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,11 +180,17 @@ def test_transferable_range_transferable_already_marked_as_error(
180180
log_messages = process_logs(caplog.messages)
181181

182182
assert log_messages == [
183-
{"log_key": "extract_transferable_range", "transferable_id": str(incoming_transferable.id)},
184183
{
185184
"log_key": "extract_transferable_range",
186185
"transferable_id": str(incoming_transferable.id),
187-
"state": "ERROR",
186+
"message": "Start extracting transferable range",
187+
"transferable_range_byte_offset": "1024",
188+
},
189+
{
190+
"log_key": "extract_transferable_range",
191+
"transferable_id": str(incoming_transferable.id),
192+
"transferable_range_byte_offset": "1024",
193+
"incoming_transferable_state": "ERROR",
188194
"message": "Ignoring the associated transferable range received.",
189195
},
190196
]

0 commit comments

Comments
 (0)