version 1.6.1

Author: gmolveau-anssi

.devcontainer/.env.destination

@@ -1,7 +1,7 @@
 DB_HOST="db-destination"
 PORT=8001
 EURYDICE_API=destination
-ALLOWED_HOSTS=destination.localhost,backend-destination
+ALLOWED_HOSTS=destination.test,backend-destination
 CSRF_TRUSTED_ORIGINS=""
 DJANGO_MANAGEPY_MIGRATE="on"
 METRICS_SLIDING_WINDOW=${METRICS_SLIDING_WINDOW:-60min}

.devcontainer/.env.origin

@@ -2,7 +2,7 @@ DB_HOST=db-origin
 TRANSFERABLE_MAX_SIZE="54975581388800"
 PORT=8000
 EURYDICE_API=origin
-ALLOWED_HOSTS="origin.localhost,backend-origin"
+ALLOWED_HOSTS="origin.test,backend-origin"
 CSRF_TRUSTED_ORIGINS=""
 DJANGO_MANAGEPY_MIGRATE="on"
 METRICS_SLIDING_WINDOW=${METRICS_SLIDING_WINDOW:-60min}

.devcontainer/Dockerfile

@@ -1,6 +1,6 @@
 ARG REGISTRY="docker.io/"
 ARG DEBIAN_RELEASE="bullseye"
-ARG PYTHON_VERSION="3.10"
+ARG PYTHON_VERSION="3.12"
 
 FROM ${REGISTRY}python:${PYTHON_VERSION}-${DEBIAN_RELEASE} AS devcontainer
 

.devcontainer/compose.devcontainer.yml

@@ -12,28 +12,6 @@ services:
     volumes:
       - "../:/home/abc/workdir/"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
-    labels:
-      traefik.enable: "true"
-      # origin backend
-      traefik.http.routers.backend_origin.rule: Host(`origin.localhost`) && PathPrefix(`/api`, `/admin`, `/static`)
-      traefik.http.routers.backend_origin.entrypoints: web
-      traefik.http.routers.backend_origin.service: backend_origin_service
-      traefik.http.services.backend_origin_service.loadbalancer.server.port: 8000
-      # origin frontend
-      traefik.http.routers.frontend_origin.rule: Host(`origin.localhost`)
-      traefik.http.routers.frontend_origin.entrypoints: web
-      traefik.http.routers.frontend_origin.service: frontend_origin_service
-      traefik.http.services.frontend_origin_service.loadbalancer.server.port: 8080
-      # destination backend
-      traefik.http.routers.backend_destination.rule: Host(`destination.localhost`) && PathPrefix(`/api`, `/admin`, `/static`)
-      traefik.http.routers.backend_destination.entrypoints: web
-      traefik.http.routers.backend_destination.service: backend_destination_service
-      traefik.http.services.backend_destination_service.loadbalancer.server.port: 8001
-      # destination frontend
-      traefik.http.routers.frontend_destination.rule: Host(`destination.localhost`)
-      traefik.http.routers.frontend_destination.entrypoints: web
-      traefik.http.routers.frontend_destination.service: frontend_destination_service
-      traefik.http.services.frontend_destination_service.loadbalancer.server.port: 8081
     extra_hosts:
       - "lidis.docker.host:${LIDIS_DOCKER_HOST:-host-gateway}"
     networks:

.devcontainer/compose.override.yml

@@ -0,0 +1,6 @@
+services:
+  apache:
+    build:
+      context: ../apache2
+      target: devcontainer
+    image: eurydice/apache:devcontainer

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Eurydice",
-  "dockerComposeFile": ["compose.devcontainer.yml"],
+  "dockerComposeFile": ["compose.devcontainer.yml", "compose.override.yml"],
   "service": "devcontainer",
   "initializeCommand": "bash .devcontainer/init.sh",
   "postCreateCommand": "bash .devcontainer/setup.sh",
@@ -28,6 +28,7 @@
       },
       "extensions": [
         "davidanson.vscode-markdownlint",
+        "charliermarsh.ruff",
         "dbaeumer.vscode-eslint",
         "editorconfig.editorconfig",
         "esbenp.prettier-vscode",
@@ -36,7 +37,6 @@
         "gruntfuggly.todo-tree",
         "iulian-radu-at.vscode-tasks-sidebar",
         "ms-azuretools.vscode-containers",
-        "ms-python.black-formatter",
         "ms-python.mypy-type-checker",
         "ms-python.python",
         "ms-python.vscode-pylance",

.env.dev.example

@@ -7,7 +7,7 @@ LOCAL_DOCKER_REGISTRY=docker.io
 REMOTE_DOCKER_REGISTRY=docker.io
 EURYDICE_VERSION=dev
 LIDI_VERSION=1.3.5
-ELK_VERSION=7.17.11
+ELK_VERSION=8.18.3
 
 # Backend configuration
 GUNICORN_CONFIGURATION=
@@ -18,6 +18,11 @@ USER_ASSOCIATION_TOKEN_SECRET_KEY=user_association_token_secret_key
 USER_ASSOCIATION_TOKEN_EXPIRES_AFTER=30min
 METRICS_SLIDING_WINDOW=60min
 
+# Encryption configuration
+ENCRYPTION_ENABLED=false
+PUBKEY_PATH=./data/keys/eurydice.pub
+PRIVKEY_PATH=./data/keys/eurydice
+
 # Autentication configuration
 REMOTE_USER_HEADER_AUTHENTICATION_ENABLED=true
 REMOTE_USER_HEADER=HTTP_X_REMOTE_USER
@@ -35,14 +40,16 @@ BACKEND_HOSTNAMES=localhost
 # Server configuration
 HTTP_SERVICES_BIND_HOST=127.0.0.1
 
-# Lidi configuration
-LIDIR_PORT=6000
-LIDIS_PORT=5000
-LIDIR_NB_DECODING_THREADS=2
+# Lidi common configuration
+LIDI_ENCODING_BLOCK_SIZE=600000
+LIDI_REPAIR_BLOCK_SIZE=30000
 LIDI_UDP_MTU=1500
-LIDI_ENCODING_BLOCK_SIZE=60000
-LIDI_REPAIR_BLOCK_SIZE=6000
+# Lidi sender configuration
 LIDIS_NB_ENCODING_THREADS=2
+LIDIS_PORT=5000
+# Lidi receiver configuration
+LIDIR_NB_DECODING_THREADS=2
+LIDIR_PORT=6000
 
 # Network configuration
 EURYDICE_IPV4_SUBNET=172.42.0.0/24
@@ -53,9 +60,13 @@ LOG_LEVEL=INFO
 RUST_LOG=INFO
 
 # File remover configuration
-FILE_REMOVER_RUN_EVERY=1min
-FILE_REMOVER_EXPIRE_TRANSFERABLES_AFTER=7days
-FILE_REMOVER_POLL_EVERY=200ms
+ORIGIN_FILE_REMOVER_RUN_EVERY=60min
+ORIGIN_FILE_REMOVER_EXPIRE_TRANSFERABLES_AFTER=7days
+ORIGIN_FILE_REMOVER_POLL_EVERY=200ms
+
+DESTINATION_FILE_REMOVER_RUN_EVERY=60min
+DESTINATION_FILE_REMOVER_EXPIRE_TRANSFERABLES_AFTER=7days
+DESTINATION_FILE_REMOVER_POLL_EVERY=200ms
 
 # DB trimmer configuration
 DBTRIMMER_RUN_EVERY=6h
@@ -67,6 +78,7 @@ FILEBEAT_CONFIG_PATH="./filebeat/filebeat.yml"
 ELASTICSEARCH_CERT_PATH="/etc/ssl/certs/ca-certificates.crt"
 ELASTICSEARCH_API_KEY=
 ELASTICSEARCH_HOSTS=elasticsearch:9200
+ELASTICSEARCH_INDEX_NAME=local-dev
 
 # Sender configuration
 SENDER_RANGE_FILLER_CLASS=UserRotatingTransferableRangeFiller
@@ -108,7 +120,6 @@ DESTINATION_DB_LOGS_DIR="./data/db-logs-destination"
 PYTHON_LOGS_DIR="./data/python-logs"
 FILEBEAT_LOGS_DIR="./data/filebeat-logs"
 FILEBEAT_DATA_DIR="./data/filebeat-data"
-HOST_TRANSFERABLE_STORAGE_DIR="./data/storage-data"
 
 # Transferable history
 TRANSFERABLE_HISTORY_DURATION=15min

.env.prod.example

@@ -4,8 +4,8 @@
 LOCAL_DOCKER_REGISTRY=docker.io
 REMOTE_DOCKER_REGISTRY=docker.io
 EURYDICE_VERSION=latest
-LIDI_VERSION=latest
-ELK_VERSION=7.17.11
+LIDI_VERSION=1.3.5
+ELK_VERSION=8.18.3
 
 # User configuration
 PUID=1000
@@ -15,11 +15,16 @@ PGID=1000
 GUNICORN_CONFIGURATION=
 CSRF_TRUSTED_ORIGINS=
 EURYDICE_TRANSFERABLE_MAX_SIZE=54975581388800
-DJANGO_SECRET_KEY=
-USER_ASSOCIATION_TOKEN_SECRET_KEY=
+DJANGO_SECRET_KEY=secretkey
+USER_ASSOCIATION_TOKEN_SECRET_KEY=token
 USER_ASSOCIATION_TOKEN_EXPIRES_AFTER=30min
 METRICS_SLIDING_WINDOW=60min
 
+# Encryption configuration
+ENCRYPTION_ENABLED=true
+PUBKEY_PATH=./data/keys/eurydice.pub
+PRIVKEY_PATH=./data/keys/eurydice
+
 # Autentication configuration
 REMOTE_USER_HEADER_AUTHENTICATION_ENABLED=true
 REMOTE_USER_HEADER=HTTP_X_REMOTE_USER
@@ -37,14 +42,17 @@ BACKEND_HOSTNAMES=localhost
 # Server configuration
 HTTP_SERVICES_BIND_HOST=127.0.0.1
 
-# Lidi configuration
-LIDIR_PORT=11011
-LIDIS_PORT=33010
-LIDIR_NB_DECODING_THREADS=2
+# Lidi common configuration
+LIDI_ENCODING_BLOCK_SIZE=600000
+LIDI_REPAIR_BLOCK_SIZE=30000
 LIDI_UDP_MTU=1500
-LIDI_ENCODING_BLOCK_SIZE=60000
-LIDI_REPAIR_BLOCK_SIZE=6000
+# Lidi sender configuration
 LIDIS_NB_ENCODING_THREADS=2
+LIDIS_PORT=33010
+# Lidi receiver configuration
+LIDIR_NB_DECODING_THREADS=2
+LIDIR_PORT=11011
+LIDIR_HOST=
 
 # Network configuration
 EURYDICE_IPV4_SUBNET=172.18.0.0/24
@@ -55,7 +63,7 @@ LOG_LEVEL=INFO
 RUST_LOG=INFO
 
 # File remover configuration
-FILE_REMOVER_RUN_EVERY=1min
+FILE_REMOVER_RUN_EVERY=10min
 FILE_REMOVER_EXPIRE_TRANSFERABLES_AFTER=7days
 FILE_REMOVER_POLL_EVERY=200ms
 
@@ -69,6 +77,7 @@ FILEBEAT_CONFIG_PATH="./filebeat/filebeat.yml"
 ELASTICSEARCH_CERT_PATH="/etc/ssl/certs/ca-certificates.crt"
 ELASTICSEARCH_API_KEY=
 ELASTICSEARCH_HOSTS=
+ELASTICSEARCH_INDEX_NAME=prod
 
 # Sender configuration
 SENDER_RANGE_FILLER_CLASS=UserRotatingTransferableRangeFiller

.gitignore

@@ -102,3 +102,13 @@ sysctl_backup.conf
 !frontend/.env.*
 # devcontainer
 .devcontainer/customize.sh
+dump*.sql
+# encryption keypair
+eurydice
+!eurydice/
+eurydice.pub
+# dev deployment
+apache2/configs/dev-local-origin.conf
+apache2/configs/dev-local-destination.conf
+# prod deployment
+deployment/prod/config-ssl.conf

.vscode/settings.json

@@ -2,6 +2,7 @@
   // Configure formater
   "editor.formatOnSave": true,
   "prettier.enable": true,
+  "ruff.lint.enable": true,
   "[vue]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
@@ -17,7 +18,7 @@
   "[python]": {
     "editor.insertSpaces": true,
     "editor.tabSize": 4,
-    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.defaultFormatter": "charliermarsh.ruff",
     "editor.formatOnType": true,
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {