Feat/extraction new source bundestag dip client

.github/workflows/ci.yml

@@ -36,4 +36,4 @@ jobs:
         run: uv run pre-commit run --all-files
 
       - name: Run tests
-        run: uv run pytest tests
+        run: uv run pytest tests --ignore=tests/e2e

.gitignore

@@ -11,6 +11,7 @@ build/**/logs
 configurations/*
 !configurations/configuration.default.json
 !configurations/configuration.local.json
+!configurations/configuration.test*.json
 data/
 tmp/
 # Byte-compiled / optimized / DLL files

configurations/configuration.test-bundestag-combined.json

@@ -0,0 +1,37 @@
+{
+  "extraction": {
+    "orchestrator_name": "basic",
+    "datasources": [
+      {
+        "name": "bundestag",
+        "include_bundestag_mine": true,
+        "include_dip": true,
+        "dip_wahlperiode": 21,
+        "dip_sources": ["protocols"],
+        "export_limit": 2
+      }
+    ]
+  },
+  "embedding": {
+    "orchestrator_name": "basic",
+    "embedder_name": "basic",
+    "embedding_model": {
+      "provider": "hugging_face",
+      "name": "intfloat/multilingual-e5-small",
+      "tokenizer_name": "intfloat/multilingual-e5-small",
+      "batch_size": 8,
+      "device": "cpu",
+      "splitter": {
+        "chunk_overlap_in_tokens": 50,
+        "chunk_size_in_tokens": 384
+      }
+    },
+    "vector_store": {
+      "name": "pgvector",
+      "database_name": "rag-local",
+      "collection_name": "e2e_test_bundestag_combined",
+      "host": "localhost",
+      "port": 5433
+    }
+  }
+}

configurations/configuration.test-bundestag-dip.json

@@ -0,0 +1,37 @@
+{
+  "extraction": {
+    "orchestrator_name": "basic",
+    "datasources": [
+      {
+        "name": "bundestag",
+        "include_bundestag_mine": false,
+        "include_dip": true,
+        "dip_wahlperiode": 21,
+        "dip_sources": ["protocols"],
+        "export_limit": 2
+      }
+    ]
+  },
+  "embedding": {
+    "orchestrator_name": "basic",
+    "embedder_name": "basic",
+    "embedding_model": {
+      "provider": "hugging_face",
+      "name": "intfloat/multilingual-e5-small",
+      "tokenizer_name": "intfloat/multilingual-e5-small",
+      "batch_size": 8,
+      "device": "cpu",
+      "splitter": {
+        "chunk_overlap_in_tokens": 50,
+        "chunk_size_in_tokens": 384
+      }
+    },
+    "vector_store": {
+      "name": "pgvector",
+      "database_name": "rag-local",
+      "collection_name": "e2e_test_bundestag_dip",
+      "host": "localhost",
+      "port": 5433
+    }
+  }
+}

configurations/configuration.test-bundestag-mine.json

@@ -0,0 +1,35 @@
+{
+  "extraction": {
+    "orchestrator_name": "basic",
+    "datasources": [
+      {
+        "name": "bundestag",
+        "include_bundestag_mine": true,
+        "include_dip": false,
+        "export_limit": 5
+      }
+    ]
+  },
+  "embedding": {
+    "orchestrator_name": "basic",
+    "embedder_name": "basic",
+    "embedding_model": {
+      "provider": "hugging_face",
+      "name": "intfloat/multilingual-e5-small",
+      "tokenizer_name": "intfloat/multilingual-e5-small",
+      "batch_size": 8,
+      "device": "cpu",
+      "splitter": {
+        "chunk_overlap_in_tokens": 50,
+        "chunk_size_in_tokens": 384
+      }
+    },
+    "vector_store": {
+      "name": "pgvector",
+      "database_name": "rag-local",
+      "collection_name": "e2e_test_bundestag_mine",
+      "host": "localhost",
+      "port": 5433
+    }
+  }
+}