refactor(x): remove XTweetFileIngestionConfiguration and related pipeline batch size doc

- Removed XTweetFileIngestionConfiguration class and its references
- Updated batch_size docstring in XFileIngestionPipelineConfiguration for clarity

This cleans up unused ingestion configuration and improves documentation clarity.

Author: FlorentRvl

libs/naas-abi-marketplace/naas_abi_marketplace/applications/x/__init__.py

@@ -112,73 +112,6 @@ class XTweetSearchWorkflowConfiguration(BaseModel):
     )
 
 
-class XTweetFileIngestionConfiguration(BaseModel):
-    """One configured drop-folder that the XOrchestration polls for tweet
-    dataset files. Each entry produces its own Dagster job + sensor pair:
-    the sensor lists *input_prefix* in object storage every
-    ``interval_seconds``, and any object that hasn't already been ingested
-    (sha256-based dedupe in the graph) triggers a run that streams the
-    file through :class:`XFileIngestionPipeline`.
-
-    Designed for large datasets — the pipeline never buffers the whole
-    file in memory; NDJSON is read line by line and JSON arrays are
-    incrementally parsed via ijson, so a 12 GB dump uses constant memory.
-    """
-
-    name: str = Field(
-        description=(
-            "Short identifier (letters/digits/underscores) used to name "
-            "the generated Dagster job and sensor."
-        )
-    )
-    input_prefix: str = Field(
-        default="x/uploads",
-        description=(
-            "Object-storage prefix to watch. Drop tweet dump files under "
-            "this prefix and the sensor will pick them up on its next "
-            "tick. Accepted shapes (auto-detected): "
-            "(a) one big JSON array of bare tweet dicts — same as the "
-            "files that XIntegration.search_recent_tweets's cache writes "
-            "to ``<datastore_path>/search_recent_tweets/<hash>.json``, so "
-            "point this at that directory to back-fill from the cache; "
-            "(b) one X v2 ``{data:[...], meta:{...}}`` response — what "
-            "you'd get if you piped the API output to a file; "
-            "(c) JSONL of either shape — one tweet per line, or one "
-            "response envelope per line. ``.gz`` variants also work."
-        ),
-    )
-    interval_seconds: int = Field(
-        default=60,
-        ge=30,
-        description="Minimum delay between two sensor evaluations.",
-    )
-    batch_size: int = Field(
-        default=500,
-        ge=10,
-        le=10_000,
-        description=(
-            "Number of tweet records to accumulate before flushing as one "
-            "SPARQL INSERT into the named graph. Larger = fewer round "
-            "trips but more memory; smaller = lower memory ceiling."
-        ),
-    )
-    recursive: bool = Field(
-        default=False,
-        description=(
-            "If True, also pick up files in sub-prefixes under "
-            "input_prefix. Default off to mirror DocumentOrchestration."
-        ),
-    )
-    delete_after_ingest: bool = Field(
-        default=False,
-        description=(
-            "If True, delete the source object once ingestion succeeds. "
-            "Off by default so re-running stays a no-op (sha256 dedupe) "
-            "and the operator can inspect the original."
-        ),
-    )
-
-
 class ABIModule(BaseModule):
     dependencies: ModuleDependencies = ModuleDependencies(
         modules=[
@@ -231,7 +164,6 @@ class Configuration(ModuleConfiguration):
         ontology_namespace: str = "http://ontology.naas.ai/x/"
         graph_name: str = "http://ontology.naas.ai/graph/x"
         search_recent_tweets_workflow: list[XTweetSearchWorkflowConfiguration] = []
-        file_ingestion_pipeline: list[XTweetFileIngestionConfiguration] = []
 
     # on_initialized is called by the engine after all modules and services have been fully loaded.
     # At this point, you can safely access other modules and services through the engine's interfaces.

libs/naas-abi-marketplace/naas_abi_marketplace/applications/x/pipelines/XFileIngestionPipeline.py

@@ -100,8 +100,8 @@ class XFileIngestionPipelineConfiguration(PipelineConfiguration):
             :meth:`get_object_stream` (added to the port in this PR).
         triple_store: Target named graph + dedupe ASK destination.
         graph_name: Named graph IRI to insert into.
-        batch_size: Records per INSERT round trip — see
-            :class:`naas_abi_marketplace.applications.x.XTweetFileIngestionConfiguration`.
+        batch_size: Records per INSERT round trip. Larger = fewer round trips
+            but more memory; smaller = lower memory ceiling.
     """
 
     object_storage: IObjectStorageDomain