Update readme and change naming in main

Author: timovdk

README.md

@@ -12,12 +12,6 @@ Requires Python 3.8 or later.
 pip install synergy-dataset
 ```
 
-For Pydantic-based work validation:
-
-```sh
-pip install "synergy-dataset[validation]"
-```
-
 ## Dataset variants
 
 | Variable | Value | Dataset |
@@ -56,7 +50,7 @@ Exports one CSV per dataset to the output folder, plus a `review_metadata.csv` t
 synergy get
 ```
 
-For SYNERGY+, only open-access works with a valid abstract (≥ 20 words or ≥ 100 characters) are exported. Datasets with fewer than 3 included records are skipped.
+For SYNERGY+, only open-access works with a valid abstract (≥ 20 words or ≥ 100 characters) are exported. Datasets with fewer than 5 included records are skipped.
 
 | Flag | Default | Description |
 |---|---|---|
@@ -102,7 +96,7 @@ referenced_works  related_works     counts_by_year
 Each run of `synergy get` produces:
 
 - **`{dataset_name}.csv`** — one file per dataset, with one row per work (filtered by the active settings).
-- **`review_metadata.csv`** — one row per dataset (≥ 3 inclusions), combining:
+- **`review_metadata.csv`** — one row per dataset (≥ 5 inclusions), combining:
   - `key` — dataset identifier (e.g. `Abgaz_2023`)
   - `split` — `train` or `test` (SYNERGY+ only)
   - `data_doi` — DOI of the dataset deposit

synergy_dataset/__main__.py

@@ -34,6 +34,8 @@
 
 Would you like to convert the inverted abstract to plaintext?"""
 
+MIN_INCLUSIONS = 5
+
 
 def main():
     if os.getenv("SYNERGY_PATH") == "development":
@@ -73,7 +75,7 @@ def info():
     parser.print_usage()
 
 
-def _count_filtered_records(dataset):
+def _count_records(dataset):
     n_records, n_included = 0, 0
     for _, label in dataset.iter(validate=False):
         n_records += 1
@@ -108,7 +110,7 @@ def _write_review_metadata(datasets, counts, active_vars, output_path):
         for dataset in datasets:
             n_records, n_included = counts[dataset.name]
 
-            if n_included < 3:
+            if n_included < MIN_INCLUSIONS:
                 continue
 
             pub_work = WorkModel.model_validate(dataset.metadata["publication"])
@@ -207,19 +209,19 @@ def build_dataset(argv):
         if args.dataset is not None:
             datasets = [Dataset(name) for name in args.dataset]
             for dataset in datasets:
-                n_records, n_included = _count_filtered_records(dataset)
+                n_records, n_included = _count_records(dataset)
                 counts[dataset.name] = (n_records, n_included)
-                if n_included < 3:
+                if n_included < MIN_INCLUSIONS:
                     continue
                 dataset.to_frame(args.vars).to_csv(
                     Path(args.output, f"{dataset.name}.csv"), index=False
                 )
         else:
             datasets = list(iter_datasets())
             for dataset in tqdm(datasets):
-                n_records, n_included = _count_filtered_records(dataset)
+                n_records, n_included = _count_records(dataset)
                 counts[dataset.name] = (n_records, n_included)
-                if n_included < 3:
+                if n_included < MIN_INCLUSIONS:
                     continue
                 dataset.to_frame(args.vars).to_csv(
                     Path(args.output, f"{dataset.name}.csv"), index=False