Merge branch 'develop' into main

Author: drorganvidez

app/modules/statistics/repositories.py

@@ -62,26 +62,61 @@ def get_feature_models_downloaded(self) -> int:
         statistics = self.get_statistics()
         return statistics.feature_models_downloaded
 
-    def refresh_statistics(self) -> Statistics:
+    def compute_totals(self) -> dict[str, int]:
+        """Re-compute every counter from the record tables without touching
+        the persisted row. Every total is restricted to datasets with a DOI
+        so the counters line up with what the `/statistics` dashboard shows.
+        """
         from app.modules.dataset.models import DataSet, DSMetaData
         from app.modules.featuremodel.models import FeatureModel
 
-        statistics = self.get_statistics()
-
         synchronized_dataset_ids = (
             self.session.query(DataSet.id).join(DSMetaData).filter(DSMetaData.dataset_doi.isnot(None)).scalar_subquery()
         )
-
-        statistics.datasets_counter = (
-            self.session.query(DataSet).join(DSMetaData).filter(DSMetaData.dataset_doi.isnot(None)).count()
+        synchronized_featuremodel_ids = (
+            self.session.query(FeatureModel.id)
+            .filter(FeatureModel.dataset_id.in_(synchronized_dataset_ids))
+            .scalar_subquery()
         )
-        statistics.feature_models_counter = (
-            self.session.query(FeatureModel).filter(FeatureModel.dataset_id.in_(synchronized_dataset_ids)).count()
+        from app.modules.hubfile.models import Hubfile
+
+        synchronized_hubfile_ids = (
+            self.session.query(Hubfile.id)
+            .filter(Hubfile.feature_model_id.in_(synchronized_featuremodel_ids))
+            .scalar_subquery()
         )
-        statistics.datasets_viewed = self.session.query(DSViewRecord).count()
-        statistics.feature_models_viewed = self.session.query(HubfileViewRecord).count()
-        statistics.datasets_downloaded = self.session.query(DSDownloadRecord).count()
-        statistics.feature_models_downloaded = self.session.query(HubfileDownloadRecord).count()
 
+        return {
+            "datasets_counter": (
+                self.session.query(DataSet).join(DSMetaData).filter(DSMetaData.dataset_doi.isnot(None)).count()
+            ),
+            "feature_models_counter": (
+                self.session.query(FeatureModel).filter(FeatureModel.dataset_id.in_(synchronized_dataset_ids)).count()
+            ),
+            "datasets_viewed": (
+                self.session.query(DSViewRecord).filter(DSViewRecord.dataset_id.in_(synchronized_dataset_ids)).count()
+            ),
+            "datasets_downloaded": (
+                self.session.query(DSDownloadRecord)
+                .filter(DSDownloadRecord.dataset_id.in_(synchronized_dataset_ids))
+                .count()
+            ),
+            "feature_models_viewed": (
+                self.session.query(HubfileViewRecord)
+                .filter(HubfileViewRecord.file_id.in_(synchronized_hubfile_ids))
+                .count()
+            ),
+            "feature_models_downloaded": (
+                self.session.query(HubfileDownloadRecord)
+                .filter(HubfileDownloadRecord.file_id.in_(synchronized_hubfile_ids))
+                .count()
+            ),
+        }
+
+    def refresh_statistics(self) -> Statistics:
+        """Persist the recomputed totals into the singleton row."""
+        statistics = self.get_statistics()
+        for field, value in self.compute_totals().items():
+            setattr(statistics, field, value)
         self.session.commit()
         return statistics

app/modules/statistics/services.py

@@ -97,6 +97,17 @@ def get_feature_models_downloaded(self) -> int:
     def refresh_statistics(self) -> Statistics:
         return self.repository.refresh_statistics()
 
+    def preview_refresh(self) -> dict[str, tuple[int, int]]:
+        """Dry-run sync: return ``{field: (before, after)}`` without writing.
+
+        Used by `rosemary counters:sync --dry-run` so operators can inspect
+        drift before committing. Every dict key is also a real field on
+        :class:`Statistics`, so callers can feed it straight into a template.
+        """
+        current = self.repository.get_statistics()
+        proposed = self.repository.compute_totals()
+        return {field: (getattr(current, field), value) for field, value in proposed.items()}
+
 
 # ─────────────────────────────────────────────────────────────────────────────
 # Dashboard service — all the aggregations in one place.

app/modules/statistics/tests/test_unit.py

@@ -78,6 +78,82 @@ def test_refresh_statistics_rebuilds_counters_from_records(mock_enqueue_task, te
     assert statistics.feature_models_downloaded == 1
 
 
+# ─── DOI filter on refresh_statistics ──────────────────────────────────────
+
+
+@patch("core.managers.task_queue_manager.TaskQueueManager.enqueue_task")
+def test_refresh_statistics_excludes_private_datasets(mock_enqueue_task, test_client, clean_database):
+    """Views and downloads for datasets without a DOI must not be counted,
+    otherwise the `/statistics` summary cards disagree with the top-N tables.
+    """
+    user = UserRepository().create(email="stats-private@example.com", password="test1234")
+
+    public_meta = DSMetaDataRepository().create(
+        title="Public",
+        description="Public",
+        publication_type=PublicationType.BOOK,
+        dataset_doi="10.1234/public",
+    )
+    public_ds = DataSetRepository().create(user_id=user.id, ds_meta_data_id=public_meta.id)
+    public_fm = FeatureModelRepository().create(dataset_id=public_ds.id)
+    public_hubfile = HubfileRepository().create(
+        name="public.uvl", checksum="pub", size=1, feature_model_id=public_fm.id
+    )
+    DSViewRecordRepository().create(dataset_id=public_ds.id, view_cookie="pub-v")
+    DSDownloadRecordRepository().create(dataset_id=public_ds.id, download_cookie="pub-d")
+    HubfileViewRecordRepository().create(file_id=public_hubfile.id, view_cookie="pub-fv")
+    HubfileDownloadRecordRepository().create(file_id=public_hubfile.id, download_cookie="pub-fd")
+
+    private_meta = DSMetaDataRepository().create(
+        title="Private",
+        description="Private",
+        publication_type=PublicationType.BOOK,
+        dataset_doi=None,
+    )
+    private_ds = DataSetRepository().create(user_id=user.id, ds_meta_data_id=private_meta.id)
+    private_fm = FeatureModelRepository().create(dataset_id=private_ds.id)
+    private_hubfile = HubfileRepository().create(
+        name="priv.uvl", checksum="priv", size=1, feature_model_id=private_fm.id
+    )
+    DSViewRecordRepository().create(dataset_id=private_ds.id, view_cookie="priv-v")
+    DSDownloadRecordRepository().create(dataset_id=private_ds.id, download_cookie="priv-d")
+    HubfileViewRecordRepository().create(file_id=private_hubfile.id, view_cookie="priv-fv")
+    HubfileDownloadRecordRepository().create(file_id=private_hubfile.id, download_cookie="priv-fd")
+
+    stats = StatisticsService().refresh_statistics()
+
+    assert stats.datasets_counter == 1
+    assert stats.feature_models_counter == 1
+    assert stats.datasets_viewed == 1
+    assert stats.datasets_downloaded == 1
+    assert stats.feature_models_viewed == 1
+    assert stats.feature_models_downloaded == 1
+
+
+def test_preview_refresh_returns_before_after_tuples(test_client, clean_database):
+    user = UserRepository().create(email="preview@example.com", password="test1234")
+    meta = DSMetaDataRepository().create(
+        title="Preview",
+        description="Preview",
+        publication_type=PublicationType.BOOK,
+        dataset_doi="10.9999/preview",
+    )
+    ds = DataSetRepository().create(user_id=user.id, ds_meta_data_id=meta.id)
+    FeatureModelRepository().create(dataset_id=ds.id)
+
+    service = StatisticsService()
+    diff = service.preview_refresh()
+
+    # Snapshot is untouched by a dry-run preview.
+    persisted = service.get_statistics()
+    assert persisted.datasets_counter == 0
+    assert persisted.feature_models_counter == 0
+
+    # The diff maps each field to (before, after).
+    assert diff["datasets_counter"] == (0, 1)
+    assert diff["feature_models_counter"] == (0, 1)
+
+
 # ─── Dashboard ──────────────────────────────────────────────────────────────
 
 

rosemary/commands/counters_commands.py

@@ -1,27 +1,69 @@
 import click
 from flask.cli import with_appcontext
 
+# Fields printed by the sync output, in the order we want them to appear.
+# Keys match `Statistics` model columns so the same list drives both the
+# live sync and the --dry-run diff.
+_FIELDS = [
+    ("datasets_counter", "Datasets (DOI-synced)"),
+    ("feature_models_counter", "Feature models (DOI-synced)"),
+    ("datasets_viewed", "Dataset views"),
+    ("feature_models_viewed", "Feature model views"),
+    ("datasets_downloaded", "Dataset downloads"),
+    ("feature_models_downloaded", "Feature model downloads"),
+]
+
+
+def _format_diff(before: int, after: int) -> str:
+    delta = after - before
+    if delta == 0:
+        return click.style(f"{after}", fg="white")
+    arrow = click.style(f"{before} → {after}", fg="yellow")
+    colour = "green" if delta > 0 else "red"
+    sign = "+" if delta > 0 else ""
+    return f"{arrow} ({click.style(f'{sign}{delta}', fg=colour)})"
+
 
 @click.command(
     "counters:sync",
-    help="Syncs all hub counters (datasets, feature models, views, downloads) from the database.",
+    help=(
+        "Recompute hub counters (datasets, feature models, views, downloads) "
+        "from the record tables. Use --dry-run to preview drift without "
+        "writing; the dashboard cache is invalidated on a successful commit."
+    ),
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    help="Show what would change without touching the Statistics row or cache.",
 )
 @with_appcontext
-def counters_sync():
-    from app.modules.statistics.services import StatisticsService
+def counters_sync(dry_run: bool):
+    from app.modules.statistics.services import DashboardService, StatisticsService
 
     service = StatisticsService()
 
     try:
-        statistics = service.refresh_statistics()
-
-        click.echo(click.style("Hub counters synced successfully.", fg="green"))
-        click.echo(f"  Datasets (synchronized):      {statistics.datasets_counter}")
-        click.echo(f"  Feature models (synchronized):{statistics.feature_models_counter}")
-        click.echo(f"  Dataset views:                {statistics.datasets_viewed}")
-        click.echo(f"  Feature model views:          {statistics.feature_models_viewed}")
-        click.echo(f"  Dataset downloads:            {statistics.datasets_downloaded}")
-        click.echo(f"  Feature model downloads:      {statistics.feature_models_downloaded}")
+        diff = service.preview_refresh()
+
+        if dry_run:
+            click.echo(click.style("Dry-run — no changes persisted.", fg="cyan"))
+        else:
+            service.refresh_statistics()
+            # Drop the cached dashboard so the next hit reflects the new totals.
+            try:
+                DashboardService().invalidate_cache()
+            except Exception as exc:  # pragma: no cover — best-effort cleanup
+                click.echo(click.style(f"[WARN] Dashboard cache invalidate failed: {exc}", fg="yellow"))
+            click.echo(click.style("Hub counters synced successfully.", fg="green"))
+
+        any_changes = any(before != after for before, after in diff.values())
+        if any_changes:
+            click.echo()
+        for field, label in _FIELDS:
+            before, after = diff.get(field, (0, 0))
+            click.echo(f"  {label:<30} {_format_diff(before, after)}")
 
     except Exception as e:
         click.echo(click.style(f"[ERROR] Failed to sync counters: {e}", fg="red"))

rosemary/commands/statistics_commands.py

@@ -1,25 +1,31 @@
 import click
 from flask.cli import with_appcontext
 
+from rosemary.commands.counters_commands import counters_sync
+
 
 @click.command(
     "statistics:refresh",
-    help="Recalculates persisted hub statistics from record tables.",
+    help=(
+        "DEPRECATED alias for `counters:sync`. Will be removed next release — "
+        "use `rosemary counters:sync` (optionally with --dry-run) instead."
+    ),
+    context_settings={"ignore_unknown_options": True},
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    help="Forwarded to `counters:sync --dry-run`.",
 )
+@click.pass_context
 @with_appcontext
-def statistics_refresh():
-    from app.modules.statistics.services import StatisticsService
-
-    service = StatisticsService()
-
-    try:
-        statistics = service.refresh_statistics()
-
-        click.echo(click.style("Statistics refreshed successfully.", fg="green"))
-        click.echo(f"Dataset views: {statistics.datasets_viewed}")
-        click.echo(f"Model views: {statistics.feature_models_viewed}")
-        click.echo(f"Dataset downloads: {statistics.datasets_downloaded}")
-        click.echo(f"Model downloads: {statistics.feature_models_downloaded}")
-
-    except Exception as e:
-        click.echo(click.style(f"[ERROR] Failed to refresh statistics: {e}", fg="red"))
+def statistics_refresh(ctx, dry_run: bool):
+    click.echo(
+        click.style(
+            "DEPRECATED: `statistics:refresh` has been renamed to `counters:sync`. "
+            "This alias will be removed next release.",
+            fg="yellow",
+        )
+    )
+    ctx.invoke(counters_sync, dry_run=dry_run)