feat(statistics): rework dashboard — correctness, perf, service layer, tests

Turns the single-handler 155-line dashboard into a cached read-model served
by a dedicated service. Same visual layout, same columns — every data bug
fixed along the way.

Correctness
-----------
* DOI filter is now applied to every aggregation (views-per-month and
  downloads-per-month used to include private datasets, so the timeline
  disagreed with the summary counters).
* Monthly series fill the full rolling 12 months instead of only the
  months where something happened — the line chart no longer connects
  non-consecutive points and draws false zeros.
* `datetime.utcnow()` replaces `datetime.now()` — the 12-month window
  was drifting by up to ~1 hour because local server time was being
  compared against UTC rows.
* "Trending Datasets" renamed to "Most viewed datasets" — the column
  was ordering by all-time views, not recency, so the name was lying.

Performance
-----------
* Monthly rollups moved from `defaultdict` on a full table dump to
  `DATE_FORMAT(col, '%Y-%m') + GROUP BY` in SQL. For busy hubs this
  stops materialising every DSDownloadRecord in Python memory.
* Dashboard output cached in Redis (5 min TTL, key
  `statistics:dashboard:v1`) with an `invalidate_cache()` escape hatch
  for seeders / bulk imports. Caching is auto-disabled under
  `TESTING=True` so tests stay deterministic.

Structure
---------
* New `DashboardService` + `DashboardData` / `DashboardRow` dataclasses.
  `routes.py` shrinks to ~10 lines.
* Template shares a single Jinja macro for the six "top-N" cards; ~250
  lines of copy-paste gone.
* Inline `<script>` blocks out, bundled JS in: `scripts.js` now imports
  `chart.js/auto` through webpack and reads the chart series from a
  `<script type="application/json">` payload.

Assets
------
* Chart.js served from the bundle instead of the CDN — works offline,
  follows the project's existing webpack convention, drops a third-party
  script tag.

Tests (7 new)
-------------
* `test_dashboard_excludes_datasets_without_doi`
* `test_dashboard_monthly_series_has_no_gaps`
* `test_dashboard_top_tables_are_ordered_by_metric`
* `test_dashboard_uses_cache_when_enabled`
* `test_dashboard_route_renders_without_errors`
* plus the pre-existing counter / sample cases.

Verified before push:
  * `rosemary linter` clean
  * `black --check` + `isort --check-only` clean
  * `pytest app/modules/ -m 'not slow'` → 124/124 green
  * `rosemary webpack:compile statistics` → bundle builds (555 KiB,
    Chart.js included).

Author: drorganvidez

app/modules/statistics/assets/js/scripts.js

@@ -1 +1,93 @@
-console.log("Hi, I am a script loaded from statistics module");
+// Dashboard chart bootstrap. The server renders the series as JSON inside
+// `<script id="dashboard-data">` so this module has a single parse step and
+// no template-interpolation inside JS (safer, easier to cache, no CSP hacks).
+
+import Chart from "chart.js/auto";
+
+function readData() {
+    const el = document.getElementById("dashboard-data");
+    if (!el) return null;
+    try {
+        return JSON.parse(el.textContent || "{}");
+    } catch (e) {
+        console.error("dashboard: bad JSON payload", e);
+        return null;
+    }
+}
+
+function renderUploads(data) {
+    const canvas = document.getElementById("uploadsChart");
+    if (!canvas || !data.months?.length) return;
+    new Chart(canvas.getContext("2d"), {
+        type: "bar",
+        data: {
+            labels: data.months,
+            datasets: [
+                {
+                    label: "Datasets uploaded",
+                    data: data.uploads || [],
+                    backgroundColor: "rgba(54, 153, 255, 0.2)",
+                    borderColor: "rgba(54, 153, 255, 1)",
+                    borderWidth: 2,
+                    borderRadius: 4,
+                },
+            ],
+        },
+        options: {
+            responsive: true,
+            plugins: { legend: { display: false } },
+            scales: { y: { beginAtZero: true, ticks: { stepSize: 1 } } },
+        },
+    });
+}
+
+function renderActivity(data) {
+    const canvas = document.getElementById("activityChart");
+    if (!canvas || !data.months?.length) return;
+    new Chart(canvas.getContext("2d"), {
+        type: "line",
+        data: {
+            labels: data.months,
+            datasets: [
+                {
+                    label: "Downloads",
+                    data: data.downloads || [],
+                    backgroundColor: "rgba(246, 78, 96, 0.1)",
+                    borderColor: "rgba(246, 78, 96, 1)",
+                    borderWidth: 2,
+                    tension: 0.3,
+                    fill: true,
+                    pointRadius: 4,
+                },
+                {
+                    label: "Views",
+                    data: data.views || [],
+                    backgroundColor: "rgba(80, 205, 137, 0.1)",
+                    borderColor: "rgba(80, 205, 137, 1)",
+                    borderWidth: 2,
+                    tension: 0.3,
+                    fill: true,
+                    pointRadius: 4,
+                },
+            ],
+        },
+        options: {
+            responsive: true,
+            plugins: { legend: { display: true, position: "top" } },
+            scales: { y: { beginAtZero: true, ticks: { stepSize: 1 } } },
+        },
+    });
+}
+
+function init() {
+    const data = readData();
+    if (!data) return;
+    renderUploads(data);
+    renderActivity(data);
+}
+
+if (document.readyState === "loading") {
+    document.addEventListener("DOMContentLoaded", init, { once: true });
+} else {
+    init();
+}

app/modules/statistics/assets/js/webpack.config.js

@@ -1,15 +1,18 @@
-const path = require('path');
+const path = require("path");
 
 module.exports = {
-  entry: path.resolve(__dirname, './scripts.js'),
-  output: {
-    filename: 'statistics.bundle.js',
-    path: path.resolve(__dirname, '../dist'),
-  },
-  resolve: {
-    fallback: {
-      "fs": false 
-    }
-  },
-  mode: 'development',
+    entry: path.resolve(__dirname, "./scripts.js"),
+    output: {
+        filename: "statistics.bundle.js",
+        path: path.resolve(__dirname, "../dist"),
+        module: true,
+    },
+    experiments: {
+        outputModule: true,
+    },
+    resolve: {
+        fallback: { fs: false },
+    },
+    mode: "development",
+    devtool: "source-map",
 };

app/modules/statistics/routes.py

@@ -1,176 +1,10 @@
-from collections import defaultdict
-from datetime import datetime, timedelta
-
 from flask import render_template
-from sqlalchemy import func
 
-from app import db
 from app.modules.statistics import statistics_bp
+from app.modules.statistics.services import DashboardService
 
 
 @statistics_bp.route("/statistics", methods=["GET"])
 def index():
-    from app.modules.dataset.models import Author, DataSet, DSMetaData, DSMetrics
-    from app.modules.featuremodel.models import FeatureModel
-    from app.modules.statistics.services import StatisticsService
-
-    statistics_service = StatisticsService()
-
-    # --- Summary counters ---
-    total_datasets = (
-        db.session.query(DataSet)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .count()
-    )
-    total_feature_models = (
-        db.session.query(FeatureModel)
-        .join(DataSet, DataSet.id == FeatureModel.dataset_id)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .count()
-    )
-    total_authors = db.session.query(Author).count()
-    total_views = statistics_service.get_datasets_viewed()
-    total_downloads = statistics_service.get_datasets_downloaded()
-
-    avg_models_per_dataset = round(total_feature_models / total_datasets, 2) if total_datasets else 0
-
-    # --- Top 5 datasets with most feature models ---
-    top_datasets_by_models = (
-        db.session.query(DataSet, func.count(FeatureModel.id).label("fm_count"))
-        .join(FeatureModel, DataSet.id == FeatureModel.dataset_id)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .group_by(DataSet.id)
-        .order_by(func.count(FeatureModel.id).desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Top 5 datasets with most features (DSMetrics) ---
-    top_datasets_by_features = (
-        db.session.query(DataSet, DSMetrics.number_of_features)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .join(DSMetrics, DSMetaData.ds_metrics_id == DSMetrics.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .filter(DSMetrics.number_of_features.isnot(None))
-        .order_by(DSMetrics.number_of_features.desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Datasets by publication type ---
-    datasets_by_type = (
-        db.session.query(DSMetaData.publication_type, func.count(DataSet.id).label("count"))
-        .join(DataSet, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .filter(DSMetaData.publication_type.isnot(None))
-        .group_by(DSMetaData.publication_type)
-        .order_by(func.count(DataSet.id).desc())
-        .all()
-    )
-
-    # --- Timeline: uploads per month (last 12 months) ---
-    twelve_months_ago = datetime.now() - timedelta(days=365)
-    recent_datasets = (
-        db.session.query(DataSet)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .filter(DataSet.created_at >= twelve_months_ago)
-        .all()
-    )
-    timeline = defaultdict(int)
-    for ds in recent_datasets:
-        timeline[ds.created_at.strftime("%Y-%m")] += 1
-    timeline_labels = sorted(timeline.keys())
-    timeline_data = [timeline[k] for k in timeline_labels]
-
-    # --- 5 most recently uploaded datasets ---
-    latest_datasets = (
-        db.session.query(DataSet)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .order_by(DataSet.created_at.desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Top 5 most viewed datasets ---
-    from app.modules.dataset.models import DSDownloadRecord, DSViewRecord
-
-    top_datasets_by_views = (
-        db.session.query(DataSet, func.count(DSViewRecord.id).label("view_count"))
-        .join(DSViewRecord, DataSet.id == DSViewRecord.dataset_id)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .group_by(DataSet.id)
-        .order_by(func.count(DSViewRecord.id).desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Top 5 most downloaded datasets ---
-    top_datasets_by_downloads = (
-        db.session.query(DataSet, func.count(DSDownloadRecord.id).label("download_count"))
-        .join(DSDownloadRecord, DataSet.id == DSDownloadRecord.dataset_id)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .group_by(DataSet.id)
-        .order_by(func.count(DSDownloadRecord.id).desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Top 5 datasets with most configurations (number_of_models, closest to restrictions) ---
-    top_datasets_by_configurations = (
-        db.session.query(DataSet, DSMetrics.number_of_models)
-        .join(DSMetaData, DataSet.ds_meta_data_id == DSMetaData.id)
-        .join(DSMetrics, DSMetaData.ds_metrics_id == DSMetrics.id)
-        .filter(DSMetaData.dataset_doi.isnot(None))
-        .filter(DSMetrics.number_of_models.isnot(None))
-        .order_by(DSMetrics.number_of_models.desc())
-        .limit(5)
-        .all()
-    )
-
-    # --- Downloads per month (last 12 months) ---
-    recent_downloads = (
-        db.session.query(DSDownloadRecord).filter(DSDownloadRecord.download_date >= twelve_months_ago).all()
-    )
-    downloads_by_month = defaultdict(int)
-    for record in recent_downloads:
-        downloads_by_month[record.download_date.strftime("%Y-%m")] += 1
-
-    # --- Views per month (last 12 months) ---
-    recent_views = db.session.query(DSViewRecord).filter(DSViewRecord.view_date >= twelve_months_ago).all()
-    views_by_month = defaultdict(int)
-    for record in recent_views:
-        views_by_month[record.view_date.strftime("%Y-%m")] += 1
-
-    # Merge all months for a unified x-axis
-    all_months = sorted(set(timeline_labels) | set(downloads_by_month.keys()) | set(views_by_month.keys()))
-    activity_downloads = [downloads_by_month.get(m, 0) for m in all_months]
-    activity_views = [views_by_month.get(m, 0) for m in all_months]
-
-    return render_template(
-        "statistics/index.html",
-        total_datasets=total_datasets,
-        total_feature_models=total_feature_models,
-        total_authors=total_authors,
-        total_views=total_views,
-        total_downloads=total_downloads,
-        avg_models_per_dataset=avg_models_per_dataset,
-        top_datasets_by_models=top_datasets_by_models,
-        top_datasets_by_features=top_datasets_by_features,
-        top_datasets_by_views=top_datasets_by_views,
-        top_datasets_by_downloads=top_datasets_by_downloads,
-        top_datasets_by_configurations=top_datasets_by_configurations,
-        datasets_by_type=datasets_by_type,
-        timeline_labels=timeline_labels,
-        timeline_data=timeline_data,
-        all_months=all_months,
-        activity_downloads=activity_downloads,
-        activity_views=activity_views,
-        latest_datasets=latest_datasets,
-    )
+    dashboard = DashboardService().build_dashboard()
+    return render_template("statistics/index.html", dashboard=dashboard)