Hardcode dashboard collection retention to 90 days with opt-in controller setting

The get_retention_days() function (which reads from the Controller's active
cleanup_jobs schedules) was not fully tested. Gate it behind
DASHBOARD_COLLECTION['USE_CONTROLLER_RETENTION'] (default False) so both
the initial backfill window and the daily cleanup task always use the fixed
90-day default until the setting is explicitly enabled.

Enable via settings.yaml or METRICS_SERVICE_DASHBOARD_COLLECTION__USE_CONTROLLER_RETENTION=true.

Assisted-by: Claude Code (claude-sonnet-4-6)
Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: cshiels-ie

apps/dashboard_reports/tasks.py

@@ -55,6 +55,12 @@ def _get_renamed_kwarg(kwargs: dict, new_key: str, old_key: str, task_name: str)
     return _MISSING
 
 
+def _use_controller_retention() -> bool:
+    """Return True when DASHBOARD_COLLECTION['USE_CONTROLLER_RETENTION'] is set in settings."""
+    dashboard_cfg = getattr(settings, "DASHBOARD_COLLECTION", None) or {}
+    return bool(dashboard_cfg.get("USE_CONTROLLER_RETENTION", False))
+
+
 def get_retention_days(db_name: str = DEFAULT_AWX_DB_NAME) -> int:
     """
     Return the effective retention period in days derived from active AWX cleanup schedules.
@@ -196,9 +202,10 @@ def _sync_jobs_atomically(job_results: list) -> list:
 def _resolve_collection_params(task_name: str, kwargs: dict) -> tuple[str, datetime, datetime, int]:
     """Resolve db_name, since, until, and batch_size from task kwargs with defaults applied.
 
-    When ``since`` is not provided and no prior ``JobData`` timestamp exists, the initial
-    retention window is determined by ``get_retention_days`` (queried from active AWX cleanup
-    schedules) rather than a static setting.
+    When ``since`` is not provided and no prior ``JobData`` timestamp exists, the backfill
+    window defaults to ``DEFAULT_RETENTION_DAYS`` (90 days). When the
+    ``DASHBOARD_COLLECTION['USE_CONTROLLER_RETENTION']`` setting is enabled, the window is
+    derived from active AWX cleanup_jobs schedules via ``get_retention_days`` instead.
     """
     dashboard_cfg = getattr(settings, "DASHBOARD_COLLECTION", None) or {}
     db_name = _get_renamed_kwarg(kwargs, "awx_database", "database", task_name)
@@ -209,9 +216,12 @@ def _resolve_collection_params(task_name: str, kwargs: dict) -> tuple[str, datet
     # used correctly and no jobs are missed in the gap between MAX(finished) and MAX(awx_modified).
     since = _parse_dt(kwargs.get("since")) or JobData.last_finished_timestamp()
     if since is None:
-        retention_days = get_retention_days(db_name)
-        if retention_days <= 0:
-            raise ValueError(f"Retention days must be > 0, got {retention_days!r}")
+        if _use_controller_retention():
+            retention_days = get_retention_days(db_name)
+            if retention_days <= 0:
+                raise ValueError(f"Retention days must be > 0, got {retention_days!r}")
+        else:
+            retention_days = DEFAULT_RETENTION_DAYS
         since = (
             (until - timedelta(days=retention_days)).replace(hour=0, minute=0, second=0, microsecond=0).astimezone(UTC)
         )
@@ -614,7 +624,8 @@ def cleanup_dashboard_reports_old_data(**kwargs) -> dict[str, Any]:
     db_name = _get_renamed_kwarg(kwargs, "awx_database", "database", task_name)
     db_name = DEFAULT_AWX_DB_NAME if db_name is _MISSING else db_name
     retention_days = _get_renamed_kwarg(kwargs, "retention_days", "retention_period_days", task_name)
-    retention_days = get_retention_days(db_name) if retention_days is _MISSING else retention_days
+    if retention_days is _MISSING:
+        retention_days = get_retention_days(db_name) if _use_controller_retention() else DEFAULT_RETENTION_DAYS
     retention_days, error_result = _normalize_retention_days(task_name, retention_days)
     if error_result is not None:
         return error_result

apps/settings/defaults.py

@@ -180,6 +180,9 @@
 # Dashboard collection schedule configuration
 DASHBOARD_COLLECTION = {
     "COLLECTION_SCHEDULE_CRON": "0 */6 * * *",
+    # Set to True to derive the backfill/retention window from the Controller's active
+    # cleanup_jobs schedules instead of the fixed 90-day default.
+    "USE_CONTROLLER_RETENTION": False,
 }
 
 # Conditional static files directory (avoids staticfiles.W004 when absent)

tests/unit/dashboard_reports/test_tasks.py

@@ -508,6 +508,12 @@ def mock_retention_days(self):
         with patch("apps.dashboard_reports.tasks.get_retention_days", return_value=DEFAULT_RETENTION_DAYS) as mock:
             yield mock
 
+    @pytest.fixture
+    def with_controller_retention(self):
+        """Enable the controller-retention path so get_retention_days() is called."""
+        with patch("apps.dashboard_reports.tasks._use_controller_retention", return_value=True):
+            yield
+
     def test_cleanup_success(self, mock_jobdata_objects, mock_log_task_execution, mock_create_task_result_cleanup):
         """Successful deletion returns a success result with the correct record count."""
         mock_jobdata_objects.objects.filter.return_value.count.return_value = 5
@@ -538,33 +544,53 @@ def test_cleanup_exception(self, mock_jobdata_objects, mock_log_task_execution,
         assert "Cleanup failed" in kwargs["error"]
 
     def test_cleanup_defaults_to_get_retention_days(
-        self, mock_retention_days, mock_jobdata_objects, mock_log_task_execution, mock_create_task_result_cleanup
+        self,
+        with_controller_retention,
+        mock_retention_days,
+        mock_jobdata_objects,
+        mock_log_task_execution,
+        mock_create_task_result_cleanup,
     ):
-        """When retention_days is omitted, get_retention_days() determines the default."""
+        """When controller retention is enabled and retention_days is omitted, get_retention_days() determines the default."""
         mock_retention_days.return_value = 60
         mock_jobdata_objects.objects.filter.return_value.count.return_value = 0
         cleanup_dashboard_reports_old_data()
         _, kwargs = mock_create_task_result_cleanup.call_args
         assert kwargs["data"]["retention_days"] == 60
 
     def test_cleanup_passes_db_name_to_get_retention_days(
-        self, mock_retention_days, mock_jobdata_objects, mock_log_task_execution, mock_create_task_result_cleanup
+        self,
+        with_controller_retention,
+        mock_retention_days,
+        mock_jobdata_objects,
+        mock_log_task_execution,
+        mock_create_task_result_cleanup,
     ):
         """The 'awx_database' kwarg is forwarded to get_retention_days so retention and collection use the same connection."""
         mock_jobdata_objects.objects.filter.return_value.count.return_value = 0
         cleanup_dashboard_reports_old_data(awx_database="custom_db")
         mock_retention_days.assert_called_once_with("custom_db")
 
     def test_cleanup_passes_default_db_name_when_no_database_kwarg(
-        self, mock_retention_days, mock_jobdata_objects, mock_log_task_execution, mock_create_task_result_cleanup
+        self,
+        with_controller_retention,
+        mock_retention_days,
+        mock_jobdata_objects,
+        mock_log_task_execution,
+        mock_create_task_result_cleanup,
     ):
         """When 'awx_database' kwarg is absent, get_retention_days is called with the 'awx' alias."""
         mock_jobdata_objects.objects.filter.return_value.count.return_value = 0
         cleanup_dashboard_reports_old_data()
         mock_retention_days.assert_called_once_with("awx")
 
     def test_cleanup_deprecated_database_kwarg_still_honored(
-        self, mock_retention_days, mock_jobdata_objects, mock_log_task_execution, mock_create_task_result_cleanup
+        self,
+        with_controller_retention,
+        mock_retention_days,
+        mock_jobdata_objects,
+        mock_log_task_execution,
+        mock_create_task_result_cleanup,
     ):
         """The pre-rename 'database' kwarg still resolves db_name rather than being silently dropped."""
         mock_jobdata_objects.objects.filter.return_value.count.return_value = 0
@@ -690,10 +716,11 @@ def test_since_falls_back_to_last_timestamp(self, mock_jobdata):
         _, got_since, _, _ = _resolve_collection_params("collect_dashboard_reports_data", {})
         assert got_since == ts
 
+    @patch("apps.dashboard_reports.tasks._use_controller_retention", return_value=True)
     @patch("apps.dashboard_reports.tasks.get_retention_days", return_value=30)
     @patch("apps.dashboard_reports.tasks.JobData")
-    def test_since_computed_from_retention_days_when_no_timestamp(self, mock_jobdata, mock_retention):
-        """When since is absent and last_timestamp() is None, compute since from get_retention_days()."""
+    def test_since_computed_from_retention_days_when_no_timestamp(self, mock_jobdata, mock_retention, _mock_use_ctrl):
+        """When controller retention is enabled and last_timestamp() is None, compute since from get_retention_days()."""
         from django.test import override_settings
 
         mock_jobdata.last_finished_timestamp.return_value = None
@@ -709,10 +736,11 @@ def test_since_computed_from_retention_days_when_no_timestamp(self, mock_jobdata
         assert batch_size == 500
         mock_retention.assert_called_once_with("awx")
 
+    @patch("apps.dashboard_reports.tasks._use_controller_retention", return_value=True)
     @patch("apps.dashboard_reports.tasks.get_retention_days", return_value=30)
     @patch("apps.dashboard_reports.tasks.JobData")
-    def test_custom_database_kwarg_forwarded_to_retention(self, mock_jobdata, mock_retention):
-        """When 'awx_database' kwarg is given and no prior timestamp exists, db_name is passed to get_retention_days."""
+    def test_custom_database_kwarg_forwarded_to_retention(self, mock_jobdata, mock_retention, _mock_use_ctrl):
+        """When controller retention is enabled and no prior timestamp exists, db_name is passed to get_retention_days."""
         mock_jobdata.last_finished_timestamp.return_value = None
         until = datetime(2024, 6, 1, tzinfo=UTC)
         _resolve_collection_params(
@@ -746,10 +774,11 @@ def test_invalid_batch_size_raises_value_error(self, mock_jobdata):
         ):
             _resolve_collection_params("collect_dashboard_reports_data", {})
 
+    @patch("apps.dashboard_reports.tasks._use_controller_retention", return_value=True)
     @patch("apps.dashboard_reports.tasks.get_retention_days", return_value=0)
     @patch("apps.dashboard_reports.tasks.JobData")
-    def test_zero_retention_days_raises_value_error(self, mock_jobdata, mock_retention):
-        """get_retention_days returning 0 raises ValueError (retention must be > 0)."""
+    def test_zero_retention_days_raises_value_error(self, mock_jobdata, mock_retention, _mock_use_ctrl):
+        """When controller retention is enabled, get_retention_days returning 0 raises ValueError."""
         mock_jobdata.last_finished_timestamp.return_value = None
         with pytest.raises(ValueError, match="Retention days must be > 0"):
             _resolve_collection_params("collect_dashboard_reports_data", {})