feat(core): whole-job deadline + scrub driver-exception messages from run error (rec #5)

src/Integrations/NeoReports.AspNetCore/NeoReportsEndpointRouteBuilderExtensions.cs

@@ -189,9 +189,10 @@
             if (result.Status == ReportRunStatus.Failed)
             {
                 await artifactStore.DeleteAsync(jobId, CancellationToken.None).ConfigureAwait(false);
-                // result.Error is the run's failure reason, which for a source read failure is a
-                // driver exception message that can echo connection-string fragments. Log it and
-                // return a generic detail, the same scrub-and-log stance as the schema endpoints.
+                // result.Error is already scrubbed at the source (a driver exception is reduced to its
+                // type name, only NeoReports' own curated messages survive). Log it and still return a
+                // generic detail — defence in depth, the same scrub-and-log stance as the schema
+                // endpoints — so the response never carries even the run's own reason string.
                 http.RequestServices.GetRequiredService<ILoggerFactory>()
                     .CreateLogger("NeoReports.Run")
                     .LogWarning("Synchronous run of report '{Report}' (job {JobId}) failed: {Reason}", name, jobId, result.Error);
@@ -1052,7 +1053,7 @@
 
         try
         {
             SchemaCatalog catalog = await explorer!.GetCatalogAsync(definition!, cancellationToken).ConfigureAwait(false);
             return Results.Ok(ToCatalogResponse(catalog));
         }
         catch (Exception ex) when (ex is not OperationCanceledException)
@@ -1074,7 +1075,7 @@
 
         try
         {
             TablePreview preview = await explorer!
                 .PreviewTableAsync(definition!, schema ?? string.Empty, table, SchemaPreviewTop, cancellationToken)
                 .ConfigureAwait(false);
             return Results.Ok(new TablePreviewResponse(preview.Columns, preview.Rows));

src/Jobs/NeoReports.Jobs/ReportJobWorker.cs

@@ -83,7 +83,13 @@ await _store.UpdateStatusAsync(jobId, ReportJobStatus.Cancelled, "Cancelled.", C
         }
         catch (Exception ex)
         {
-            await _store.UpdateStatusAsync(jobId, ReportJobStatus.Failed, ex.Message, CancellationToken.None)
+            // Persisted as job.Error (surfaced by GET /jobs). This path handles exceptions that
+            // escape the runner rather than becoming a Failed result (e.g. a source/writer that
+            // throws during setup). Keep a NeoReports message (curated, secret-free); reduce any
+            // other — a driver exception can echo the connection string — to its type name. The full
+            // exception is logged just below for diagnosis.
+            var reason = ex is NeoReportsException ? ex.Message : ex.GetType().Name;
+            await _store.UpdateStatusAsync(jobId, ReportJobStatus.Failed, reason, CancellationToken.None)
                 .ConfigureAwait(false);
             _logger.LogError(ex, "Job {JobId} for report {Report} failed.", jobId, reportName);
             throw;

src/NeoReports.Core/Building/ReportBuilder.cs

@@ -25,6 +25,7 @@ public sealed class ReportBuilder<TRow>
     private readonly List<DestinationSpec> _destinations = new();
     private readonly RetryOptions _retry = new();
     private readonly FailureStrategyBuilder _failure = new();
+    private TimeSpan? _deadline;
 
     private IBatchSource<TRow>? _batchSource;
     private IStreamingSource<TRow>? _streamingSource;
@@ -244,6 +245,21 @@ public ReportBuilder<TRow> Retry(Action<RetryOptions> configure)
         return this;
     }
 
+    /// <summary>
+    /// Sets an overall wall-clock deadline for the whole run — reads, writes and uploads together.
+    /// Complements the per-attempt read timeout (<c>Retry(r =&gt; r.Timeout(...))</c>): the deadline
+    /// bounds the entire report so a run that never hangs on a single step but drags on overall is
+    /// still stopped. Off by default. On expiry the run is cooperatively cancelled (for work that
+    /// honors cancellation) and surfaces as a cancelled run.
+    /// </summary>
+    /// <param name="deadline">A positive overall deadline.</param>
+    public ReportBuilder<TRow> Deadline(TimeSpan deadline)
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(deadline, TimeSpan.Zero);
+        _deadline = deadline;
+        return this;
+    }
+
     /// <summary>Configures what happens after a batch exhausts its retries.</summary>
     /// <param name="configure">Action that mutates the failure-strategy builder.</param>
     public ReportBuilder<TRow> OnFailure(Action<FailureStrategyBuilder> configure)
@@ -358,7 +374,8 @@ IProjectedBatchReader ReaderFactory(ReportExecutionContext execution, IServicePr
             _schedule,
             _sourceRef,
             _trackProgress,
-            countRows);
+            countRows,
+            _deadline);
     }
 
     private (ReportSchema Schema, OutputProjection<TRow> Projection) ResolveView(OutputView<TRow>? view, string what)

src/NeoReports.Core/CompiledReport.cs

@@ -15,7 +15,7 @@
 /// </summary>
 public sealed class CompiledReport
 {
     internal CompiledReport(
         string name,
         ReportSchema schema,
         int pageSize,
@@ -30,7 +30,8 @@
         ScheduleConfig? schedule = null,
         string? sourceRef = null,
         bool trackProgress = true,
-        Func<ReportExecutionContext, IServiceProvider, CancellationToken, Task<long?>>? countRows = null)
+        Func<ReportExecutionContext, IServiceProvider, CancellationToken, Task<long?>>? countRows = null,
+        TimeSpan? deadline = null)
     {
         Name = name;
         Schema = schema;
@@ -47,6 +48,7 @@
         SourceRef = sourceRef;
         TrackProgress = trackProgress;
         RowCountFactory = countRows;
+        Deadline = deadline;
 
         OutputFormats = outputs.Select(o => o.Factory.Format).ToArray();
         DestinationTypes = destinations.Select(d => d.Factory.Type).ToArray();
@@ -129,4 +131,14 @@
     /// needs only this one check, not <see cref="TrackProgress"/> separately.
     /// </summary>
     internal Func<ReportExecutionContext, IServiceProvider, CancellationToken, Task<long?>>? RowCountFactory { get; }
+
+    /// <summary>
+    /// An overall wall-clock deadline for a whole run, declared via <c>ReportBuilder&lt;T&gt;.Deadline(...)</c>,
+    /// or <c>null</c> (the default) for no deadline. Complements the per-attempt read timeout
+    /// (<c>RetryOptions.Timeout</c>, which bounds each read): the deadline bounds the entire report —
+    /// reads, writes and uploads together — so a run that never hangs on any single step but drags on
+    /// overall is still stopped. Enforced cooperatively by <c>ReportRunner.RunAsync</c>: on expiry the
+    /// run is cancelled (it surfaces as a cancelled run), for any work that honors cancellation.
+    /// </summary>
+    public TimeSpan? Deadline { get; }
 }

src/NeoReports.Core/Pipeline/ReportRunner.cs

@@ -38,7 +38,7 @@
     }
 
     /// <inheritdoc />
-    public Task<ReportRunResult> RunAsync(
+    public async Task<ReportRunResult> RunAsync(
         string reportName,
         IReadOnlyDictionary<string, object?>? parameters = null,
         string? jobId = null,
@@ -49,9 +49,31 @@
 
         jobId ??= Guid.NewGuid().ToString("N");
         var logger = _loggerFactory.CreateLogger($"NeoReports.Report.{report.Name}");
-        var execution = new ReportExecutionContext(jobId, report.Name, parameters, logger, cancellationToken);
 
-        return ExecuteAsync(report, execution, _services, cancellationToken);
+        // An overall wall-clock deadline (opt-in): a linked source cancels the whole run once the
+        // report's Deadline elapses, on top of any per-attempt read timeout. On expiry the run
+        // cancels like any cooperative cancellation (Cancelled outcome) — a warning distinguishes it
+        // in the log from a caller-requested cancel.
+        using CancellationTokenSource? deadlineCts = report.Deadline is not null
+            ? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)
+            : null;
+        // Only evaluated when deadlineCts is non-null — i.e. exactly when report.Deadline has a value.
+        deadlineCts?.CancelAfter(report.Deadline!.Value);
+        CancellationToken runToken = deadlineCts?.Token ?? cancellationToken;
+
+        var execution = new ReportExecutionContext(jobId, report.Name, parameters, logger, runToken);
+
+        try
+        {
+            return await ExecuteAsync(report, execution, _services, runToken).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (deadlineCts is { IsCancellationRequested: true } && !cancellationToken.IsCancellationRequested)
+        {
+            logger.LogWarning(
+                "Report {Report} (job {JobId}) exceeded its {Deadline} deadline and was cancelled.",
+                report.Name, jobId, report.Deadline);
+            throw;
+        }
     }
 
     /// <summary>
@@ -62,7 +84,7 @@
     /// <param name="execution">The execution context (carries job id, parameters, logger, token).</param>
     /// <param name="services">Service provider passed to writer/destination factories.</param>
     /// <param name="cancellationToken">Token that cooperatively cancels the run.</param>
     public static async Task<ReportRunResult> ExecuteAsync(
         CompiledReport report,
         ReportExecutionContext execution,
         IServiceProvider services,
@@ -86,7 +108,7 @@
 
         var outputs = new List<RunningOutput>(report.Outputs.Count);
         var sectioned = new List<RunningSectioned>(report.SectionedOutputs.Count);
         var reader = report.ReaderFactory(execution, services);
 
         long recordsRead = 0, recordsWritten = 0;
         int retries = 0, batches = 0, skipped = 0;
@@ -100,19 +122,24 @@
         long? totalRecords = null;
 
         var jobEventStore = services.GetService(typeof(IJobEventStore)) as IJobEventStore;
         var events = await JobEventEmitter.CreateAsync(jobEventStore, execution.JobId, execution.Logger, cancellationToken)
             .ConfigureAwait(false);
 
         // The retry hook only ever emits an event (ADR D38, ground rule 3: telemetry must never
         // change a run's outcome) — ShouldHandle/backoff/jitter above are untouched. It also logs at
         // Debug so a host that hasn't opted into an IJobEventStore still sees retries in its ILogger
         // sink; the event stream carries the full structured detail regardless.
         var resilience = ResiliencePipelineFactory.Build(report.Retry, async (attempt, delay, ex) =>
         {
             execution.Logger.LogDebug(
                 "Retrying batch {Page} (attempt {Attempt}) after {DelayMs}ms: {ExceptionType}",
                 pageNumber, attempt, delay.TotalMilliseconds, ex?.GetType().Name ?? "Unknown");
-            await events.EmitAsync(JobEventTypes.Retry, ex?.Message, new Dictionary<string, string>
+            // The event message is the persisted retry reason (shown by GET /jobs/{id}/events). Keep a
+            // NeoReports message (curated, secret-free); reduce any other exception (a transient driver
+            // exception whose message can echo the connection string) to its type — which the data
+            // dictionary's "exceptionType" already carries anyway. The full exception is at Debug above.
+            var retryReason = ex is NeoReportsException ? ex.Message : ex?.GetType().Name;
+            await events.EmitAsync(JobEventTypes.Retry, retryReason, new Dictionary<string, string>
             {
                 ["page"] = pageNumber.ToString(CultureInfo.InvariantCulture),
                 ["attempt"] = attempt.ToString(CultureInfo.InvariantCulture),
@@ -126,9 +153,9 @@
         // all-or-nothing publish guarantee), only in a dedicated partial-artifact store, and only
         // when one is registered. A capture failure (writer refuses to finalize mid-failure, store
         // I/O error, ...) must never change the run's own outcome.
         async Task CaptureOnePartialAsync(
             Func<CancellationToken, Task> finalize, Func<ValueTask> disposeWriter, FileStream stream,
             Action markClosed, string path, string fileName, string mimeType, IPartialArtifactStore partialStore)
         {
             try
             {
@@ -282,9 +309,14 @@
                     // A read failure cannot advance keyset pagination, so skipping is impossible:
                     // either the strategy aborts, or we abort to avoid silently truncating data.
                     status = ReportRunStatus.Failed;
+                    // This string is persisted as the run's error (surfaced by GET /jobs and the
+                    // RunFailed event). NeoReports' own exceptions carry secret-free messages and are
+                    // kept; any other (a driver exception that can echo connection-string fragments) is
+                    // reduced to its type name. The full exception is logged just below.
+                    var readDetail = ex is NeoReportsException ? ex.Message : ex.GetType().Name;
                     error = decision.Action == FailureAction.AbortReport
                         ? decision.Reason
-                        : $"Batch {pageNumber} could not be read and cannot be skipped (no cursor to advance): {ex.Message}";
+                        : $"Batch {pageNumber} could not be read and cannot be skipped (no cursor to advance): {readDetail}";
                     execution.Logger.LogError(
                         ex, "Report {ReportName} failed at batch {Page} (read failure, run aborted): {Reason}",
                         report.Name, pageNumber, error);
@@ -458,7 +490,7 @@
             else
             {
                 // status == Failed (from either a read failure or a write failure escalated to
                 // abort) — CompletedPartial runs legitimately publish above and never reach here;
                 // only a genuine failure captures partials (D11's batch-atomicity: the partial file
                 // contains exactly the fully-written batches).
                 await CapturePartialArtifactsAsync().ConfigureAwait(false);

src/NeoReports.Core/Resilience/FailureStrategies.cs

@@ -13,8 +13,16 @@ public sealed class AbortStrategy : IFailureStrategy
     public Task<FailureDecision> HandleAsync(BatchFailureContext context, CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(context);
+        // The reason is persisted as the run's error (ReportRunResult.Error, the RunFailed event, and
+        // GET /jobs). NeoReports' own exceptions carry curated, secret-free messages and are kept; any
+        // other exception (a driver exception, whose message can echo the connection string) is
+        // reduced to its type name. The full exception is logged through ILogger by the runner's
+        // abort path for diagnosis regardless.
+        var detail = context.Exception is NeoReportsException
+            ? context.Exception.Message
+            : context.Exception.GetType().Name;
         return Task.FromResult(FailureDecision.Abort(
-            $"Batch {context.PageNumber} failed after {context.AttemptsExhausted} attempt(s): {context.Exception.Message}"));
+            $"Batch {context.PageNumber} failed after {context.AttemptsExhausted} attempt(s): {detail}"));
     }
 }
 

tests/NeoReports.Core.UnitTests/DeadlineTests.cs

@@ -0,0 +1,68 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Logging.Abstractions;
+using NeoReports.Core;
+using NeoReports.Core.Building;
+using NeoReports.Core.Pipeline;
+using NeoReports.Core.Registry;
+using NeoReports.Core.UnitTests.Fakes;
+using Shouldly;
+using Xunit;
+
+namespace NeoReports.Core.UnitTests;
+
+public class DeadlineTests
+{
+    private static CompiledReport Report(string name, TimeSpan? deadline, TimeSpan perPageDelay)
+    {
+        var builder = new ReportBuilder<Sale>(name)
+            .From(new LazySaleSource(rowCount: 1_000_000, perPageDelay: perPageDelay))
+            .WithPageSize(10)
+            .Column(v => v.Id, "Id")
+            .To(new OutputSpec(new FakeWriterFactory()));
+        if (deadline is { } d)
+            builder.Deadline(d);
+        return builder.Build();
+    }
+
+    private static ReportRunner Runner(CompiledReport report)
+    {
+        var registry = new ReportRegistry();
+        registry.Register(report);
+        return new ReportRunner(registry, new EmptyServiceProvider(), NullLoggerFactory.Instance);
+    }
+
+    [Fact]
+    public void Deadline_rejects_a_non_positive_value()
+    {
+        Should.Throw<ArgumentOutOfRangeException>(() => new ReportBuilder<Sale>("r").Deadline(TimeSpan.Zero));
+    }
+
+    [Fact]
+    public async Task Run_is_cancelled_when_it_exceeds_its_deadline()
+    {
+        // The source would take ~forever (1M rows, 5s per 10-row page); the 100ms deadline must stop it.
+        var runner = Runner(Report("slow", deadline: TimeSpan.FromMilliseconds(100), perPageDelay: TimeSpan.FromSeconds(5)));
+
+        var stopwatch = Stopwatch.StartNew();
+        await Should.ThrowAsync<OperationCanceledException>(() => runner.RunAsync("slow"));
+        stopwatch.Stop();
+
+        // Stopped promptly by the deadline, not after a page delay (5s) or the whole source.
+        stopwatch.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(3));
+    }
+
+    [Fact]
+    public async Task Run_without_a_deadline_is_not_cancelled()
+    {
+        // A quick report with no deadline completes normally (5 rows, no delay).
+        var report = new ReportBuilder<Sale>("fast")
+            .From(new LazySaleSource(rowCount: 5, perPageDelay: TimeSpan.Zero))
+            .WithPageSize(10)
+            .Column(v => v.Id, "Id")
+            .To(new OutputSpec(new FakeWriterFactory()))
+            .Build();
+
+        ReportRunResult result = await Runner(report).RunAsync("fast");
+        result.Status.ShouldBe(ReportRunStatus.Completed);
+    }
+}

tests/NeoReports.Core.UnitTests/ErrorScrubTests.cs

@@ -0,0 +1,82 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using NeoReports.Abstractions;
+using NeoReports.Core.Building;
+using NeoReports.Core.Pipeline;
+using NeoReports.Core.UnitTests.Fakes;
+using Shouldly;
+using Xunit;
+
+namespace NeoReports.Core.UnitTests;
+
+public class ErrorScrubTests
+{
+    /// <summary>A source whose read throws a fixed exception, to drive the run's error composition.</summary>
+    private sealed class ThrowingSource(Exception toThrow) : IBatchSource<Sale>
+    {
+        public ReportSchema Schema { get; } = new(new[] { new ReportColumn("Id", ColumnType.Integer) });
+
+        public Task<BatchResult<Sale>> ReadBatchAsync(BatchContext context, CancellationToken cancellationToken) =>
+            throw toThrow;
+    }
+
+    private static async Task<ReportRunResult> RunWithReadFailure(Exception toThrow)
+    {
+        var report = new ReportBuilder<Sale>("r")
+            .From(new ThrowingSource(toThrow))
+            .WithPageSize(10)
+            .Column(v => v.Id, "Id")
+            .To(new OutputSpec(new FakeWriterFactory()))
+            .OnFailure(f => f.AbortReport())
+            .Build();
+
+        var execution = new ReportExecutionContext("job", "r", null, NullLogger.Instance, CancellationToken.None);
+        return await ReportRunner.ExecuteAsync(report, execution, new EmptyServiceProvider(), CancellationToken.None);
+    }
+
+    [Fact]
+    public async Task A_driver_exception_message_is_reduced_to_its_type_in_the_run_error()
+    {
+        // Simulates a driver exception whose message echoes a connection string.
+        var driver = new InvalidOperationException("Login failed for user 'sa'. Server=db.internal:1433;Database=payroll");
+        ReportRunResult result = await RunWithReadFailure(driver);
+
+        result.Status.ShouldBe(ReportRunStatus.Failed);
+        result.Error!.ShouldContain("InvalidOperationException");
+        result.Error!.ShouldNotContain("db.internal");
+        result.Error!.ShouldNotContain("sa");
+        result.Error!.ShouldNotContain("payroll");
+    }
+
+    [Fact]
+    public async Task Skip_strategy_read_failure_also_scrubs_the_driver_message()
+    {
+        // A read failure can't be skipped (no cursor to advance), so even SkipBatchAndLog aborts here
+        // via the runner's own readDetail composition — which must scrub the driver message too.
+        var driver = new InvalidOperationException("Cannot open server 'sql-prod.corp' requested by the login.");
+        var report = new ReportBuilder<Sale>("r")
+            .From(new ThrowingSource(driver))
+            .WithPageSize(10)
+            .Column(v => v.Id, "Id")
+            .To(new OutputSpec(new FakeWriterFactory()))
+            .OnFailure(f => f.SkipBatchAndLog())
+            .Build();
+
+        var execution = new ReportExecutionContext("job", "r", null, NullLogger.Instance, CancellationToken.None);
+        ReportRunResult result = await ReportRunner.ExecuteAsync(report, execution, new EmptyServiceProvider(), CancellationToken.None);
+
+        result.Status.ShouldBe(ReportRunStatus.Failed);
+        result.Error!.ShouldContain("InvalidOperationException");
+        result.Error!.ShouldNotContain("sql-prod.corp");
+    }
+
+    [Fact]
+    public async Task A_NeoReports_exception_message_is_kept_because_it_is_curated()
+    {
+        // Our own exceptions carry secret-free, useful messages (e.g. a missing named source).
+        var ours = new ConfigurationException("No source named 'sales-db' is registered.");
+        ReportRunResult result = await RunWithReadFailure(ours);
+
+        result.Status.ShouldBe(ReportRunStatus.Failed);
+        result.Error!.ShouldContain("sales-db");
+    }
+}