perf(reporting): address PR #333 review — real disk swap, spill IT, export-failure logging, async timeout

Follow-up to the streaming/virtualizer change, addressing the CHANGES_REQUESTED review:

1. Spill-and-read-back coverage: new ReportVirtualizerSpillIT forces
   ilcr.reporting.virtualizer.max-size=1 on an allSchedules print and asserts the
   extracted PDF text is complete — proving virtualized pages are read back off the swap
   file on the async dispatch thread (the default max-size=300 never pages out in tests).

2. Export failures are now observable: RenderedReport.writeTo throws after the 200 +
   application/pdf headers are committed, so no @ExceptionHandler can run. ReportController
   now logs at ERROR with mill/year before rethrowing, so ops can correlate a user's
   "PDF won't open" with a server-side cause.

3. spring.mvc.async.request-timeout set explicitly (120s, env-overridable) — the streamed
   export runs under the async clock; make the ceiling a decision, not a container default.

4. Real disk-backed swap in OpenShift: the container runs readOnlyRootFilesystem and /tmp
   is a memory-backed emptyDir, so java.io.tmpdir would spill to RAM and defeat the heap
   relief. Mount a dedicated DISK-backed report-swap emptyDir (sizeLimit 512Mi, bounded by
   an ephemeral-storage limit) and set ILCR_REPORTING_SWAP_DIR to it.

5. Documented the max-size=300 rationale (conservative page-count default, tunable) and the
   swap-directory→real-ephemeral-volume requirement in application.yml.

All 34 reporting tests green (incl. the new spill IT); checkstyle + compile green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gpascucci

backend/openshift.deploy.yml

@@ -137,6 +137,11 @@ objects:
                   value: "true"
                 - name: ILCR_DATASOURCE_ENABLED
                   value: ${ILCR_DATASOURCE_ENABLED}
+                # Story 29.2 — point the Jasper report virtualizer's swap file at the dedicated
+                # DISK-backed ephemeral volume below, NOT java.io.tmpdir (/tmp here is a Memory-backed
+                # emptyDir, so spilling there would stay in RAM and defeat the heap relief).
+                - name: ILCR_REPORTING_SWAP_DIR
+                  value: /var/tmp/report-swap
                 - name: ORACLEDB_HOST
                   valueFrom:
                     secretKeyRef:
@@ -182,9 +187,13 @@ objects:
                 requests:
                   cpu: ${CPU_REQUEST}
                   memory: ${MEMORY_REQUEST}
-                  ephemeral-storage: "200Mi"
+                  ephemeral-storage: "256Mi"
                 limits:
                   memory: ${MEMORY_LIMIT}
+                  # Bound total ephemeral (node) storage: base image scratch + the 512Mi report-swap
+                  # volume below (Story 29.2). A print that would exceed this is evicted — a bounded,
+                  # observable failure — rather than filling node disk.
+                  ephemeral-storage: "1Gi"
               securityContext:
                 allowPrivilegeEscalation: false
                 runAsNonRoot: true
@@ -196,6 +205,8 @@ objects:
               volumeMounts:
                 - name: tmp
                   mountPath: /tmp
+                - name: report-swap
+                  mountPath: /var/tmp/report-swap
                 - name: api-cert
                   mountPath: /cert
               startupProbe:
@@ -221,6 +232,15 @@ objects:
               emptyDir:
                 medium: Memory
                 sizeLimit: 256Mi
+            # Story 29.2 — dedicated DISK-backed ephemeral volume for the Jasper report swap file.
+            # Deliberately NOT medium: Memory (unlike /tmp above): spilling large report page objects
+            # here relieves the JVM heap onto node ephemeral storage instead of RAM, which is the whole
+            # point of the virtualizer. Bounded by sizeLimit so a runaway print is evicted (a bounded,
+            # observable failure) rather than silently consuming node disk. Writable despite the
+            # read-only root filesystem because it is a mounted volume.
+            - name: report-swap
+              emptyDir:
+                sizeLimit: 512Mi
             # Oracle keystore written by the init container, read by the app
             - name: api-cert
               persistentVolumeClaim:

backend/src/main/java/ca/bc/gov/nrs/ilcr/reporting/ReportController.java

@@ -4,6 +4,8 @@
 import ca.bc.gov.nrs.ilcr.millcontext.MillContextService.MillYearContext;
 import ca.bc.gov.nrs.ilcr.reporting.api.PrintRequest;
 import ca.bc.gov.nrs.ilcr.reporting.api.ReportApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
@@ -29,6 +31,8 @@
 @ConditionalOnProperty(name = "ilcr.datasource.enabled", havingValue = "true")
 public class ReportController implements ReportApi {
 
+  private static final Logger log = LoggerFactory.getLogger(ReportController.class);
+
   private final MillContextService millContextService;
   private final ReportService reportService;
   private final PrintService printService;
@@ -51,7 +55,7 @@ public ResponseEntity<StreamingResponseBody> getSchedule9Pdf(
     // export streams, so a rejected render still produces a problem+json error, never a half-written PDF.
     RenderedReport report = reportService.renderSchedule9(context.millId(), context.year());
     String filename = "schedule9_" + context.millId() + "_" + context.year() + ".pdf";
-    return pdfResponse(filename, report);
+    return pdfResponse(filename, context.millId(), context.year(), report);
   }
 
   @Override
@@ -62,7 +66,7 @@ public ResponseEntity<StreamingResponseBody> printSchedules(
     MillYearContext context = millContextService.validateMillYearActive(millId, year);
     validateSelection(request);
     RenderedReport report = printService.render(context, request);
-    return pdfResponse("schedules_print.pdf", report);
+    return pdfResponse("schedules_print.pdf", context.millId(), context.year(), report);
   }
 
   /**
@@ -71,12 +75,23 @@ public ResponseEntity<StreamingResponseBody> printSchedules(
    * and try-with-resources closes the {@link RenderedReport} on both success and failure, so the
    * virtualizer's swap file is never leaked. The status + headers are set on the ResponseEntity here,
    * before any byte is written, so the attachment filename and content type are always applied.
+   *
+   * <p>An export failure surfaces DIFFERENTLY from the pre-fill guards: by the time bytes are written
+   * the 200 + {@code application/pdf} headers are already committed, so no {@code @ExceptionHandler}
+   * can turn it into a {@code problem+json} — the client just gets a truncated PDF. It is therefore
+   * logged at ERROR with the mill/year (the only server-side signal ops can correlate with a user's
+   * "the PDF won't open") before being rethrown so the container aborts the response. The async render
+   * runs under {@code spring.mvc.async.request-timeout}; a timeout produces the same truncated shape.
    */
   private static ResponseEntity<StreamingResponseBody> pdfResponse(
-      String filename, RenderedReport report) {
+      String filename, long millId, int year, RenderedReport report) {
     StreamingResponseBody body = out -> {
       try (report) {
         report.writeTo(out);
+      } catch (RuntimeException e) {
+        log.error("Report export failed after the response was committed for mill {} year {} ({}) — "
+            + "the client received a truncated PDF", millId, year, filename, e);
+        throw e;
       }
     };
     return ResponseEntity.ok()

backend/src/main/resources/application.yml

@@ -16,6 +16,15 @@ spring:
       # deferred-work.md:180; turning the feature off closes it for every schedule at once rather
       # than leaving Schedule 5 the only safe one.
       accept-float-as-int: false
+  mvc:
+    async:
+      # Story 29.2 — report/print PDFs now stream via StreamingResponseBody, so the export runs under
+      # the async request clock. Set it EXPLICITLY (rather than inheriting the container default) so a
+      # slow or large "all schedules" render has a deliberate ceiling. On timeout Spring completes the
+      # response; for an already-committed streamed PDF that means a truncated body (logged at ERROR in
+      # ReportController) — so size this to the worst-case render, not tight. 120s covers a big combined
+      # print with headroom; raise via SPRING_MVC_ASYNC_REQUEST_TIMEOUT if a legitimate render exceeds it.
+      request-timeout: ${SPRING_MVC_ASYNC_REQUEST_TIMEOUT:120000}
   datasource:
     url: ${SPRING_DATASOURCE_URL:}
     username: ${SPRING_DATASOURCE_USERNAME:}
@@ -81,10 +90,19 @@ ilcr:
   reporting:
     # Story 29.2 — the Jasper swap-file virtualizer that spills report page objects to disk under a
     # large or combined ("all schedules") fill, so the section object graph never pins the JVM heap.
-    # swap-directory empty → the JVM temp dir; max-size is the number of pages kept in memory before
-    # the least-recently-used ones page out. The swap file is deleted when each render finishes.
     virtualizer:
+      # swap-directory: where the per-render swap file lives. Empty → the JVM temp dir (java.io.tmpdir).
+      # In OpenShift the container root fs is read-only AND /tmp is a MEMORY-backed emptyDir, so the
+      # deploy sets ILCR_REPORTING_SWAP_DIR to a dedicated DISK-backed ephemeral volume (see
+      # backend/openshift.deploy.yml) — otherwise "spill to disk" would spill to RAM and defeat the
+      # whole point. The file is created eagerly per render and deleted when the render finishes.
       swap-directory: ${ILCR_REPORTING_SWAP_DIR:}
+      # max-size: pages kept in memory before the least-recently-used ones page out to the swap file.
+      # A deliberate, conservative default (a page count, not a measured byte budget): high enough that
+      # an ordinary single-schedule print never pages out (avoiding swap churn on the common path), low
+      # enough to bound heap on a big combined print. Tune via ILCR_REPORTING_VIRTUALIZER_MAX_SIZE if
+      # profiling shows a real workload wants a different ceiling. (The spill path itself is proven by
+      # ReportVirtualizerSpillIT, which forces max-size=1.)
       max-size: ${ILCR_REPORTING_VIRTUALIZER_MAX_SIZE:300}
 
 features:

backend/src/test/java/ca/bc/gov/nrs/ilcr/reporting/ReportVirtualizerSpillIT.java

@@ -0,0 +1,77 @@
+package ca.bc.gov.nrs.ilcr.reporting;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import ca.bc.gov.nrs.ilcr.support.AbstractOracleIT;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.text.PDFTextStripper;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.web.servlet.MvcResult;
+
+/**
+ * Spill-and-read-back acceptance test for the Jasper virtualizer (Story 29.2). {@code
+ * ilcr.reporting.virtualizer.max-size=1} forces the fill to keep just ONE page in memory and page all
+ * the rest out to the swap file — so a real "all schedules" print (mill 517/2021 has data for all six
+ * in-scope schedules) genuinely exercises the riskiest new path: reading virtualized pages BACK off
+ * the swap file during export, which runs on the async dispatch thread AFTER the request thread that
+ * filled them has returned. The other reporting ITs run at the default max-size (300) and never page
+ * out, so this is the only test that would catch a devirtualization regression.
+ *
+ * <p>The assertion is that the extracted PDF text is still COMPLETE — every section's heading and a
+ * seeded body value from each — proving no page was lost or corrupted across the spill/read-back.
+ * Security is OFF (isolated from authz).
+ */
+@DisplayName("POST /api/v1/reports/print — virtualizer spills to swap and reads back (max-size=1)")
+@TestPropertySource(properties = {
+    "ilcr.security.enabled=false",
+    "ilcr.reporting.virtualizer.max-size=1"
+})
+class ReportVirtualizerSpillIT extends AbstractOracleIT {
+
+  private static final String ENDPOINT = "/api/v1/reports/print";
+
+  @Test
+  @DisplayName("allSchedules print with max-size=1 -> complete PDF (every section survives the spill)")
+  void allSchedules_withAggressiveSpill_producesCompletePdf() throws Exception {
+    // max-size=1 guarantees pages page out to the swap file mid-fill; a complete PDF proves they are
+    // read back correctly on the async export thread.
+    String selection = """
+        {"allSchedules":true,"printScheduleInformation":true,"printComments":true}
+        """;
+    MvcResult result = streamPdf(post(ENDPOINT).param("millId", "517").param("year", "2021")
+            .contentType(MediaType.APPLICATION_JSON).content(selection)
+            .accept(MediaType.APPLICATION_PDF))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_PDF))
+        .andReturn();
+
+    byte[] pdf = result.getResponse().getContentAsByteArray();
+    assertThat(new String(pdf, 0, 4)).isEqualTo("%PDF");
+
+    String text;
+    try (PDDocument document = Loader.loadPDF(pdf)) {
+      text = new PDFTextStripper().getText(document);
+    }
+    // Every in-scope section's heading is present despite the aggressive spill...
+    assertThat(text).contains("Schedule 5:  Camp and Access Expense");
+    assertThat(text).contains("Schedule 6:  Road Management Costs");
+    assertThat(text).contains("Schedule 7A:  Bridge Costs");
+    assertThat(text).contains("Schedule 7B:  Culvert Costs");
+    assertThat(text).contains("Miscellaneous");
+    assertThat(text).contains("Schedule 11:  Basic Silviculture");
+    // ...and a seeded body value from each, proving real page content survived the swap round-trip.
+    assertThat(text).contains("Submitted Camp");        // Schedule 5 camp name (517/2021)
+    assertThat(text).contains("03B");                   // Schedule 6 supply block (RMR 8303)
+    assertThat(text).contains("Harbour Overpass");      // Schedule 7A bridge location (7651)
+    assertThat(text).contains("Pipe Arch");             // Schedule 7B culvert type desc (7851 'PA')
+    assertThat(text).contains("CTR-517");               // Schedule 9 contractor id (record 9110)
+    assertThat(text).contains("Cedar Ridge Reforest");  // Schedule 11 location (V20260816)
+  }
+}