support cursor pagination on data endpoint

Author: wheelsandcogs

src/controllers/consumer-v2.ts

@@ -17,6 +17,7 @@ import { DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE } from '../utils/page-defaults';
 import { clamp } from '../utils/clamp';
 import { ConsumerRevisionDTO } from '../dtos/consumer-revision-dto';
 import {
+  BuildDataQueryResult,
   buildDataQuery,
   sendCsv,
   sendExcel,
@@ -157,8 +158,16 @@ export const getPublishedDatasetData = async (req: Request, res: Response, next:
       ? await QueryStoreRepository.getById(filterId)
       : await QueryStoreRepository.getByRequest(dataset.id, publishedRevision.id, dataOptions);
 
-    const query = await buildDataQuery(queryStore, pageOptions);
-    await sendFormattedResponse(query, queryStore, pageOptions, res);
+    // ensurePublishedDataset loads a lean dataset (no factTable). buildDataQuery
+    // needs the fact-table columns to resolve a deterministic sort plan for
+    // both cursor-mode and offset-mode pagination, so reload with factTable
+    // when it isn't already present.
+    const datasetForBuild = dataset.factTable
+      ? dataset
+      : await PublishedDatasetRepository.getById(dataset.id, { factTable: true });
+
+    const buildResult = await buildDataQuery(queryStore, pageOptions, datasetForBuild);
+    await sendFormattedResponse(buildResult, queryStore, pageOptions, res);
   } catch (err) {
     if (res.headersSent) {
       logger.error(err, 'Error detected fetching data after headers already sent');
@@ -521,29 +530,30 @@ export const getPublicationHistory = async (_req: Request, res: Response): Promi
 };
 
 export const sendFormattedResponse = async (
-  query: string,
+  buildResult: BuildDataQueryResult,
   queryStore: QueryStore,
   pageOptions: PageOptions,
   res: Response
 ): Promise<void> => {
+  const sql = buildResult.sql;
   switch (pageOptions.format) {
     case OutputFormats.Frontend:
       // consumer view: load via PublishedDatasetRepository (consumer pool, published-only)
       return sendFrontendView(
-        query,
+        buildResult,
         queryStore,
         pageOptions,
         res,
         PublishedDatasetRepository.getById.bind(PublishedDatasetRepository)
       );
     case OutputFormats.Csv:
-      return sendCsv(query, queryStore, res);
+      return sendCsv(sql, queryStore, res);
     case OutputFormats.Excel:
-      return sendExcel(query, queryStore, res);
+      return sendExcel(sql, queryStore, res);
     case OutputFormats.Json:
-      return sendJson(query, queryStore, res);
+      return sendJson(sql, queryStore, res);
     case OutputFormats.Html:
-      return sendHtml(query, queryStore, res);
+      return sendHtml(sql, queryStore, res);
     default:
       res.status(400).json({ error: 'Format not supported' });
   }

src/controllers/dataset.ts

@@ -68,7 +68,14 @@ import { QueryStoreRepository } from '../repositories/query-store';
 import { parsePageOptions } from '../utils/parse-page-options';
 import { resolvePreviewRevisionId } from '../utils/revision';
 import { OutputFormats } from '../enums/output-formats';
-import { buildDataQuery, sendCsv, sendExcel, sendFrontendView, sendJson } from '../services/consumer-view-v2';
+import {
+  BuildDataQueryResult,
+  buildDataQuery,
+  sendCsv,
+  sendExcel,
+  sendFrontendView,
+  sendJson
+} from '../services/consumer-view-v2';
 import { QueryStore } from '../entities/query-store';
 import { PageOptions } from '../interfaces/page-options';
 import { rebuildAllFilterTablesForRevisions, rebuildCubesForRevisions } from '../services/revision';
@@ -317,8 +324,8 @@ export const datasetPreview = async (req: Request, res: Response, next: NextFunc
       ? await QueryStoreRepository.getById(filterId)
       : await QueryStoreRepository.getByRequest(dataset.id, previewRevisionId, dataOptions);
 
-    const query = await buildDataQuery(queryStore, pageOptions);
-    await sendFormattedResponse(query, queryStore, pageOptions, res);
+    const buildResult = await buildDataQuery(queryStore, pageOptions, dataset);
+    await sendFormattedResponse(buildResult, queryStore, pageOptions, res);
   } catch (err) {
     if (err instanceof NotFoundException || err instanceof BadRequestException) {
       return next(err);
@@ -329,21 +336,28 @@ export const datasetPreview = async (req: Request, res: Response, next: NextFunc
 };
 
 export const sendFormattedResponse = async (
-  query: string,
+  buildResult: BuildDataQueryResult,
   queryStore: QueryStore,
   pageOptions: PageOptions,
   res: Response
 ): Promise<void> => {
+  const sql = buildResult.sql;
   switch (pageOptions.format) {
     case OutputFormats.Frontend:
       // publisher preview: load via DatasetRepository so drafts resolve and we stay on the publisher pool
-      return sendFrontendView(query, queryStore, pageOptions, res, DatasetRepository.getById.bind(DatasetRepository));
+      return sendFrontendView(
+        buildResult,
+        queryStore,
+        pageOptions,
+        res,
+        DatasetRepository.getById.bind(DatasetRepository)
+      );
     case OutputFormats.Csv:
-      return sendCsv(query, queryStore, res);
+      return sendCsv(sql, queryStore, res);
     case OutputFormats.Excel:
-      return sendExcel(query, queryStore, res);
+      return sendExcel(sql, queryStore, res);
     case OutputFormats.Json:
-      return sendJson(query, queryStore, res);
+      return sendJson(sql, queryStore, res);
     default:
       res.status(400).json({ error: 'Format not supported' });
   }

src/interfaces/page-options.ts

@@ -9,4 +9,8 @@ export interface PageOptions {
   locale: Locale;
   y?: string[] | string;
   x?: string[] | string;
+  // Opaque keyset-pagination cursor. When supplied, the caller is opting in
+  // to cursor-based pagination and page_number is ignored beyond its default
+  // of 1. Mutually exclusive with non-default page_number.
+  cursor?: string;
 }

src/routes/consumer/v2/api.ts

@@ -371,7 +371,11 @@ publicApiV2Router.get(
       specified by the required <code>format</code> parameter. <code>csv</code> and <code>xlsx</code>
       return the full dataset as a streamed file attachment. <code>json</code>, <code>frontend</code>
       and <code>html</code> return paginated responses (default <code>page_size</code> 100, max 10,000).
-      To apply filters, first create a filter via POST /{dataset_id}/data, then use
+      <br><br>Pagination supports two modes: <code>page_number</code>-based (OFFSET) for shallow paging,
+      and opaque <code>cursor</code>-based (keyset) for efficient deep paging. The response carries both
+      <code>next_cursor</code> and <code>prev_cursor</code> in <code>page_info</code> so clients can
+      switch into cursor mode at any point. The two modes are mutually exclusive within a single request.
+      <br><br>To apply filters, first create a filter via POST /{dataset_id}/data, then use
       GET /{dataset_id}/data/{filter_id}."
     #swagger.autoQuery = false
     #swagger.parameters['$ref'] = [
@@ -380,7 +384,8 @@ publicApiV2Router.get(
       '#/components/parameters/output_format',
       '#/components/parameters/page_number',
       '#/components/parameters/page_size',
-      '#/components/parameters/sort_by'
+      '#/components/parameters/sort_by',
+      '#/components/parameters/cursor'
     ]
     #swagger.responses[200] = {
       description: 'A JSON array of data row objects',
@@ -405,7 +410,8 @@ publicApiV2Router.get(
       chosen options for a specific filter ID. The required <code>format</code> parameter selects the response shape:
       <code>csv</code> and <code>xlsx</code> stream the full filtered dataset, while <code>json</code>,
       <code>frontend</code> and <code>html</code> return paginated responses (default <code>page_size</code> 100,
-      max 10,000)."
+      max 10,000). Both <code>page_number</code> and opaque <code>cursor</code> pagination are supported (mutually
+      exclusive within a single request) — see GET /{dataset_id}/data for details."
     #swagger.autoQuery = false
     #swagger.parameters['$ref'] = [
       '#/components/parameters/language',
@@ -414,6 +420,7 @@ publicApiV2Router.get(
       '#/components/parameters/page_number',
       '#/components/parameters/page_size',
       '#/components/parameters/sort_by',
+      '#/components/parameters/cursor',
       '#/components/parameters/filter_id'
     ]
     #swagger.responses[200] = {

src/routes/consumer/v2/openapi-cy.json

@@ -291,6 +291,9 @@
           },
           {
             "$ref": "#/components/parameters/sort_by"
+          },
+          {
+            "$ref": "#/components/parameters/cursor"
           }
         ],
         "responses": {
@@ -374,6 +377,9 @@
           {
             "$ref": "#/components/parameters/sort_by"
           },
+          {
+            "$ref": "#/components/parameters/cursor"
+          },
           {
             "$ref": "#/components/parameters/filter_id"
           }
@@ -572,6 +578,16 @@
         },
         "example": "title:asc,last_updated_at:desc"
       },
+      "cursor": {
+        "name": "cursor",
+        "in": "query",
+        "description": "Opaque keyset cursor returned in <code>page_info.next_cursor</code> / <code>prev_cursor</code>. Pass it back to fetch the next (or previous) slice of rows without paying the OFFSET cost. Cursors are bound to a specific dataset revision, language and sort order — they become invalid if any of those change, and the server will return <code>400</code> in that case. Mutually exclusive with <code>page_number</code> > 1.",
+        "required": false,
+        "schema": {
+          "type": "string",
+          "maxLength": 2048
+        }
+      },
       "filter": {
         "name": "filter",
         "in": "query",

src/routes/consumer/v2/openapi-en.json

@@ -272,7 +272,7 @@
           "Data"
         ],
         "summary": "Get paginated data for a dataset",
-        "description": "Returns rows for the latest published revision in the format  specified by the required <code>format</code> parameter. <code>csv</code> and <code>xlsx</code>  return the full dataset as a streamed file attachment. <code>json</code>, <code>frontend</code>  and <code>html</code> return paginated responses (default <code>page_size</code> 100, max 10,000).  To apply filters, first create a filter via POST /{dataset_id}/data, then use  GET /{dataset_id}/data/{filter_id}.",
+        "description": "Returns rows for the latest published revision in the format  specified by the required <code>format</code> parameter. <code>csv</code> and <code>xlsx</code>  return the full dataset as a streamed file attachment. <code>json</code>, <code>frontend</code>  and <code>html</code> return paginated responses (default <code>page_size</code> 100, max 10,000).  <br><br>Pagination supports two modes: <code>page_number</code>-based (OFFSET) for shallow paging,  and opaque <code>cursor</code>-based (keyset) for efficient deep paging. The response carries both  <code>next_cursor</code> and <code>prev_cursor</code> in <code>page_info</code> so clients can  switch into cursor mode at any point. The two modes are mutually exclusive within a single request.  <br><br>To apply filters, first create a filter via POST /{dataset_id}/data, then use  GET /{dataset_id}/data/{filter_id}.",
         "parameters": [
           {
             "$ref": "#/components/parameters/language"
@@ -291,6 +291,9 @@
           },
           {
             "$ref": "#/components/parameters/sort_by"
+          },
+          {
+            "$ref": "#/components/parameters/cursor"
           }
         ],
         "responses": {
@@ -354,7 +357,7 @@
           "Data"
         ],
         "summary": "Get a filtered data table for a dataset",
-        "description": "Returns current data for a published dataset, filtered and displayed according to the  chosen options for a specific filter ID. The required <code>format</code> parameter selects the response shape:  <code>csv</code> and <code>xlsx</code> stream the full filtered dataset, while <code>json</code>,  <code>frontend</code> and <code>html</code> return paginated responses (default <code>page_size</code> 100,  max 10,000).",
+        "description": "Returns current data for a published dataset, filtered and displayed according to the  chosen options for a specific filter ID. The required <code>format</code> parameter selects the response shape:  <code>csv</code> and <code>xlsx</code> stream the full filtered dataset, while <code>json</code>,  <code>frontend</code> and <code>html</code> return paginated responses (default <code>page_size</code> 100,  max 10,000). Both <code>page_number</code> and opaque <code>cursor</code> pagination are supported (mutually  exclusive within a single request) — see GET /{dataset_id}/data for details.",
         "parameters": [
           {
             "$ref": "#/components/parameters/language"
@@ -374,6 +377,9 @@
           {
             "$ref": "#/components/parameters/sort_by"
           },
+          {
+            "$ref": "#/components/parameters/cursor"
+          },
           {
             "$ref": "#/components/parameters/filter_id"
           }
@@ -572,6 +578,16 @@
         },
         "example": "title:asc,last_updated_at:desc"
       },
+      "cursor": {
+        "name": "cursor",
+        "in": "query",
+        "description": "Opaque keyset cursor returned in <code>page_info.next_cursor</code> / <code>prev_cursor</code>. Pass it back to fetch the next (or previous) slice of rows without paying the OFFSET cost. Cursors are bound to a specific dataset revision, language and sort order — they become invalid if any of those change, and the server will return <code>400</code> in that case. Mutually exclusive with <code>page_number</code> > 1.",
+        "required": false,
+        "schema": {
+          "type": "string",
+          "maxLength": 2048
+        }
+      },
       "filter": {
         "name": "filter",
         "in": "query",

src/routes/consumer/v2/schema.ts

@@ -94,6 +94,18 @@ export const schemaV2 = {
         schema: { type: 'string' },
         example: 'title:asc,last_updated_at:desc'
       },
+      cursor: {
+        name: 'cursor',
+        in: 'query',
+        description:
+          `Opaque keyset cursor returned in <code>page_info.next_cursor</code> / <code>prev_cursor</code>. ` +
+          `Pass it back to fetch the next (or previous) slice of rows without paying the OFFSET cost. ` +
+          `Cursors are bound to a specific dataset revision, language and sort order — they become invalid ` +
+          `if any of those change, and the server will return <code>400</code> in that case. Mutually exclusive ` +
+          `with <code>page_number</code> > 1.`,
+        required: false,
+        schema: { type: 'string', maxLength: 2048 }
+      },
       filter: {
         name: 'filter',
         in: 'query',