Expose fact query max_rows cap

docs/operator-runbook.md

@@ -313,7 +313,7 @@ wyctl --daemon-url "$BASE_URL" graph create \
 wyctl --daemon-url "$BASE_URL" fact schema register \
   --tenant "$TENANT" --graph "$GRAPH" \
   --namespace shop --relation orders --schema-version 1 \
-  --columns order_id:symbol,amount:int64 \
+  --columns order_id:symbol,amount:int64 --max-rows 1000 \
   --access-token-file "$TOKEN" \
   --guard-timestamp $(date +%s) --guard-loc-class trusted --guard-risk 29
 
@@ -333,6 +333,11 @@ wyctl --daemon-url "$BASE_URL" datalog query \
   --guard-timestamp $(date +%s) --guard-loc-class trusted --guard-risk 29
 ```
 
+Omit `--max-rows` during schema registration to keep the default 1000-row
+Datalog query cap. Set it explicitly for larger materialized JSON queries;
+accepted values are 1 through 1000000, and `wyctl datalog query --limit`
+cannot exceed the registered cap.
+
 To verify recovery, restart `wyrelogd` with the same policy DB, audit DB, key,
 and fact root. Mint a fresh token after restart and run the same
 `wyctl datalog query`; the fact graph is replayed from the per-graph DuckDB fact

tests/test-daemon-http-facts.c

@@ -325,6 +325,29 @@ check_fact_http_contract (WylHandle *handle, const gchar *base_url)
       != WYRELOG_E_NOT_FOUND)
     return 26;
 
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *bad_max_rows_query = g_strdup_printf
+      ("tenant=%s&graph=orders&namespace=shop&relation=bad_rows&"
+      "schema_version=1&max_rows=0&%s", WYL_TENANT_DEFAULT, FACT_GUARD);
+  rc = send_raw (session, "POST", base_url, "/facts/schema/register",
+      bad_max_rows_query, admin_token, schema_body, &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 400 || strstr (body, "\"invalid_schema_request\"") == NULL)
+    return 260;
+
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *too_many_max_rows_query = g_strdup_printf
+      ("tenant=%s&graph=orders&namespace=shop&relation=too_many_rows&"
+      "schema_version=1&max_rows=1000001&%s", WYL_TENANT_DEFAULT,
+      FACT_GUARD);
+  rc = send_raw (session, "POST", base_url, "/facts/schema/register",
+      too_many_max_rows_query, admin_token, schema_body, &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 400 || strstr (body, "\"invalid_schema_request\"") == NULL)
+    return 261;
+
   const gchar *fact_body = "order_id\tamount\no-1\t42\n";
   g_clear_pointer (&body, g_free);
   g_autofree gchar *append_query = g_strdup_printf
@@ -454,6 +477,65 @@ check_fact_http_contract (WylHandle *handle, const gchar *base_url)
       strstr (body, "\"truncated\":true") == NULL)
     return 337;
 
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *create_bulk_query = g_strdup_printf
+      ("tenant=%s&graph=bulk&%s", WYL_TENANT_DEFAULT, FACT_GUARD);
+  rc = send_raw (session, "POST", base_url, "/graphs/create",
+      create_bulk_query, admin_token, NULL, &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 200 || strstr (body, "\"created\":true") == NULL)
+    return 350;
+
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *bulk_schema_query = g_strdup_printf
+      ("tenant=%s&graph=bulk&namespace=shop&relation=orders&"
+      "schema_version=1&max_rows=1100&%s", WYL_TENANT_DEFAULT, FACT_GUARD);
+  rc = send_raw (session, "POST", base_url, "/facts/schema/register",
+      bulk_schema_query, admin_token, schema_body, &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 200 || strstr (body, "\"ok\":true") == NULL)
+    return 351;
+
+  g_autoptr (GString) bulk_rows = g_string_new ("order_id\tamount\n");
+  for (guint i = 0; i < 1105; i++)
+    g_string_append_printf (bulk_rows, "bulk-%u\t%u\n", i, i);
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *bulk_append_query = g_strdup_printf
+      ("tenant=%s&namespace=shop&schema_version=1&batch_id=bulk-1&"
+      "idempotency_key=bulk-key-1&%s", WYL_TENANT_DEFAULT, FACT_GUARD);
+  rc = send_raw (session, "POST", base_url,
+      "/facts/__wr_default/bulk/orders:append", bulk_append_query,
+      admin_token, bulk_rows->str, &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 200 || strstr (body, "\"inserted\":true") == NULL)
+    return 352;
+
+  g_clear_pointer (&body, g_free);
+  g_autofree gchar *bulk_datalog_query = g_strdup_printf ("tenant=%s&%s",
+      WYL_TENANT_DEFAULT, FACT_GUARD);
+  rc = send_raw (session, "POST", base_url,
+      "/datalog/__wr_default/bulk/query", bulk_datalog_query, admin_token,
+      "{\"query\":\"orders(O,A)\",\"output\":\"json\",\"limit\":1005}",
+      &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 200 || strstr (body, "\"row_count\":1005") == NULL ||
+      strstr (body, "\"truncated\":true") == NULL)
+    return 353;
+
+  g_clear_pointer (&body, g_free);
+  rc = send_raw (session, "POST", base_url,
+      "/datalog/__wr_default/bulk/query", bulk_datalog_query, admin_token,
+      "{\"query\":\"orders(O,A)\",\"output\":\"json\"}", &status, &body);
+  if (rc != 0)
+    return rc;
+  if (status != 200 || strstr (body, "\"row_count\":1100") == NULL ||
+      strstr (body, "\"truncated\":true") == NULL)
+    return 354;
+
   /* Retract case 1: normal retract of o-2 -> 200 inserted=true. */
   g_clear_pointer (&body, g_free);
   g_autofree gchar *retract_query = g_strdup_printf

wyrelog/client.h

@@ -156,6 +156,17 @@ wyrelog_error_t wyl_client_fact_schema_register (WylClient * client,
     const WylClientFactColumn * columns,
     gsize n_columns,
     gint64 guard_timestamp, const gchar * guard_loc_class, gint64 guard_risk);
+wyrelog_error_t wyl_client_fact_schema_register_with_max_rows
+    (WylClient * client,
+    const gchar * tenant,
+    const gchar * graph,
+    const gchar * namespace_id,
+    const gchar * relation,
+    guint32 schema_version,
+    const WylClientFactColumn * columns,
+    gsize n_columns,
+    guint max_rows,
+    gint64 guard_timestamp, const gchar * guard_loc_class, gint64 guard_risk);
 wyrelog_error_t wyl_client_fact_put_batch (WylClient * client,
     const gchar * tenant,
     const gchar * graph,

wyrelog/daemon/http.c

@@ -2533,15 +2533,20 @@ schema_register_handler (SoupServer *server, SoupServerMessage *msg,
   const gchar *namespace_id = lookup_required_query_string (query, "namespace");
   const gchar *relation = lookup_required_query_string (query, "relation");
   guint32 schema_version = 0;
+  guint32 max_rows = 0;
   gboolean relation_visible = TRUE;
   const gchar *visible = query != NULL ? g_hash_table_lookup (query,
       "relation_visible") : NULL;
+  const gchar *max_rows_text = query != NULL ? g_hash_table_lookup (query,
+      "max_rows") : NULL;
   if (!wyl_policy_store_tenant_id_is_valid (tenant) ||
       !fact_http_customer_name_is_valid (graph) ||
       !fact_http_customer_name_is_valid (namespace_id) ||
       !fact_http_customer_name_is_valid (relation) ||
       !parse_uint32_query_param (lookup_required_query_string (query,
               "schema_version"), &schema_version) ||
+      (max_rows_text != NULL &&
+          !parse_uint32_query_param (max_rows_text, &max_rows)) ||
       (visible != NULL && !parse_bool_token (visible, &relation_visible))) {
     set_json_error (msg, 400, "invalid_schema_request");
     return;
@@ -2584,6 +2589,11 @@ schema_register_handler (SoupServer *server, SoupServerMessage *msg,
     return;
   }
 
+  wyl_policy_fact_relation_schema_query_t schema_query = {
+    .query_name = relation,
+    .required_permission_id = "wr.datalog.query",
+    .max_rows = max_rows,
+  };
   wyl_policy_fact_relation_schema_options_t opts = {
     .tenant_id = tenant,
     .graph_id = graph,
@@ -2593,6 +2603,8 @@ schema_register_handler (SoupServer *server, SoupServerMessage *msg,
     .relation_visible = relation_visible,
     .columns = columns,
     .n_columns = n_columns,
+    .queries = max_rows > 0 ? &schema_query : NULL,
+    .n_queries = max_rows > 0 ? 1 : 0,
   };
   g_mutex_lock (&ctx->policy_mutation_lock);
   rc = wyl_policy_store_register_fact_relation_schema

wyrelog/fact/query.c

@@ -10,7 +10,6 @@
 
 #define WYL_FACT_DATALOG_QUERY_MAX_LEN 512
 #define WYL_FACT_DATALOG_MAX_ARGS 32
-#define WYL_FACT_DATALOG_HARD_ROW_LIMIT 1000
 
 typedef enum
 {
@@ -438,7 +437,7 @@ wyl_fact_datalog_query_json (WylHandle *handle,
 
   guint limit = opts->limit == 0 ? query_info.max_rows : opts->limit;
   limit = MIN (limit, query_info.max_rows);
-  limit = MIN (limit, (guint) WYL_FACT_DATALOG_HARD_ROW_LIMIT);
+  limit = MIN (limit, (guint) WYL_POLICY_FACT_QUERY_MAX_ROWS);
   if (limit == 0)
     limit = 1;
 

wyrelog/policy/store-private.h

@@ -10,6 +10,9 @@
 
 G_BEGIN_DECLS;
 
+#define WYL_POLICY_FACT_QUERY_DEFAULT_MAX_ROWS 1000
+#define WYL_POLICY_FACT_QUERY_MAX_ROWS 1000000
+
 typedef struct wyl_policy_store_t wyl_policy_store_t;
 
 typedef struct

wyrelog/policy/store.c

@@ -2418,7 +2418,8 @@ validate_fact_graph_options (wyl_policy_store_t *store,
     if (!fact_graph_customer_name_is_valid (query->query_name)
         || !fact_graph_options_relation_exists (opts, query->relation_name)
         || query->required_permission_id == NULL
-        || query->required_permission_id[0] == '\0' || query->max_rows == 0)
+        || query->required_permission_id[0] == '\0' || query->max_rows == 0
+        || query->max_rows > WYL_POLICY_FACT_QUERY_MAX_ROWS)
       return WYRELOG_E_POLICY;
     gboolean permission_exists = FALSE;
     wyrelog_error_t rc = wyl_policy_store_permission_exists (store,
@@ -2901,7 +2902,8 @@ validate_fact_relation_schema_options (wyl_policy_store_t *store,
     const wyl_policy_fact_relation_schema_query_t *query = &opts->queries[i];
     if (!fact_graph_customer_name_is_valid (query->query_name)
         || query->required_permission_id == NULL
-        || query->required_permission_id[0] == '\0' || query->max_rows == 0)
+        || query->required_permission_id[0] == '\0' || query->max_rows == 0
+        || query->max_rows > WYL_POLICY_FACT_QUERY_MAX_ROWS)
       return WYRELOG_E_POLICY;
     gboolean permission_exists = FALSE;
     wyrelog_error_t rc = wyl_policy_store_permission_exists (store,
@@ -3064,7 +3066,7 @@ wyl_policy_store_register_fact_relation_schema (wyl_policy_store_t *store,
     const wyl_policy_fact_relation_schema_query_t default_query = {
       .query_name = opts->relation_name,
       .required_permission_id = "wr.datalog.query",
-      .max_rows = 1000,
+      .max_rows = WYL_POLICY_FACT_QUERY_DEFAULT_MAX_ROWS,
     };
     rc = insert_fact_relation_query_metadata (store, opts, &default_query);
   }

wyrelog/wyctl/wyctl.c

@@ -87,6 +87,7 @@ typedef struct
   gchar *relation;
   gchar *schema_version_arg;
   gchar *columns_arg;
+  gchar *max_rows_arg;
   gchar *access_token_file;
   gchar *guard_timestamp_arg;
   gchar *guard_loc_class;
@@ -1436,6 +1437,8 @@ run_fact_schema_register (const WyctlOptions *global_opts, gint argc,
         "Schema version", "N"},
     {"columns", 0, 0, G_OPTION_ARG_STRING, &opts.columns_arg,
         "Columns as name:type,...", "COLUMNS"},
+    {"max-rows", 0, 0, G_OPTION_ARG_STRING, &opts.max_rows_arg,
+        "Maximum rows authorized for the default relation query", "N"},
     {"access-token-file", 0, 0, G_OPTION_ARG_STRING, &opts.access_token_file,
         "Bearer access token file", "PATH"},
     {"guard-timestamp", 0, 0, G_OPTION_ARG_STRING,
@@ -1485,6 +1488,12 @@ run_fact_schema_register (const WyctlOptions *global_opts, gint argc,
     g_printerr ("wyctl: invalid --schema-version\n");
     return 2;
   }
+  guint32 max_rows = 0;
+  if (opts.max_rows_arg != NULL && !parse_positive_uint32 (opts.max_rows_arg,
+          &max_rows)) {
+    g_printerr ("wyctl: invalid --max-rows\n");
+    return 2;
+  }
   WylClientFactColumn *columns = NULL;
   gsize n_columns = 0;
   if (!parse_columns_arg (opts.columns_arg, &columns, &n_columns)) {
@@ -1505,9 +1514,10 @@ run_fact_schema_register (const WyctlOptions *global_opts, gint argc,
     client_fact_columns_clear (columns, n_columns);
     return client_rc;
   }
-  wyrelog_error_t rc = wyl_client_fact_schema_register (client, tenant,
+  wyrelog_error_t rc = wyl_client_fact_schema_register_with_max_rows (client,
+      tenant,
       graph, opts.namespace_id, opts.relation, schema_version, columns,
-      n_columns, guard_timestamp, opts.guard_loc_class, guard_risk);
+      n_columns, max_rows, guard_timestamp, opts.guard_loc_class, guard_risk);
   client_fact_columns_clear (columns, n_columns);
   int exit_rc = fact_remote_exit (client, "fact schema register", rc,
       "schema_register_failed");

wyrelog/wyl-client.c

@@ -826,11 +826,12 @@ wyl_client_graph_create (WylClient *client, const gchar *tenant,
 }
 
 wyrelog_error_t
-wyl_client_fact_schema_register (WylClient *client, const gchar *tenant,
+wyl_client_fact_schema_register_with_max_rows (WylClient *client,
+    const gchar *tenant,
     const gchar *graph, const gchar *namespace_id, const gchar *relation,
     guint32 schema_version, const WylClientFactColumn *columns,
-    gsize n_columns, gint64 guard_timestamp, const gchar *guard_loc_class,
-    gint64 guard_risk)
+    gsize n_columns, guint max_rows, gint64 guard_timestamp,
+    const gchar *guard_loc_class, gint64 guard_risk)
 {
   if (graph == NULL || graph[0] == '\0' || namespace_id == NULL ||
       namespace_id[0] == '\0' || relation == NULL || relation[0] == '\0' ||
@@ -864,11 +865,14 @@ wyl_client_fact_schema_register (WylClient *client, const gchar *tenant,
       NULL, TRUE);
   g_autofree gchar *escaped_relation = g_uri_escape_string (relation, NULL,
       TRUE);
-  g_autofree gchar *uri = g_strdup_printf
-      ("%s/facts/schema/register?%s&graph=%s&namespace=%s&relation=%s"
+  g_autoptr (GString) uri = g_string_new (NULL);
+  g_string_printf (uri,
+      "%s/facts/schema/register?%s&graph=%s&namespace=%s&relation=%s"
       "&schema_version=%u", base_url, guard_query, escaped_graph,
       escaped_namespace, escaped_relation, schema_version);
-  g_autoptr (SoupMessage) message = soup_message_new ("POST", uri);
+  if (max_rows > 0)
+    g_string_append_printf (uri, "&max_rows=%u", max_rows);
+  g_autoptr (SoupMessage) message = soup_message_new ("POST", uri->str);
   if (message == NULL)
     return WYRELOG_E_INVALID;
   client_fact_attach_auth (message, access_token);
@@ -878,6 +882,18 @@ wyl_client_fact_schema_register (WylClient *client, const gchar *tenant,
   return client_send_fact_message (client, message, NULL);
 }
 
+wyrelog_error_t
+wyl_client_fact_schema_register (WylClient *client, const gchar *tenant,
+    const gchar *graph, const gchar *namespace_id, const gchar *relation,
+    guint32 schema_version, const WylClientFactColumn *columns,
+    gsize n_columns, gint64 guard_timestamp, const gchar *guard_loc_class,
+    gint64 guard_risk)
+{
+  return wyl_client_fact_schema_register_with_max_rows (client, tenant, graph,
+      namespace_id, relation, schema_version, columns, n_columns, 0,
+      guard_timestamp, guard_loc_class, guard_risk);
+}
+
 wyrelog_error_t
 wyl_client_fact_put_batch (WylClient *client, const gchar *tenant,
     const gchar *graph, const gchar *namespace_id, const gchar *relation,