fix: Reduce Solana data dashboard Databricks traffic (#1602)

* fix: update data fetching logic to remove unnecessary parameters and improve cache handling

* fix: simplify cache key generation by removing unused refresh logic

* fix: improve error handling in in-memory Databricks data caching

Author: h4rkl

apps/web/src/app/[locale]/data/solana-data-dashboard.tsx

@@ -1664,7 +1664,7 @@ function aggregate(sum: number, count: number, aggregation: Aggregation) {
 }
 
 async function fetchData(url: string, errorMessages: DataFetchErrorMessages) {
-  const response = await fetch(url, { cache: "no-store" });
+  const response = await fetch(url);
   const payload = await readDataPayload(response, errorMessages);
 
   if (!response.ok) {

apps/web/src/app/api/databricks/data/route.ts

@@ -20,15 +20,13 @@ export const dynamic = "force-dynamic";
 export const revalidate = 0;
 
 const DATABRICKS_CACHE_REVALIDATE_SECONDS = 12 * 60 * 60;
-const BROWSER_CACHE_SECONDS = 60;
 const EDGE_STALE_SECONDS = 24 * 60 * 60;
 const DEFAULT_RANGE_DAYS = 90;
+const MAX_RANGE_DAYS = Math.max(...rangeOptions.map((option) => option.value));
+const DATABRICKS_CACHE_KEY_VERSION = "solana-data-databricks-metric-rows-v4";
 const IS_PRODUCTION = isProduction();
 const NO_STORE_CACHE_CONTROL = "no-store, max-age=0";
 const DAY_IN_MS = 24 * 60 * 60 * 1000;
-const DATA_REFRESH_TIME_ZONE = "America/New_York";
-// Source jobs run around 9 AM and 9 PM Eastern; roll dashboard caches an hour later.
-const DATA_REFRESH_HOURS = [10, 22] as const;
 const DATA_UNAVAILABLE_ERROR =
   "Solana data is unavailable right now. Try again in a moment.";
 
@@ -56,7 +54,7 @@ export async function GET(request: NextRequest) {
   const rangeDays = parseRangeDays(request.nextUrl.searchParams.get("days"));
 
   try {
-    const result = await getDatabricksData(configResult.config, rangeDays);
+    const result = await getDatabricksData(configResult.config);
 
     return json<DataApiResponse>(
       buildDataResponse(result, rangeDays),
@@ -105,33 +103,29 @@ function filterRowsByRange(rows: MetricRow[], rangeDays: number) {
   });
 }
 
-function getDatabricksData(config: DatabricksConfig, rangeDays: number) {
+function getDatabricksData(config: DatabricksConfig) {
   return IS_PRODUCTION
-    ? getCachedDatabricksData(config, rangeDays)
-    : fetchDatabricksData(config, rangeDays);
+    ? getCachedDatabricksData(config)
+    : fetchDatabricksData(config);
 }
 
-function getCachedDatabricksData(config: DatabricksConfig, rangeDays: number) {
+function getCachedDatabricksData(config: DatabricksConfig) {
+  const dataCacheKey = getDatabricksCacheKey(config);
+  const cacheKeyParts = [DATABRICKS_CACHE_KEY_VERSION, dataCacheKey];
+
   return unstable_cache(
-    () => fetchDatabricksData(config, rangeDays),
-    [
-      "solana-data-databricks-metric-rows-v3",
-      getDatabricksRefreshCacheKey(),
-      getDatabricksCacheKey(config, rangeDays),
-    ],
+    () => getInMemoryCachedDatabricksData(config, cacheKeyParts.join("|")),
+    cacheKeyParts,
     {
       revalidate: DATABRICKS_CACHE_REVALIDATE_SECONDS,
       tags: ["solana-data-databricks"],
     },
   )();
 }
 
-async function fetchDatabricksData(
-  config: DatabricksConfig,
-  rangeDays: number,
-) {
+async function fetchDatabricksData(config: DatabricksConfig) {
   const result = await getDatabricksSqlMetricRows(config, {
-    lookbackDays: rangeDays,
+    lookbackDays: MAX_RANGE_DAYS,
     metricNames,
   });
 
@@ -141,139 +135,64 @@ async function fetchDatabricksData(
   };
 }
 
-function getDatabricksCacheKey(config: DatabricksConfig, rangeDays: number) {
-  return [
-    config.serverHostname,
-    config.httpPath,
-    config.warehouseId,
-    rangeDays,
-    metricNames.join(","),
-  ].join("|");
-}
-
-function getDatabricksRefreshCacheKey(now = new Date()) {
-  const parts = getTimeZoneDateTimeParts(now, DATA_REFRESH_TIME_ZONE);
-  const latestRefreshHour = getLatestScheduledRefreshHour(parts.hour);
-  const refreshHour =
-    latestRefreshHour ?? DATA_REFRESH_HOURS[DATA_REFRESH_HOURS.length - 1];
-  const refreshDate = new Date(
-    Date.UTC(
-      parts.year,
-      parts.month - 1,
-      parts.day + (latestRefreshHour === undefined ? -1 : 0),
-      refreshHour,
-    ),
-  );
+const databricksDataRequests = new Map<
+  string,
+  Promise<Awaited<ReturnType<typeof fetchDatabricksData>>>
+>();
 
-  return `${refreshDate.toISOString().slice(0, 10)}-${String(refreshHour).padStart(2, "0")}`;
-}
+function getInMemoryCachedDatabricksData(
+  config: DatabricksConfig,
+  cacheKey: string,
+) {
+  const cachedRequest = databricksDataRequests.get(cacheKey);
 
-function getSuccessCacheControl(now = new Date()) {
-  if (!IS_PRODUCTION) {
-    return NO_STORE_CACHE_CONTROL;
+  if (cachedRequest) {
+    return cachedRequest;
   }
 
-  return [
-    "public",
-    `max-age=${BROWSER_CACHE_SECONDS}`,
-    `s-maxage=${getSecondsUntilNextScheduledRefresh(now)}`,
-    `stale-while-revalidate=${EDGE_STALE_SECONDS}`,
-  ].join(", ");
-}
+  pruneDatabricksDataRequests(cacheKey);
 
-function getSecondsUntilNextScheduledRefresh(now = new Date()) {
-  const parts = getTimeZoneDateTimeParts(now, DATA_REFRESH_TIME_ZONE);
-  const nextRefresh = getScheduledRefreshDate(parts, now);
-  const seconds = Math.ceil((nextRefresh.getTime() - now.getTime()) / 1000);
+  const request = fetchDatabricksData(config);
 
-  return Math.max(1, seconds);
-}
-
-function getScheduledRefreshDate(
-  parts: ReturnType<typeof getTimeZoneDateTimeParts>,
-  now: Date,
-) {
-  const nextRefreshHour = getNextScheduledRefreshHour(parts.hour);
-  const refreshHour = nextRefreshHour ?? DATA_REFRESH_HOURS[0];
-  const dayOffset = nextRefreshHour === undefined ? 1 : 0;
-
-  return zonedTimeToUtc(
-    parts.year,
-    parts.month,
-    parts.day + dayOffset,
-    refreshHour,
-    DATA_REFRESH_TIME_ZONE,
-    now,
+  request.then(
+    () => databricksDataRequests.delete(cacheKey),
+    () => databricksDataRequests.delete(cacheKey),
   );
-}
 
-function getLatestScheduledRefreshHour(hour: number) {
-  return DATA_REFRESH_HOURS.findLast((refreshHour) => hour >= refreshHour);
-}
+  databricksDataRequests.set(cacheKey, request);
 
-function getNextScheduledRefreshHour(hour: number) {
-  return DATA_REFRESH_HOURS.find((refreshHour) => hour < refreshHour);
+  return request;
 }
 
-function zonedTimeToUtc(
-  year: number,
-  month: number,
-  day: number,
-  hour: number,
-  timeZone: string,
-  referenceDate: Date,
-) {
-  const utcGuess = new Date(Date.UTC(year, month - 1, day, hour));
-  const firstPass = new Date(
-    utcGuess.getTime() - getTimeZoneOffsetMs(utcGuess, timeZone),
-  );
-  const secondPass = new Date(
-    utcGuess.getTime() - getTimeZoneOffsetMs(firstPass, timeZone),
-  );
-
-  return Math.abs(secondPass.getTime() - referenceDate.getTime()) <
-    DAY_IN_MS * 2
-    ? secondPass
-    : firstPass;
+function pruneDatabricksDataRequests(activeCacheKey: string) {
+  for (const cacheKey of databricksDataRequests.keys()) {
+    if (cacheKey !== activeCacheKey) {
+      databricksDataRequests.delete(cacheKey);
+    }
+  }
 }
 
-function getTimeZoneOffsetMs(date: Date, timeZone: string) {
-  const parts = getTimeZoneDateTimeParts(date, timeZone);
-  const zonedAsUtc = Date.UTC(
-    parts.year,
-    parts.month - 1,
-    parts.day,
-    parts.hour,
-    parts.minute,
-    parts.second,
-  );
-
-  return zonedAsUtc - date.getTime();
+function getDatabricksCacheKey(config: DatabricksConfig) {
+  return [
+    config.serverHostname,
+    config.httpPath,
+    config.warehouseId,
+    MAX_RANGE_DAYS,
+    metricNames.join(","),
+  ].join("|");
 }
 
-function getTimeZoneDateTimeParts(date: Date, timeZone: string) {
-  const parts = new Intl.DateTimeFormat("en-US", {
-    day: "2-digit",
-    hour: "2-digit",
-    hourCycle: "h23",
-    minute: "2-digit",
-    month: "2-digit",
-    second: "2-digit",
-    timeZone,
-    year: "numeric",
-  }).formatToParts(date);
-  const valueByType = new Map(
-    parts.map((part) => [part.type, Number(part.value)]),
-  );
+function getSuccessCacheControl() {
+  if (!IS_PRODUCTION) {
+    return NO_STORE_CACHE_CONTROL;
+  }
 
-  return {
-    day: valueByType.get("day") ?? 1,
-    hour: valueByType.get("hour") ?? 0,
-    minute: valueByType.get("minute") ?? 0,
-    month: valueByType.get("month") ?? 1,
-    second: valueByType.get("second") ?? 0,
-    year: valueByType.get("year") ?? 1970,
-  };
+  return [
+    "public",
+    "max-age=0",
+    `s-maxage=${DATABRICKS_CACHE_REVALIDATE_SECONDS}`,
+    `stale-while-revalidate=${EDGE_STALE_SECONDS}`,
+  ].join(", ");
 }
 
 function getConfigErrorResponse(