feat: support dynamic map file size limits based on song length (#78)

Author: bitnimble

AGENTS.md

@@ -7,6 +7,8 @@ This is a website that allows users to host custom maps and songs for a rhythm d
 Custom maps consist of a zip file, which contains a .rlrr metadata file along with the audio tracks for the song. The audio tracks can either be the song itself, or the audio stems of the song that can allow the game to play the song without any drum track (as the player will be drumming along themselves).
 The codebase uses Docker to run third-party services locally (Minio for a local S3 instance), the local Supabase CLI for running the Supabase database locally, and the standad Next.js dev mode to run the backend and frontend locally.
 
+The official .rlrr schema (fields and types) is at https://raw.githubusercontent.com/emretanirgan/ParadiddleUtilities/refs/heads/master/docs/rlrrschema.json - fetch it if you need to reference what a .rlrr file contains.
+
 # Tech stack
 
 - Typescript, Next.js with App Routing running on Vercel, Postgres running on Supabase

jest.config.unit.ts

@@ -11,6 +11,11 @@ const config: Config = {
   testMatch: ['**/*.unit.test.[jt]s?(x)'],
   setupFilesAfterEnv: ['<rootDir>/src/services/jest_setup.unit.ts'],
   modulePaths: ['<rootDir>/src'],
+  moduleNameMapper: {
+    // Jest's resolver doesn't read the "exports" map, and zip.js' subpaths are ESM-only. The CJS
+    // bundle is the same native-codec build, just not tree-shaken.
+    '^@zip\\.js/zip\\.js/lib/zip-core-native\\.js$': '@zip.js/zip.js/index-native.cjs',
+  },
   transform: {
     '^.+\\.(t|j)sx?$': [
       '@swc/jest',

src/app/api/maps/submit/complete/complete_upload.ts

@@ -1,5 +1,6 @@
 import { MapValidity } from 'schema/maps';
 import { actionError } from 'services/helpers';
+import { MAX_MAP_FILE_SIZE, formatMaxFileSize } from 'services/maps/map_size';
 import { submitErrorMap } from 'services/maps/maps_repo';
 import { getServerContext } from 'services/server_context';
 import { getUserSession } from 'services/session/session';
@@ -81,11 +82,11 @@ export async function completeMapUpload(id: string, isReupload: boolean) {
     });
   }
   const archive = openMapResult.value;
-  if (archive.size > 1024 * 1024 * 100) {
+  // Cheap upfront reject; the length-based limit is applied during validation, once the rlrr is read.
+  if (archive.size > MAX_MAP_FILE_SIZE) {
     await cleanupFailedUpload();
-    // 100MiB. We use MiB because that's what Windows displays in Explorer and therefore what users will expect.
     return actionError({
-      message: 'File is over the filesize limit (100MB)',
+      message: `File is over the filesize limit (${formatMaxFileSize(MAX_MAP_FILE_SIZE)})`,
       errorBody: {},
     });
   }

src/services/maps/map_size.ts

@@ -0,0 +1,44 @@
+const MIB = 1024 * 1024;
+
+/** Ceiling regardless of song length. */
+export const MAX_MAP_FILE_SIZE = 500 * MIB;
+
+// Rates are for the song+drums pair: FLAC ~13MiB/min, 256kbps Opus ~3.7MiB/min. The base fits FLAC
+// for a standard-length song, and the per-minute rate is well under it, so past ~8 minutes a map has
+// to be lossy.
+const BASE_ALLOWANCE = 80 * MIB;
+const ALLOWANCE_PER_MINUTE = 3 * MIB;
+// `length` is optional in the rlrr schema, so a map that predates it can't be sized. Falling back to
+// the old flat limit keeps those uploadable without making an absent length the most generous
+// budget going - which would just be an incentive to drop the field.
+const UNKNOWN_LENGTH_ALLOWANCE = 100 * MIB;
+
+export function maxMapFileSize(seconds: number | undefined): number {
+  if (seconds == null || seconds <= 0 || !Number.isFinite(seconds)) {
+    return UNKNOWN_LENGTH_ALLOWANCE;
+  }
+  return Math.min(MAX_MAP_FILE_SIZE, BASE_ALLOWANCE + (seconds / 60) * ALLOWANCE_PER_MINUTE);
+}
+
+/** Longest of the lengths the difficulties declare; they drift when recorded separately. */
+export function longestSongLength(lengths: (number | undefined)[]): number | undefined {
+  const declared = lengths.filter((l): l is number => l != null);
+  return declared.length === 0 ? undefined : Math.max(...declared);
+}
+
+export function overBudgetMessage(limit: number): string {
+  return `File is over the ${formatMaxFileSize(limit)} limit for a song of this length`;
+}
+
+/**
+ * A limit, as a size the user can compare their file against. Always rounds down, so a file the
+ * size it names is never over the limit it names.
+ */
+export function formatMaxFileSize(bytes: number): string {
+  // Labelled MB but computed as MiB: matches what Windows Explorer shows users.
+  const mib = bytes / MIB;
+  // A whole megabyte is a big share of a small limit, so keep a decimal until the limits get big
+  // enough for it to be noise.
+  const rounded = mib < 50 ? Math.floor(mib * 10) / 10 : Math.floor(mib);
+  return `${rounded}MB`;
+}

src/services/maps/map_validator.ts

@@ -1,6 +1,9 @@
-import { FileEntry, Reader, ZipReader } from '@zip.js/zip.js';
+import { FileEntry, ZipReader } from '@zip.js/zip.js';
 import { PromisedResult, Result, ResultError, wrapError } from 'base/result';
 import { PDMap } from 'schema/maps';
+import { longestSongLength, maxMapFileSize, overBudgetMessage } from 'services/maps/map_size';
+import { parseRlrr, rlrrSongLength } from 'services/maps/rlrr';
+import { MapArchive } from 'services/maps/s3_handler_types';
 import { readEntry, zipBasename, zipDirname } from 'services/maps/zip';
 
 type RawMap = Pick<
@@ -15,6 +18,7 @@ export const enum ValidateMapError {
   NO_DATA = 'no_data',
   MISSING_ALBUM_ART = 'missing_album_art',
   DESCRIPTION_TOO_LONG = 'description_too_long',
+  FILE_TOO_LARGE = 'file_too_large',
 }
 export const enum ValidateMapDifficultyError {
   INVALID_FORMAT = 'invalid_format',
@@ -24,7 +28,7 @@ export const enum ValidateMapDifficultyError {
 
 export async function validateMap(opts: {
   id: string;
-  reader: Reader<unknown>;
+  archive: MapArchive;
 }): PromisedResult<
   RawMap & { albumArtFiles: FileEntry[] },
   ValidateMapError | ValidateMapDifficultyError
@@ -33,7 +37,7 @@ export async function validateMap(opts: {
   // anything in here can throw. A throw escaping to the caller would strand the map mid-validation:
   // its upload only gets rolled back on an error Result.
   try {
-    const entries = await new ZipReader(opts.reader).getEntries();
+    const entries = await new ZipReader(opts.archive.reader).getEntries();
     const files = entries.filter((e): e is FileEntry => !e.directory);
     if (files.length === 0) {
       return { success: false, errors: [{ type: ValidateMapError.NO_DATA }] };
@@ -47,7 +51,11 @@ export async function validateMap(opts: {
     if (mapName == null || !files.every((f) => zipDirname(f.filename) === mapName)) {
       return { success: false, errors: [{ type: ValidateMapError.INCORRECT_FOLDER_STRUCTURE }] };
     }
-    return await validateMapFiles({ expectedMapName: mapName, mapFiles: files });
+    return await validateMapFiles({
+      expectedMapName: mapName,
+      mapFiles: files,
+      archiveSize: opts.archive.size,
+    });
   } catch (e) {
     // Corrupted, not a zip at all, or unreadable from storage.
     return { success: false, errors: [wrapError(e, ValidateMapError.NO_DATA)] };
@@ -64,6 +72,7 @@ type RawMapMetadata = Pick<
 async function validateMapFiles(opts: {
   expectedMapName: string;
   mapFiles: FileEntry[];
+  archiveSize: number;
 }): PromisedResult<
   RawMap & { albumArtFiles: FileEntry[] },
   ValidateMapError | ValidateMapDifficultyError
@@ -104,7 +113,8 @@ async function validateMapFiles(opts: {
       // Complexity is not, but some existing maps have mismatched complexities between rlrr files,
       // and so this check has been skipped temporarily.
       // TODO: fix all maps with mismatched complexities
-      if (key === 'difficultyName' || key === 'complexity') {
+      // Song length can differ slightly between separately-recorded difficulties.
+      if (key === 'difficultyName' || key === 'complexity' || key === 'length') {
         continue;
       }
       const expected = validDifficultyResults[0].value[key as keyof RawMapMetadata];
@@ -128,6 +138,21 @@ async function validateMapFiles(opts: {
     return { success: false, errors: [{ type: ValidateMapError.DESCRIPTION_TOO_LONG }] };
   }
 
+  const sizeLimit = maxMapFileSize(
+    longestSongLength(validDifficultyResults.map((d) => d.value.length))
+  );
+  if (opts.archiveSize > sizeLimit) {
+    return {
+      success: false,
+      errors: [
+        {
+          type: ValidateMapError.FILE_TOO_LARGE,
+          userMessage: overBudgetMessage(sizeLimit),
+        },
+      ],
+    };
+  }
+
   const albumArtFiles = validDifficultyResults
     .map((v) => v.value.albumArt)
     .filter((s): s is string => s != null)
@@ -162,10 +187,13 @@ function validateMapDifficulty(
   filename: string,
   rlrr: Uint8Array,
   getMapFile: (filename: string) => FileEntry | undefined
-): Result<RawMapMetadata & { difficultyName: string }, ValidateMapDifficultyError> {
+): Result<
+  RawMapMetadata & { difficultyName: string; length: number | undefined },
+  ValidateMapDifficultyError
+> {
   let map: any;
   try {
-    map = parseJson(rlrr);
+    map = parseRlrr(rlrr);
   } catch {
     return { success: false, errors: [{ type: ValidateMapDifficultyError.INVALID_FORMAT }] };
   }
@@ -245,13 +273,11 @@ function validateMapDifficulty(
 
   return {
     success: true,
-    value: { ...requiredFields, ...optionalFields, difficultyName: difficultyMatch[1] },
+    value: {
+      ...requiredFields,
+      ...optionalFields,
+      difficultyName: difficultyMatch[1],
+      length: rlrrSongLength(map),
+    },
   };
 }
-
-function parseJson(bytes: Uint8Array) {
-  // Paradiddle writes some rlrr files as UTF-16LE with a byte order mark. Both decoders strip the
-  // mark themselves; leaving one in front would fail the parse.
-  const isUtf16le = bytes[0] === 0xff && bytes[1] === 0xfe;
-  return JSON.parse(new TextDecoder(isUtf16le ? 'utf-16le' : 'utf-8').decode(bytes));
-}

src/services/maps/maps_repo.ts

@@ -413,7 +413,7 @@ export class MapsRepo {
       existingMap.success && existingMap.value.validity === MapValidity.REUPLOADED;
 
     await this.setValidity(id, MapValidity.VALIDATING);
-    const validatedMapResult = await validateMap({ id, reader: archive.reader });
+    const validatedMapResult = await validateMap({ id, archive });
     if (!validatedMapResult.success) {
       return validatedMapResult;
     }
@@ -532,6 +532,7 @@ export const submitErrorMap: Record<
   [ValidateMapError.MISSING_ALBUM_ART]: [400, 'Missing album art'],
   [ValidateMapError.NO_DATA]: [400, 'Invalid map archive; could not find map data'],
   [ValidateMapError.DESCRIPTION_TOO_LONG]: [400, 'Description is too long'],
+  [ValidateMapError.FILE_TOO_LARGE]: [400, 'File is over the filesize limit for this song length'],
   [ValidateMapDifficultyError.NO_AUDIO]: [400, 'Invalid map archive; missing audio files'],
   [ValidateMapDifficultyError.INVALID_FORMAT]: [
     400,

src/services/maps/rlrr.ts

@@ -0,0 +1,12 @@
+export function parseRlrr(bytes: Uint8Array): unknown {
+  // Paradiddle writes some rlrr files as UTF-16LE with a byte order mark. Both decoders strip the
+  // mark themselves; leaving one in front would fail the parse.
+  const isUtf16le = bytes[0] === 0xff && bytes[1] === 0xfe;
+  return JSON.parse(new TextDecoder(isUtf16le ? 'utf-16le' : 'utf-8').decode(bytes));
+}
+
+/** Song length in seconds, as declared by an rlrr's metadata. */
+export function rlrrSongLength(rlrr: unknown): number | undefined {
+  const length = (rlrr as { recordingMetadata?: { length?: unknown } })?.recordingMetadata?.length;
+  return typeof length === 'number' ? length : undefined;
+}

src/services/maps/tests/map_generator.ts

@@ -9,14 +9,15 @@ import * as path from 'path';
  * `files/` (`silence.ogg`, `album.jpg`), same as the Python script.
  */
 
-const FILES_DIR = path.resolve(__dirname, 'files');
+export const FILES_DIR = path.resolve(__dirname, 'files');
 const albumArt = fs.readFileSync(path.join(FILES_DIR, 'album.jpg'));
 const silence = fs.readFileSync(path.join(FILES_DIR, 'silence.ogg'));
 
 export type MapZipSpec = {
   /** Top-level folder name. Must match the .rlrr filename prefix, which the validator enforces. */
   folder: string;
-  difficulty?: string;
+  /** `lengthSeconds: null` writes no length at all, as an rlrr predating the field would. */
+  difficulties?: { name: string; lengthSeconds?: number | null }[];
   title: string;
   artist: string;
   creator?: string;
@@ -29,30 +30,30 @@ export type MapZipSpec = {
 };
 
 export function buildMapZip(spec: MapZipSpec): Buffer {
-  const difficulty = spec.difficulty ?? 'Easy';
-  const rlrr = {
+  const difficulties = spec.difficulties ?? [{ name: 'Easy' }];
+  const rlrrFor = (difficulty: (typeof difficulties)[number]) => ({
     version: 0.6,
     recordingMetadata: {
       title: spec.title,
       description: spec.description ?? '',
       coverImagePath: 'album.jpg',
       artist: spec.artist,
       creator: spec.creator ?? '',
-      length: 11.1814,
+      ...(difficulty.lengthSeconds === null ? {} : { length: difficulty.lengthSeconds ?? 11.1814 }),
       complexity: spec.complexity ?? 1,
     },
     audioFileData: { songTracks: ['song.ogg'], drumTracks: ['drums.ogg'], calibrationOffset: 0.0 },
     instruments: [],
     events: [],
     bpmEvents: [{ bpm: 120.0, time: 0.0 }],
-  };
+  });
 
-  // The .rlrr must come first: the validator derives the map name from the first file entry.
+  // The .rlrrs must come first: the validator derives the map name from the first file entry.
   const entries: ZipEntry[] = [
-    {
-      name: `${spec.folder}/${spec.folder}_${difficulty}.rlrr`,
-      data: encodeRlrr(JSON.stringify(rlrr, null, 2), spec.utf16le ?? false),
-    },
+    ...difficulties.map((d) => ({
+      name: `${spec.folder}/${spec.folder}_${d.name}.rlrr`,
+      data: encodeRlrr(JSON.stringify(rlrrFor(d), null, 2), spec.utf16le ?? false),
+    })),
     { name: `${spec.folder}/album.jpg`, data: albumArt },
     { name: `${spec.folder}/song.ogg`, data: silence },
     { name: `${spec.folder}/drums.ogg`, data: silence },

src/services/maps/tests/map_size.unit.test.ts

@@ -0,0 +1,79 @@
+import {
+  MAX_MAP_FILE_SIZE,
+  formatMaxFileSize,
+  longestSongLength,
+  maxMapFileSize,
+  overBudgetMessage,
+} from 'services/maps/map_size';
+
+const MIB = 1024 * 1024;
+
+describe('maxMapFileSize', () => {
+  it('scales with song length', () => {
+    expect(maxMapFileSize(600)).toBeGreaterThan(maxMapFileSize(60));
+  });
+
+  it('fits lossless audio for a standard-length song', () => {
+    expect(maxMapFileSize(5 * 60)).toEqual(95 * MIB);
+  });
+
+  it('only fits lossy audio for an hour-long song', () => {
+    expect(maxMapFileSize(60 * 60)).toEqual(260 * MIB);
+  });
+
+  it('never exceeds the hard cap', () => {
+    expect(maxMapFileSize(60 * 60 * 24)).toEqual(MAX_MAP_FILE_SIZE);
+  });
+
+  // An unreadable length mustn't be the most generous budget on offer, or dropping the field would
+  // be the way to get one.
+  it('falls back to a fixed allowance, not the cap, when the length is unusable', () => {
+    for (const unusable of [undefined, 0, -1, NaN, Infinity]) {
+      expect(maxMapFileSize(unusable)).toEqual(100 * MIB);
+    }
+    expect(maxMapFileSize(undefined)).toBeLessThan(MAX_MAP_FILE_SIZE);
+  });
+});
+
+describe('longestSongLength', () => {
+  it('takes the longest declared length', () => {
+    expect(longestSongLength([213.5, undefined, 214.25])).toEqual(214.25);
+  });
+
+  it('is undefined when nothing declares one', () => {
+    expect(longestSongLength([])).toBeUndefined();
+    expect(longestSongLength([undefined, undefined])).toBeUndefined();
+  });
+});
+
+describe('formatMaxFileSize', () => {
+  it('reports MiB, which is what the user sees in their file browser', () => {
+    expect(formatMaxFileSize(100 * MIB)).toEqual('100MB');
+  });
+
+  // Naming a limit larger than it is would tell the user a file that gets rejected should fit.
+  it('never names a size larger than the limit it was given', () => {
+    for (const mib of [0.05, 1.99, 49.99, 50.5, 95.9, 260.75]) {
+      expect(parseFloat(formatMaxFileSize(mib * MIB))).toBeLessThanOrEqual(mib);
+    }
+  });
+
+  it('keeps a decimal below 50MB, where a whole megabyte is a big share of the limit', () => {
+    expect(formatMaxFileSize(1.66 * MIB)).toEqual('1.6MB');
+    expect(formatMaxFileSize(49.99 * MIB)).toEqual('49.9MB');
+    expect(formatMaxFileSize(40 * MIB)).toEqual('40MB');
+  });
+
+  it('drops to whole megabytes above that, where the decimal is noise', () => {
+    expect(formatMaxFileSize(50.5 * MIB)).toEqual('50MB');
+    expect(formatMaxFileSize(260.75 * MIB)).toEqual('260MB');
+  });
+});
+
+describe('overBudgetMessage', () => {
+  it('names the limit', () => {
+    expect(overBudgetMessage(maxMapFileSize(5 * 60))).toEqual(
+      'File is over the 95MB limit for a song of this length'
+    );
+  });
+});