feat(sdk-commands): namespace and durably persist local dev storage per project

Local preview server storage (server-storage.json) had three issues:

- World values were not namespaced by scene, so previewing different scenes
  shared one bucket. World storage is now keyed by scene base coordinates
  ("x,y"), read from scene.json.
- The file lived inside node_modules/@dcl/sdk-commands, so every SDK upgrade
  wiped all local dev progress. It now lives in the project's .runtime-data/
  directory (threaded via baseDir), surviving `npm i @dcl/sdk@newer` and keeping
  same-base-coord scenes in different projects isolated.
- A legacy flat-format world file was discarded on load. It is now migrated once
  at preview-server startup into the currently previewed scene's bucket, so no
  local data is lost on the format change.

Reconciles with the upstream serialize() write-lock (#1545): every
read-modify-write still runs under the queue.

Author: cyaiox

packages/@dcl/sdk-commands/src/commands/start/server/routes.ts

@@ -68,7 +68,7 @@ export async function wireRouter(components: PreviewComponents, workspace: Works
   })
 
   setupRealmAndComms(components, router, localSceneParcels)
-  setupStorageEndpoints(components, router, workspace)
+  await setupStorageEndpoints(components, router, workspace)
   await setupEcs6Endpoints(components, router, workspace)
 
   components.server.setContext(components)

packages/@dcl/sdk-commands/src/commands/start/server/runtime-env.ts

@@ -1,29 +1,44 @@
 import path from 'path'
 import { CliComponents } from '../../../components'
+import { getObject } from '../../../logic/coordinates'
 
-// Find the sdk-commands package root by resolving its package.json
-const SDK_COMMANDS_ROOT = path.dirname(require.resolve('@dcl/sdk-commands/package.json'))
-const RUNTIME_DATA_DIR = path.join(SDK_COMMANDS_ROOT, '.runtime-data')
+const RUNTIME_DATA_DIRNAME = '.runtime-data'
 const SERVER_STORAGE_FILE = 'server-storage.json'
 
+const storageDirFor = (baseDir: string): string => path.join(baseDir, RUNTIME_DATA_DIRNAME)
+const storagePathFor = (baseDir: string): string => path.join(storageDirFor(baseDir), SERVER_STORAGE_FILE)
+
 /**
  * Structure for all server-side storage data.
- * Stored in sdk-commands package directory (hidden from users).
+ * Stored in the project's `.runtime-data/` directory so local dev progress survives
+ * SDK version upgrades (which replace `node_modules`).
+ *
+ * `world` is namespaced by scene base coordinates (`"x,y"`) so that previewing
+ * different scenes does not share the same scene-storage bucket.
  */
 export interface ServerStorage {
   env: Record<string, string>
-  world: Record<string, unknown>
+  world: Record<string, Record<string, unknown>>
   players: Record<string, Record<string, unknown>>
 }
 
-function createDefaultStorage(): ServerStorage {
-  return {
-    env: {},
-    world: {},
-    players: {}
-  }
+/**
+ * Normalizes a scene base parcel (e.g. `"60, -9"`) into the canonical `"x,y"`
+ * key used to namespace world storage, so equivalent spellings map to one bucket.
+ */
+export function getSceneStorageKey(base: string): string {
+  const { x, y } = getObject(base)
+  return `${x},${y}`
 }
 
+const isPlainObject = (value: unknown): boolean => typeof value === 'object' && value !== null && !Array.isArray(value)
+
+const createDefaultStorage = (): ServerStorage => ({
+  env: {},
+  world: {},
+  players: {}
+})
+
 let writeQueue: Promise<unknown> = Promise.resolve()
 
 /**
@@ -44,11 +59,12 @@ function serialize<T>(task: () => Promise<T>): Promise<T> {
 /**
  * Ensures the runtime data directory exists.
  */
-async function ensureRuntimeDir(components: Pick<CliComponents, 'fs' | 'logger'>): Promise<void> {
+async function ensureRuntimeDir(components: Pick<CliComponents, 'fs' | 'logger'>, baseDir: string): Promise<void> {
+  const dir = storageDirFor(baseDir)
   try {
-    const exists = await components.fs.directoryExists(RUNTIME_DATA_DIR)
+    const exists = await components.fs.directoryExists(dir)
     if (!exists) {
-      await components.fs.mkdir(RUNTIME_DATA_DIR, { recursive: true })
+      await components.fs.mkdir(dir, { recursive: true })
     }
   } catch (error) {
     components.logger.error(`Failed to create runtime data directory: ${error}`)
@@ -58,8 +74,11 @@ async function ensureRuntimeDir(components: Pick<CliComponents, 'fs' | 'logger'>
 /**
  * Loads all server-side storage data from server-storage.json.
  */
-export async function loadServerStorage(components: Pick<CliComponents, 'fs' | 'logger'>): Promise<ServerStorage> {
-  const storagePath = path.join(RUNTIME_DATA_DIR, SERVER_STORAGE_FILE)
+export async function loadServerStorage(
+  components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string
+): Promise<ServerStorage> {
+  const storagePath = storagePathFor(baseDir)
 
   try {
     const exists = await components.fs.fileExists(storagePath)
@@ -86,10 +105,11 @@ export async function loadServerStorage(components: Pick<CliComponents, 'fs' | '
  */
 export async function saveServerStorage(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
   data: ServerStorage
 ): Promise<void> {
-  await ensureRuntimeDir(components)
-  const storagePath = path.join(RUNTIME_DATA_DIR, SERVER_STORAGE_FILE)
+  await ensureRuntimeDir(components, baseDir)
+  const storagePath = storagePathFor(baseDir)
 
   try {
     const tmpPath = `${storagePath}.tmp`
@@ -101,6 +121,30 @@ export async function saveServerStorage(
   }
 }
 
+/**
+ * Migrates a legacy flat-format world file (`world: key -> value`, written before
+ * scene-coordinate namespacing) into the currently-previewed scene's bucket, so no
+ * local dev data is lost on the format change. No-op once the file is already
+ * namespaced (`world: "x,y" -> key -> value`).
+ */
+export async function migrateLegacyWorldStorage(
+  components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  sceneKey: string
+): Promise<void> {
+  return serialize(async () => {
+    const storage = await loadServerStorage(components, baseDir)
+    const world = storage.world ?? {}
+    const isLegacyFlat = Object.values(world).some((value) => !isPlainObject(value))
+    if (!isLegacyFlat) {
+      return
+    }
+    components.logger.debug(`Migrating legacy flat world storage into scene bucket ${sceneKey}`)
+    storage.world = { [sceneKey]: world }
+    await saveServerStorage(components, baseDir, storage)
+  })
+}
+
 /**
  * Loads environment variables from a .env file in the project directory.
  * Returns a Map of key-value pairs.
@@ -123,7 +167,6 @@ export async function loadEnvFile(
 
     for (const line of lines) {
       const trimmed = line.trim()
-      // Skip empty lines and comments
       if (!trimmed || trimmed.startsWith('#')) {
         continue
       }
@@ -133,7 +176,6 @@ export async function loadEnvFile(
         const key = trimmed.slice(0, equalIndex).trim()
         let value = trimmed.slice(equalIndex + 1).trim()
 
-        // Remove surrounding quotes if present
         if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
           value = value.slice(1, -1)
         }
@@ -151,23 +193,26 @@ export async function loadEnvFile(
 /**
  * Gets runtime environment variables.
  */
-export async function getEnvStorage(components: Pick<CliComponents, 'fs' | 'logger'>): Promise<Record<string, string>> {
-  const storage = await loadServerStorage(components)
+export async function getEnvStorage(
+  components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string
+): Promise<Record<string, string>> {
+  const storage = await loadServerStorage(components, baseDir)
   return storage.env
 }
 
 /**
  * Gets merged environment variables.
  * Runtime values (from server-storage.json) override .env values.
+ * The storage file and the `.env` file share the project directory.
  */
 export async function getMergedEnv(
   components: Pick<CliComponents, 'fs' | 'logger'>,
   projectDirectory: string
 ): Promise<Map<string, string>> {
   const envFile = await loadEnvFile(components, projectDirectory)
-  const runtimeEnv = await getEnvStorage(components)
+  const runtimeEnv = await getEnvStorage(components, projectDirectory)
 
-  // Runtime overrides .env
   for (const [key, value] of Object.entries(runtimeEnv)) {
     envFile.set(key, value)
   }
@@ -180,83 +225,99 @@ export async function getMergedEnv(
  */
 export async function setEnvValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
   key: string,
   value: string
 ): Promise<void> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
+    const storage = await loadServerStorage(components, baseDir)
     storage.env[key] = value
-    await saveServerStorage(components, storage)
+    await saveServerStorage(components, baseDir, storage)
   })
 }
 
 /**
  * Deletes a runtime environment variable.
  * Returns true if key existed and was deleted, false otherwise.
  */
-export async function deleteEnvValue(components: Pick<CliComponents, 'fs' | 'logger'>, key: string): Promise<boolean> {
+export async function deleteEnvValue(
+  components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  key: string
+): Promise<boolean> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
+    const storage = await loadServerStorage(components, baseDir)
     if (!(key in storage.env)) {
       return false
     }
     delete storage.env[key]
-    await saveServerStorage(components, storage)
+    await saveServerStorage(components, baseDir, storage)
     return true
   })
 }
 
 /**
- * Gets all world storage data.
+ * Gets all world storage data for a scene, keyed by its base-coordinate bucket.
  */
 export async function getWorldStorage(
-  components: Pick<CliComponents, 'fs' | 'logger'>
+  components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  sceneKey: string
 ): Promise<Record<string, unknown>> {
-  const storage = await loadServerStorage(components)
-  return storage.world
+  const storage = await loadServerStorage(components, baseDir)
+  return storage.world[sceneKey] ?? {}
 }
 
 /**
- * Gets a value from world storage.
+ * Gets a value from a scene's world storage.
  */
 export async function getWorldValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  sceneKey: string,
   key: string
 ): Promise<unknown | undefined> {
-  const storage = await loadServerStorage(components)
-  return storage.world[key]
+  const storage = await loadServerStorage(components, baseDir)
+  return storage.world[sceneKey]?.[key]
 }
 
 /**
- * Sets a value in world storage.
+ * Sets a value in a scene's world storage.
  */
 export async function setWorldValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  sceneKey: string,
   key: string,
   value: unknown
 ): Promise<void> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
-    storage.world[key] = value
-    await saveServerStorage(components, storage)
+    const storage = await loadServerStorage(components, baseDir)
+    if (!storage.world[sceneKey]) {
+      storage.world[sceneKey] = {}
+    }
+    storage.world[sceneKey][key] = value
+    await saveServerStorage(components, baseDir, storage)
   })
 }
 
 /**
- * Deletes a value from world storage.
+ * Deletes a value from a scene's world storage.
  * Returns true if key existed and was deleted, false otherwise.
  */
 export async function deleteWorldValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
+  sceneKey: string,
   key: string
 ): Promise<boolean> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
-    if (!(key in storage.world)) {
+    const storage = await loadServerStorage(components, baseDir)
+    if (!storage.world[sceneKey] || !(key in storage.world[sceneKey])) {
       return false
     }
-    delete storage.world[key]
-    await saveServerStorage(components, storage)
+    delete storage.world[sceneKey][key]
+    await saveServerStorage(components, baseDir, storage)
     return true
   })
 }
@@ -266,10 +327,11 @@ export async function deleteWorldValue(
  */
 export async function getPlayerValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
   address: string,
   key: string
 ): Promise<unknown | undefined> {
-  const storage = await loadServerStorage(components)
+  const storage = await loadServerStorage(components, baseDir)
   return storage.players[address]?.[key]
 }
 
@@ -278,17 +340,18 @@ export async function getPlayerValue(
  */
 export async function setPlayerValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
   address: string,
   key: string,
   value: unknown
 ): Promise<void> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
+    const storage = await loadServerStorage(components, baseDir)
     if (!storage.players[address]) {
       storage.players[address] = {}
     }
     storage.players[address][key] = value
-    await saveServerStorage(components, storage)
+    await saveServerStorage(components, baseDir, storage)
   })
 }
 
@@ -298,16 +361,17 @@ export async function setPlayerValue(
  */
 export async function deletePlayerValue(
   components: Pick<CliComponents, 'fs' | 'logger'>,
+  baseDir: string,
   address: string,
   key: string
 ): Promise<boolean> {
   return serialize(async () => {
-    const storage = await loadServerStorage(components)
+    const storage = await loadServerStorage(components, baseDir)
     if (!storage.players[address] || !(key in storage.players[address])) {
       return false
     }
     delete storage.players[address][key]
-    await saveServerStorage(components, storage)
+    await saveServerStorage(components, baseDir, storage)
     return true
   })
 }