Skip to content

Commit 06f0b7e

Browse files
eordanoeordano
authored
feat: replace the text-encoding polyfill with a built-in utf-8 codec (#1452)
The text-encoding package adds 549 kB to every scene that imports the ethereum provider, almost all of it legacy codepage tables. Replace it with a small validating utf-8 codec implementing the WHATWG algorithms, differentially tested against the node implementations. The polyfill keeps its API and still yields to native implementations, and the composite provider now decodes with the codec directly, so loading composites at runtime no longer requires importing the ethereum provider first. TextDecoder throws a RangeError for non-utf-8 labels.
1 parent fb197d3 commit 06f0b7e

20 files changed

Lines changed: 273 additions & 68 deletions

‎packages/@dcl/sdk/package.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@
99
"@dcl/explorer": "1.0.164509-20240802172549.commit-fb95b9b",
1010
"@dcl/js-runtime": "file:../js-runtime",
1111
"@dcl/react-ecs": "file:../react-ecs",
12-
"@dcl/sdk-commands": "file:../sdk-commands",
13-
"text-encoding": "0.7.0"
12+
"@dcl/sdk-commands": "file:../sdk-commands"
1413
},
1514
"keywords": [],
1615
"license": "Apache-2.0",

‎packages/@dcl/sdk/src/composite-provider.ts‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { compositeFromLoader } from '~sdk/all-composites'
22
import { readFile } from '~system/Runtime'
3-
import { Composite, getGlobal } from '@dcl/ecs'
3+
import { Composite } from '@dcl/ecs'
4+
import { decodeUtf8 } from './internal/utf8'
45

56
const composites = new Map<string, Composite.Resource>()
67

@@ -28,16 +29,8 @@ function decodeFromBytes(content: Uint8Array): Composite.Definition {
2829
// The first-byte check is a fast path; a JSON.parse failure falls back to fromBinary
2930
// because a protobuf message can also begin with 0x7b.
3031
if (content[0] === 0x7b /* '{' */) {
31-
const TD = getGlobal<new () => { decode(input: Uint8Array): string }>('TextDecoder')
32-
if (!TD) {
33-
throw new Error(
34-
'loadComposite: TextDecoder is not available in this runtime. ' +
35-
'Use a .composite.bin file, or import `@dcl/sdk/ethereum-provider` ' +
36-
'to install the TextEncoder/TextDecoder polyfill.'
37-
)
38-
}
3932
try {
40-
return Composite.fromJson(JSON.parse(new TD().decode(content)))
33+
return Composite.fromJson(JSON.parse(decodeUtf8(content, { fatal: true })))
4134
} catch {
4235
return Composite.fromBinary(content)
4336
}

‎packages/@dcl/sdk/src/ethereum-provider/text-encoding.d.ts‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎packages/@dcl/sdk/src/internal/utf8.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/**
2+
* Self-contained UTF-8 codec implementing the WHATWG Encoding Standard
3+
* algorithms (https://encoding.spec.whatwg.org/), so SDK code never depends
4+
* on the host providing `TextEncoder` / `TextDecoder` globals.
5+
*/
6+
7+
const REPLACEMENT_CHARACTER = 0xfffd
8+
9+
export type DecodeUtf8Options = {
10+
fatal?: boolean
11+
ignoreBOM?: boolean
12+
}
13+
14+
export function decodeUtf8(input: Uint8Array, options: DecodeUtf8Options = {}): string {
15+
const fatal = options.fatal === true
16+
const start =
17+
!options.ignoreBOM && input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf ? 3 : 0
18+
19+
let result = ''
20+
let pending: number[] = []
21+
let codePoint = 0
22+
let bytesNeeded = 0
23+
let bytesSeen = 0
24+
let lowerBoundary = 0x80
25+
let upperBoundary = 0xbf
26+
27+
function emit(value: number) {
28+
if (value <= 0xffff) {
29+
pending.push(value)
30+
} else {
31+
const offset = value - 0x10000
32+
pending.push(0xd800 + (offset >> 10), 0xdc00 + (offset & 0x3ff))
33+
}
34+
if (pending.length >= 4096) {
35+
result += String.fromCharCode(...pending)
36+
pending = []
37+
}
38+
}
39+
40+
function malformed() {
41+
if (fatal) throw new TypeError('decodeUtf8: the encoded data is not valid utf-8')
42+
emit(REPLACEMENT_CHARACTER)
43+
}
44+
45+
for (let i = start; i < input.length; i++) {
46+
const byte = input[i]
47+
if (bytesNeeded === 0) {
48+
if (byte <= 0x7f) {
49+
emit(byte)
50+
} else if (byte >= 0xc2 && byte <= 0xdf) {
51+
bytesNeeded = 1
52+
codePoint = byte & 0x1f
53+
} else if (byte >= 0xe0 && byte <= 0xef) {
54+
if (byte === 0xe0) lowerBoundary = 0xa0
55+
if (byte === 0xed) upperBoundary = 0x9f
56+
bytesNeeded = 2
57+
codePoint = byte & 0xf
58+
} else if (byte >= 0xf0 && byte <= 0xf4) {
59+
if (byte === 0xf0) lowerBoundary = 0x90
60+
if (byte === 0xf4) upperBoundary = 0x8f
61+
bytesNeeded = 3
62+
codePoint = byte & 0x7
63+
} else {
64+
malformed()
65+
}
66+
} else if (byte < lowerBoundary || byte > upperBoundary) {
67+
codePoint = 0
68+
bytesNeeded = 0
69+
bytesSeen = 0
70+
lowerBoundary = 0x80
71+
upperBoundary = 0xbf
72+
malformed()
73+
// the spec prepends the byte back to the stream: it starts a new sequence
74+
i--
75+
} else {
76+
lowerBoundary = 0x80
77+
upperBoundary = 0xbf
78+
codePoint = (codePoint << 6) | (byte & 0x3f)
79+
if (++bytesSeen === bytesNeeded) {
80+
emit(codePoint)
81+
codePoint = 0
82+
bytesNeeded = 0
83+
bytesSeen = 0
84+
}
85+
}
86+
}
87+
if (bytesNeeded !== 0) malformed()
88+
89+
if (pending.length > 0) result += String.fromCharCode(...pending)
90+
return result
91+
}
92+
93+
export function encodeUtf8(input: string): Uint8Array {
94+
const output = new Uint8Array(utf8ByteLength(input))
95+
let offset = 0
96+
for (let i = 0; i < input.length; i++) {
97+
const codePoint = codePointAt(input, i)
98+
if (codePoint > 0xffff) i++
99+
offset = writeCodePoint(output, offset, codePoint)
100+
}
101+
return output
102+
}
103+
104+
export function encodeUtf8Into(source: string, destination: Uint8Array): { read: number; written: number } {
105+
let read = 0
106+
let written = 0
107+
for (let i = 0; i < source.length; i++) {
108+
const codePoint = codePointAt(source, i)
109+
const size = byteSize(codePoint)
110+
if (written + size > destination.length) break
111+
written = writeCodePoint(destination, written, codePoint)
112+
if (codePoint > 0xffff) {
113+
i++
114+
read += 2
115+
} else {
116+
read += 1
117+
}
118+
}
119+
return { read, written }
120+
}
121+
122+
// Reads the code point at index, replacing unpaired surrogates with U+FFFD
123+
// (the USVString conversion `TextEncoder` mandates).
124+
function codePointAt(input: string, index: number): number {
125+
const first = input.charCodeAt(index)
126+
if (first >= 0xd800 && first <= 0xdbff) {
127+
const second = index + 1 < input.length ? input.charCodeAt(index + 1) : 0
128+
if (second >= 0xdc00 && second <= 0xdfff) return 0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00)
129+
return REPLACEMENT_CHARACTER
130+
}
131+
if (first >= 0xdc00 && first <= 0xdfff) return REPLACEMENT_CHARACTER
132+
return first
133+
}
134+
135+
function byteSize(codePoint: number): number {
136+
return codePoint <= 0x7f ? 1 : codePoint <= 0x7ff ? 2 : codePoint <= 0xffff ? 3 : 4
137+
}
138+
139+
function utf8ByteLength(input: string): number {
140+
let length = 0
141+
for (let i = 0; i < input.length; i++) {
142+
const codePoint = codePointAt(input, i)
143+
if (codePoint > 0xffff) i++
144+
length += byteSize(codePoint)
145+
}
146+
return length
147+
}
148+
149+
function writeCodePoint(output: Uint8Array, offset: number, codePoint: number): number {
150+
let next = offset
151+
if (codePoint <= 0x7f) {
152+
output[next++] = codePoint
153+
} else if (codePoint <= 0x7ff) {
154+
output[next++] = 0xc0 | (codePoint >> 6)
155+
output[next++] = 0x80 | (codePoint & 0x3f)
156+
} else if (codePoint <= 0xffff) {
157+
output[next++] = 0xe0 | (codePoint >> 12)
158+
output[next++] = 0x80 | ((codePoint >> 6) & 0x3f)
159+
output[next++] = 0x80 | (codePoint & 0x3f)
160+
} else {
161+
output[next++] = 0xf0 | (codePoint >> 18)
162+
output[next++] = 0x80 | ((codePoint >> 12) & 0x3f)
163+
output[next++] = 0x80 | ((codePoint >> 6) & 0x3f)
164+
output[next++] = 0x80 | (codePoint & 0x3f)
165+
}
166+
return next
167+
}

‎packages/@dcl/sdk/src/text-codec.ts‎

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,70 @@
1-
import TextEncodingPolyfill from 'text-encoding'
21
import { setGlobalPolyfill } from '@dcl/ecs'
2+
import { decodeUtf8, encodeUtf8, encodeUtf8Into } from './internal/utf8'
33

44
/**
5-
* Install the `TextEncoder` / `TextDecoder` polyfill on `globalThis`.
5+
* WHATWG-shaped `TextEncoder` / `TextDecoder` over the SDK's own UTF-8 codec.
66
*
7-
* The QuickJS scene runtime ships no native `TextEncoder` / `TextDecoder`, yet
8-
* `compositeProvider.loadComposite` decodes `.composite` JSON file bytes via
9-
* `TextDecoder`. Callers that load composites at runtime (e.g. `@dcl/asset-packs`'
10-
* `SPAWN_ENTITY`) must install this first.
7+
* UTF-8 is the only supported encoding: the decoder throws a `RangeError` for
8+
* any other label instead of silently mis-decoding, and rejects `{ stream: true }`.
9+
*/
10+
11+
// Labels the WHATWG Encoding Standard maps to utf-8.
12+
const UTF8_LABELS = ['unicode-1-1-utf-8', 'unicode11utf8', 'unicode20utf8', 'utf-8', 'utf8', 'x-unicode20utf8']
13+
14+
function toUint8Array(input?: ArrayBuffer | ArrayBufferView): Uint8Array {
15+
if (input === undefined) return new Uint8Array(0)
16+
if (input instanceof Uint8Array) return input
17+
if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength)
18+
return new Uint8Array(input)
19+
}
20+
21+
export class TextEncoder {
22+
readonly encoding = 'utf-8'
23+
24+
encode(input: string = ''): Uint8Array {
25+
return encodeUtf8(String(input))
26+
}
27+
28+
encodeInto(source: string, destination: Uint8Array): { read: number; written: number } {
29+
return encodeUtf8Into(String(source), destination)
30+
}
31+
}
32+
33+
export class TextDecoder {
34+
readonly encoding = 'utf-8'
35+
readonly fatal: boolean
36+
readonly ignoreBOM: boolean
37+
38+
constructor(label: string = 'utf-8', options: { fatal?: boolean; ignoreBOM?: boolean } = {}) {
39+
// the spec strips ASCII whitespace only, so String.prototype.trim is too wide
40+
const normalized = String(label)
41+
.replace(/^[\t\n\f\r ]+|[\t\n\f\r ]+$/g, '')
42+
.toLowerCase()
43+
if (!UTF8_LABELS.includes(normalized)) {
44+
throw new RangeError(`TextDecoder: only utf-8 is supported, got "${label}"`)
45+
}
46+
this.fatal = options.fatal === true
47+
this.ignoreBOM = options.ignoreBOM === true
48+
}
49+
50+
decode(input?: ArrayBuffer | ArrayBufferView, options: { stream?: boolean } = {}): string {
51+
if (options.stream) throw new TypeError('TextDecoder: streaming is not supported')
52+
return decodeUtf8(toUint8Array(input), { fatal: this.fatal, ignoreBOM: this.ignoreBOM })
53+
}
54+
}
55+
56+
/**
57+
* Install `TextEncoder` / `TextDecoder` on `globalThis` where the runtime does
58+
* not provide them (native implementations always win over these polyfills).
1159
*
1260
* Exposed from the lean `@dcl/sdk/text-codec` subpath so consumers can install
13-
* the polyfill without pulling in the ethereum provider, and without bundling
14-
* `text-encoding` into scenes that never reach this module.
61+
* the polyfill without pulling in the ethereum provider.
1562
*/
1663
// NOTE: this function mutates globalThis — it is NOT pure. Do not annotate
1764
// with /* @__PURE__ */; minifiers would treat the call as dead code and drop
18-
// the polyfill installation, breaking runtime callers like
19-
// compositeProvider.loadComposite that rely on globalThis.TextDecoder.
65+
// the polyfill installation, breaking third-party code that relies on
66+
// globalThis.TextDecoder.
2067
export function polyfillTextEncoder() {
21-
setGlobalPolyfill('TextEncoder', TextEncodingPolyfill.TextEncoder)
22-
setGlobalPolyfill('TextDecoder', TextEncodingPolyfill.TextDecoder)
68+
setGlobalPolyfill('TextEncoder', TextEncoder)
69+
setGlobalPolyfill('TextDecoder', TextDecoder)
2370
}

‎test/build-ecs/compile-ecs7.spec.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,13 @@ describe('build: scene with etherum', () => {
4242
const binPath = ensureFileExists('bin/game.js', cwd)
4343
const fileText = readFileSync(binPath, 'utf8')
4444

45+
const polyfillIncluded = fileText.includes('polyfillTextEncoder')
46+
if (!polyfillIncluded) {
47+
throw new Error(`scene doesn't include the TextEncoder/TextDecoder polyfill`)
48+
}
4549
const textEncodingLibraryIncluded = fileText.includes('text-encoding')
46-
if (!textEncodingLibraryIncluded) {
47-
throw new Error(`scene doesn't include textEncoding`)
50+
if (textEncodingLibraryIncluded) {
51+
throw new Error('textEncoding is being bundled in the scene.')
4852
}
4953
})
5054
})

‎test/sdk/text-codec.spec.ts‎

6.66 KB
Binary file not shown.

‎test/snapshots/development-bundles/static-scene.test.ts.crdt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
SCENE_COMPILED_JS_SIZE_PROD=565.8k bytes
1+
SCENE_COMPILED_JS_SIZE_PROD=567.4k bytes
22
THE BUNDLE HAS SOURCEMAPS
33
(start empty vm 0.21.0-3680274614.commit-1808aa1)
44
OPCODES ~= 0k
@@ -12,7 +12,7 @@ EVAL test/snapshots/development-bundles/static-scene.test.js
1212
REQUIRE: ~system/EngineApi
1313
REQUIRE: ~system/Runtime
1414
OPCODES ~= 73k
15-
MALLOC_COUNT = 16310
15+
MALLOC_COUNT = 16324
1616
ALIVE_OBJS_DELTA ~= 3.25k
1717
CALL onStart()
1818
main.crdt: PUT_COMPONENT e=0x200 c=1 t=0 data={"position":{"x":5.880000114440918,"y":2.7916901111602783,"z":7.380000114440918},"rotation":{"x":0,"y":0,"z":0,"w":1},"scale":{"x":1,"y":1,"z":1},"parent":0}
@@ -57,4 +57,4 @@ CALL onUpdate(0.1)
5757
OPCODES ~= 5k
5858
MALLOC_COUNT = -5
5959
ALIVE_OBJS_DELTA ~= 0.00k
60-
MEMORY_USAGE_COUNT ~= 1428.83k bytes
60+
MEMORY_USAGE_COUNT ~= 1432.25k bytes

‎test/snapshots/development-bundles/testing-fw.test.ts.crdt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
SCENE_COMPILED_JS_SIZE_PROD=566.4k bytes
1+
SCENE_COMPILED_JS_SIZE_PROD=567.9k bytes
22
THE BUNDLE HAS SOURCEMAPS
33
(start empty vm 0.21.0-3680274614.commit-1808aa1)
44
OPCODES ~= 0k
@@ -12,7 +12,7 @@ EVAL test/snapshots/development-bundles/testing-fw.test.js
1212
REQUIRE: ~system/EngineApi
1313
REQUIRE: ~system/Runtime
1414
OPCODES ~= 83k
15-
MALLOC_COUNT = 16862
15+
MALLOC_COUNT = 16876
1616
ALIVE_OBJS_DELTA ~= 3.40k
1717
CALL onStart()
1818
LOG: ["Adding one to position.y=0"]
@@ -63,4 +63,4 @@ CALL onUpdate(0.1)
6363
OPCODES ~= 6k
6464
MALLOC_COUNT = -53
6565
ALIVE_OBJS_DELTA ~= -0.01k
66-
MEMORY_USAGE_COUNT ~= 1434.60k bytes
66+
MEMORY_USAGE_COUNT ~= 1438.01k bytes

‎test/snapshots/development-bundles/two-way-crdt.test.ts.crdt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
SCENE_COMPILED_JS_SIZE_PROD=566.4k bytes
1+
SCENE_COMPILED_JS_SIZE_PROD=567.9k bytes
22
THE BUNDLE HAS SOURCEMAPS
33
(start empty vm 0.21.0-3680274614.commit-1808aa1)
44
OPCODES ~= 0k
@@ -12,7 +12,7 @@ EVAL test/snapshots/development-bundles/two-way-crdt.test.js
1212
REQUIRE: ~system/EngineApi
1313
REQUIRE: ~system/Runtime
1414
OPCODES ~= 83k
15-
MALLOC_COUNT = 16862
15+
MALLOC_COUNT = 16876
1616
ALIVE_OBJS_DELTA ~= 3.40k
1717
CALL onStart()
1818
LOG: ["Adding one to position.y=0"]
@@ -63,4 +63,4 @@ CALL onUpdate(0.1)
6363
OPCODES ~= 6k
6464
MALLOC_COUNT = -53
6565
ALIVE_OBJS_DELTA ~= -0.01k
66-
MEMORY_USAGE_COUNT ~= 1434.60k bytes
66+
MEMORY_USAGE_COUNT ~= 1438.01k bytes

0 commit comments

Comments
 (0)