Draft: experimental UTF-8 support

projects/common/src/client/java/dan200/computercraft/client/gui/ClientComputerInput.java

@@ -37,12 +37,12 @@ public void keyUp(int key) {
     }
 
     @Override
-    public void charTyped(byte chr) {
-        ClientNetworking.sendToServer(new KeyEventServerMessage(menu, KeyEventServerMessage.Action.CHAR, chr));
+    public void charTyped(int codepoint) {
+        ClientNetworking.sendToServer(new KeyEventServerMessage(menu, KeyEventServerMessage.Action.CHAR, codepoint));
     }
 
     @Override
-    public void paste(ByteBuffer contents) {
+    public void paste(String contents) {
         ClientNetworking.sendToServer(new PasteEventComputerMessage(menu, contents));
     }
 

projects/common/src/client/java/dan200/computercraft/client/gui/widgets/TerminalWidget.java

@@ -12,6 +12,7 @@
 import dan200.computercraft.client.render.text.FixedWidthFontRenderer;
 import dan200.computercraft.core.input.UserComputerInput;
 import dan200.computercraft.core.terminal.Terminal;
+import dan200.computercraft.core.util.Colour;
 import net.minecraft.client.Minecraft;
 import net.minecraft.client.gui.GuiGraphics;
 import net.minecraft.client.gui.components.AbstractWidget;
@@ -204,6 +205,36 @@ public void renderWidget(GuiGraphics graphics, int mouseX, int mouseY, float par
         );
 
         bufferSource.endBatch();
+
+        renderUnicodeOverlay(graphics);
+    }
+
+    private void renderUnicodeOverlay(GuiGraphics graphics) {
+        var font = Minecraft.getInstance().font;
+        var palette = terminal.getPalette();
+
+        for (var y = 0; y < terminal.getHeight(); y++) {
+            var textLine = terminal.getLine(y);
+            var textColourLine = terminal.getTextColourLine(y);
+
+            for (var x = 0; x < textLine.length(); x++) {
+                var codepoint = textLine.codePointAt(x);
+                if (codepoint >= 0 && codepoint <= 255) continue;
+
+                var text = new String(Character.toChars(codepoint));
+                var colour = palette.getRenderColours(
+                    FixedWidthFontRenderer.getColour(textColourLine.charAt(x), Colour.BLACK)
+                );
+
+                var drawX = innerX + x * FONT_WIDTH;
+                var drawY = innerY + y * FONT_HEIGHT;
+
+                var glyphWidth = font.width(text);
+                var xOffset = Math.max(0, (FONT_WIDTH - glyphWidth) / 2);
+
+                graphics.drawString(font, text, drawX + xOffset, drawY, colour, false);
+            }
+        }
     }
 
     @Override

projects/common/src/client/java/dan200/computercraft/client/render/monitor/MonitorBlockEntityRenderer.java

@@ -22,6 +22,7 @@
 import dan200.computercraft.shared.peripheral.monitor.MonitorBlockEntity;
 import dan200.computercraft.shared.peripheral.monitor.MonitorRenderer;
 import dan200.computercraft.shared.util.DirectionUtil;
+import dan200.computercraft.core.util.Colour;
 import net.minecraft.client.renderer.MultiBufferSource;
 import net.minecraft.client.renderer.blockentity.BlockEntityRenderer;
 import net.minecraft.client.renderer.blockentity.BlockEntityRendererProvider;
@@ -39,6 +40,9 @@
 import static dan200.computercraft.client.render.text.FixedWidthFontRenderer.FONT_WIDTH;
 import static dan200.computercraft.core.util.Nullability.assertNonNull;
 
+import net.minecraft.client.Minecraft;
+import net.minecraft.client.gui.Font;
+
 public class MonitorBlockEntityRenderer implements BlockEntityRenderer<MonitorBlockEntity> {
     /**
      * {@link MonitorBlockEntity#RENDER_MARGIN}, but a tiny bit of additional padding to ensure that there is no space between
@@ -55,6 +59,41 @@ public class MonitorBlockEntityRenderer implements BlockEntityRenderer<MonitorBl
     public MonitorBlockEntityRenderer(BlockEntityRendererProvider.Context context) {
     }
 
+    private static void renderUnicodeOverlay(PoseStack transform, MultiBufferSource bufferSource, Terminal terminal) {
+        var font = Minecraft.getInstance().font;
+        var palette = terminal.getPalette();
+
+        transform.pushPose();
+        transform.translate(0.0f, 0.0f, 0.001f);
+
+        var matrix = transform.last().pose();
+
+        for (var y = 0; y < terminal.getHeight(); y++) {
+            var textLine = terminal.getLine(y);
+            var textColourLine = terminal.getTextColourLine(y);
+
+            for (var x = 0; x < textLine.length(); x++) {
+                var codepoint = textLine.codePointAt(x);
+                if (codepoint >= 0 && codepoint <= 255) continue;
+
+                var text = new String(Character.toChars(codepoint));
+                var colour = palette.getRenderColours(
+                    FixedWidthFontRenderer.getColour(textColourLine.charAt(x), Colour.BLACK)
+                );
+
+                var drawX = x * FONT_WIDTH + Math.max(0.0f, (FONT_WIDTH - font.width(text)) / 2.0f);
+                var drawY = y * FONT_HEIGHT;
+
+                font.drawInBatch(
+                    text, drawX, drawY, colour, false, matrix, bufferSource,
+                    Font.DisplayMode.NORMAL, 0, RenderTypes.FULL_BRIGHT_LIGHTMAP
+                );
+            }
+        }
+
+        transform.popPose();
+    }
+
     @Override
     public void render(MonitorBlockEntity monitor, float partialTicks, PoseStack transform, MultiBufferSource bufferSource, int lightmapCoord, int overlayLight) {
         // Render from the origin monitor
@@ -117,6 +156,7 @@ public void render(MonitorBlockEntity monitor, float partialTicks, PoseStack tra
             var matrix = transform.last().pose();
 
             renderTerminal(matrix, originTerminal, renderState, terminal, (float) (MARGIN / xScale), (float) (MARGIN / yScale));
+            renderUnicodeOverlay(transform, bufferSource, terminal);
 
             transform.popPose();
         } else {

projects/common/src/client/java/dan200/computercraft/client/render/monitor/MonitorTextureBufferShader.java

@@ -88,7 +88,8 @@ public static void setTerminalData(ByteBuffer buffer, Terminal terminal) {
         for (var y = 0; y < height; y++) {
             TextBuffer text = terminal.getLine(y), textColour = terminal.getTextColourLine(y), background = terminal.getBackgroundColourLine(y);
             for (var x = 0; x < width; x++) {
-                buffer.put(pos, (byte) (text.charAt(x) & 0xFF));
+                var codepoint = text.codePointAt(x);
+                buffer.put(pos, codepoint >= 0 && codepoint <= 255 ? (byte) codepoint : (byte) 0);
                 buffer.put(pos + 1, (byte) getColour(textColour.charAt(x), Colour.WHITE));
                 buffer.put(pos + 2, (byte) getColour(background.charAt(x), Colour.BLACK));
                 pos += 3;

projects/common/src/client/java/dan200/computercraft/client/render/text/DirectFixedWidthFontRenderer.java

@@ -102,9 +102,10 @@ public static void drawString(QuadEmitter emitter, float x, float y, TextBuffer
         for (var i = 0; i < text.length(); i++) {
             var colour = palette.getRenderColours(getColour(textColour.charAt(i), Colour.BLACK));
 
-            int index = text.charAt(i);
-            if (index > 255) index = '?';
-            drawChar(emitter, x + i * FONT_WIDTH, y, index, colour);
+            var codepoint = text.codePointAt(i);
+            if (codepoint >= 0 && codepoint <= 255) {
+                drawChar(emitter, x + i * FONT_WIDTH, y, codepoint, colour);
+            }
         }
 
     }

projects/common/src/client/java/dan200/computercraft/client/render/text/FixedWidthFontRenderer.java

@@ -121,9 +121,10 @@ public static void drawString(QuadEmitter emitter, float x, float y, TextBuffer
         for (var i = 0; i < text.length(); i++) {
             var colour = palette.getRenderColours(getColour(textColour.charAt(i), Colour.BLACK));
 
-            int index = text.charAt(i);
-            if (index > 255) index = '?';
-            drawChar(emitter, x + i * FONT_WIDTH, y, index, colour, light);
+            var codepoint = text.codePointAt(i);
+            if (codepoint >= 0 && codepoint <= 255) {
+                drawChar(emitter, x + i * FONT_WIDTH, y, codepoint, colour, light);
+            }
         }
 
     }

projects/common/src/main/java/dan200/computercraft/shared/computer/terminal/NetworkedTerminal.java

@@ -19,26 +19,45 @@ public NetworkedTerminal(int width, int height, boolean colour, Runnable changed
     }
 
     synchronized TerminalState write() {
-        var contents = new byte[width * height * 2 + Palette.PALETTE_SIZE * 3];
-        var idx = 0;
+        var textContents = new int[width * height];
+        var colours = new byte[width * height];
+        var paletteBytes = new byte[Palette.PALETTE_SIZE * 3];
+
+        var textIdx = 0;
+        var colourIdx = 0;
+        var paletteIdx = 0;
 
         for (var y = 0; y < height; y++) {
-            var text = this.text[y];
-            var textColour = this.textColour[y];
-            var backColour = backgroundColour[y];
+            var textLine = this.text[y];
+            var textColourLine = this.textColour[y];
+            var backColourLine = backgroundColour[y];
+
+            for (var x = 0; x < width; x++) {
+                textContents[textIdx++] = textLine.codePointAt(x);
+            }
 
-            for (var x = 0; x < width; x++) contents[idx++] = (byte) (text.charAt(x) & 0xFF);
             for (var x = 0; x < width; x++) {
-                contents[idx++] = (byte) (getColour(backColour.charAt(x), Colour.BLACK) << 4 | getColour(textColour.charAt(x), Colour.WHITE));
+                colours[colourIdx++] = (byte) (
+                    getColour(backColourLine.charAt(x), Colour.BLACK) << 4 |
+                        getColour(textColourLine.charAt(x), Colour.WHITE)
+                );
             }
         }
 
         for (var i = 0; i < Palette.PALETTE_SIZE; i++) {
-            for (var channel : palette.getColour(i)) contents[idx++] = (byte) ((int) (channel * 0xFF) & 0xFF);
+            for (var channel : palette.getColour(i)) {
+                paletteBytes[paletteIdx++] = (byte) ((int) (channel * 0xFF) & 0xFF);
+            }
         }
 
-        assert idx == contents.length;
-        return new TerminalState(colour, width, height, cursorX, cursorY, cursorBlink, cursorColour, cursorBackgroundColour, contents);
+        assert textIdx == textContents.length;
+        assert colourIdx == colours.length;
+        assert paletteIdx == paletteBytes.length;
+
+        return new TerminalState(
+            colour, width, height, cursorX, cursorY, cursorBlink, cursorColour, cursorBackgroundColour,
+            textContents, colours, paletteBytes
+        );
     }
 
     synchronized void read(TerminalState state) {
@@ -50,29 +69,41 @@ synchronized void read(TerminalState state) {
         cursorBackgroundColour = state.cursorBgColour;
         this.cursorColour = state.cursorFgColour;
 
-        var contents = state.contents;
-        var idx = 0;
+        var textContents = state.text;
+        var colours = state.colours;
+        var paletteBytes = state.palette;
+
+        var textIdx = 0;
+        var colourIdx = 0;
+        var paletteIdx = 0;
+
         for (var y = 0; y < height; y++) {
-            var text = this.text[y];
-            var textColour = this.textColour[y];
-            var backColour = backgroundColour[y];
+            var textLine = this.text[y];
+            var textColourLine = this.textColour[y];
+            var backColourLine = backgroundColour[y];
+
+            for (var x = 0; x < width; x++) {
+                textLine.setCodePoint(x, textContents[textIdx++]);
+            }
 
-            for (var x = 0; x < width; x++) text.setChar(x, (char) (contents[idx++] & 0xFF));
             for (var x = 0; x < width; x++) {
-                var colour = contents[idx++];
-                backColour.setChar(x, BASE_16.charAt((colour >> 4) & 0xF));
-                textColour.setChar(x, BASE_16.charAt(colour & 0xF));
+                var packedColour = colours[colourIdx++];
+                backColourLine.setChar(x, BASE_16.charAt((packedColour >> 4) & 0xF));
+                textColourLine.setChar(x, BASE_16.charAt(packedColour & 0xF));
             }
         }
 
         for (var i = 0; i < Palette.PALETTE_SIZE; i++) {
-            var r = (contents[idx++] & 0xFF) / 255.0;
-            var g = (contents[idx++] & 0xFF) / 255.0;
-            var b = (contents[idx++] & 0xFF) / 255.0;
+            var r = (paletteBytes[paletteIdx++] & 0xFF) / 255.0;
+            var g = (paletteBytes[paletteIdx++] & 0xFF) / 255.0;
+            var b = (paletteBytes[paletteIdx++] & 0xFF) / 255.0;
             palette.setColour(i, r, g, b);
         }
 
-        assert idx == contents.length;
+        assert textIdx == textContents.length;
+        assert colourIdx == colours.length;
+        assert paletteIdx == paletteBytes.length;
+
         setChanged();
     }
 
@@ -125,8 +156,8 @@ public synchronized void readFromNBT(CompoundTag nbt) {
                     palette.setColour(i, colours[0], colours[1], colours[2]);
                 }
             }
-
         }
+
         setChanged();
     }
 }

projects/common/src/main/java/dan200/computercraft/shared/computer/terminal/TerminalState.java

@@ -24,10 +24,13 @@ public class TerminalState {
     final boolean cursorBlink;
     final int cursorBgColour;
     final int cursorFgColour;
-    final byte[] contents;
+    final int[] text;
+    final byte[] colours;
+    final byte[] palette;
 
     TerminalState(
-        boolean colour, int width, int height, int cursorX, int cursorY, boolean cursorBlink, int cursorFgColour, int cursorBgColour, byte[] contents
+        boolean colour, int width, int height, int cursorX, int cursorY, boolean cursorBlink, int cursorFgColour, int cursorBgColour,
+        int[] text, byte[] colours, byte[] palette
     ) {
         this.colour = colour;
         this.width = width;
@@ -37,7 +40,9 @@ public class TerminalState {
         this.cursorBlink = cursorBlink;
         this.cursorFgColour = cursorFgColour;
         this.cursorBgColour = cursorBgColour;
-        this.contents = contents;
+        this.text = text;
+        this.colours = colours;
+        this.palette = palette;
     }
 
     @Contract("null -> null; !null -> !null")
@@ -57,7 +62,12 @@ public TerminalState(FriendlyByteBuf buf) {
         this.cursorBgColour = (cursorColour >> 4) & 0xF;
         this.cursorFgColour = cursorColour & 0xF;
 
-        contents = buf.readByteArray();
+        var textLength = buf.readVarInt();
+        text = new int[textLength];
+        for (var i = 0; i < textLength; i++) text[i] = buf.readVarInt();
+
+        colours = buf.readByteArray();
+        palette = buf.readByteArray();
     }
 
     public void write(FriendlyByteBuf buf) {
@@ -69,11 +79,15 @@ public void write(FriendlyByteBuf buf) {
         buf.writeBoolean(cursorBlink);
         buf.writeByte(cursorBgColour << 4 | cursorFgColour);
 
-        buf.writeByteArray(contents);
+        buf.writeVarInt(text.length);
+        for (var codepoint : text) buf.writeVarInt(codepoint);
+
+        buf.writeByteArray(colours);
+        buf.writeByteArray(palette);
     }
 
     public int size() {
-        return contents.length;
+        return text.length * Integer.BYTES + colours.length + palette.length;
     }
 
     public void apply(NetworkedTerminal terminal) {

projects/common/src/main/java/dan200/computercraft/shared/network/server/KeyEventServerMessage.java

@@ -41,7 +41,7 @@ protected void handle(ServerNetworkContext context, ComputerMenu container) {
             case UP -> input.keyUp(key);
             case DOWN -> input.keyDown(key, false);
             case REPEAT -> input.keyDown(key, true);
-            case CHAR -> input.charTyped((byte) key);
+            case CHAR -> input.charTyped(key);
         }
     }
 

projects/common/src/main/java/dan200/computercraft/shared/network/server/PasteEventComputerMessage.java

@@ -19,34 +19,25 @@
 /**
  * Paste a string on a {@link ServerComputer}.
  *
- * @see ComputerInput#paste(ByteBuffer)
+ * @see ComputerInput#paste(String)"
  */
 public class PasteEventComputerMessage extends ComputerServerMessage {
-    private final ByteBuffer text;
+    private final String text;
 
-    public PasteEventComputerMessage(AbstractContainerMenu menu, ByteBuffer text) {
+    public PasteEventComputerMessage(AbstractContainerMenu menu, String text) {
         super(menu);
         this.text = text;
     }
 
     public PasteEventComputerMessage(FriendlyByteBuf buf) {
         super(buf);
-
-        var length = buf.readVarInt();
-        if (length > StringUtil.MAX_PASTE_LENGTH) {
-            throw new DecoderException("ByteArray with size " + length + " is bigger than allowed " + StringUtil.MAX_PASTE_LENGTH);
-        }
-
-        var text = new byte[length];
-        buf.readBytes(text);
-        this.text = ByteBuffer.wrap(text);
+        this.text = buf.readUtf(StringUtil.MAX_PASTE_LENGTH);
     }
 
     @Override
     public void write(FriendlyByteBuf buf) {
         super.write(buf);
-        buf.writeVarInt(text.remaining());
-        buf.writeBytes(text.duplicate());
+        buf.writeUtf(text, StringUtil.MAX_PASTE_LENGTH);
     }
 
     @Override