fix: preserve 64-bit integers in Tarantool builds

Fix: preserve 64-bit integers in Tarantool builds

lua_pushinteger loses precision for values beyond double mantissa,
so large JSON integers were rounded (e.g. 123456789012345678,
123456789012345680). Use luaL_pushint64/luaL_pushuint64 in TT_COMPAT
builds to match tarantool json behavior: small ints as numbers, large
ints as cdata (long/unsigned long).

Author: alrsorokin

.github/workflows/ci.yml

@@ -3,33 +3,56 @@ name: CI
 on: push
 
 jobs:
-  lua:
+  test:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - lua: lua=5.1
-          - lua: lua=5.2
-          - lua: lua=5.3
-          - lua: lua=5.4
-          - lua: luajit=2.0
-          - lua: luajit=2.1
+        lua: [lua5.1, lua5.2, lua5.3, lua5.4, luajit2.0, luajit2.1]
+
     runs-on: ubuntu-22.04
+
     steps:
-      # Checks-out the repository under $GITHUB_WORKSPACE.
-      - uses: actions/checkout@v3
-      - name: Install Lua (${{ matrix.lua }})
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hererocks
+        run: pip install hererocks
+
+      - name: Setup Lua (${{ matrix.lua }})
         run: |
-          pip install hererocks
-          hererocks lua_install -r^ --${{ matrix.lua }}
-          export PATH=$PATH:$PWD/lua_install/bin
-          luarocks install lua-cjson2
-      - name: Build lua-simdjson
+          case "${{ matrix.lua }}" in
+            lua5.1)    FLAG="--lua=5.1" ;;
+            lua5.2)    FLAG="--lua=5.2" ;;
+            lua5.3)    FLAG="--lua=5.3" ;;
+            lua5.4)    FLAG="--lua=5.4" ;;
+            luajit2.0) FLAG="--luajit=2.0" ;;
+            luajit2.1) FLAG="--luajit=2.1" ;;
+          esac
+          hererocks lua_env -r^ $FLAG
+          echo "LUA_ENV=$PWD/lua_env" >> $GITHUB_ENV
+          echo "$PWD/lua_env/bin" >> $GITHUB_PATH
+
+      - name: Upgrade LuaRocks (LuaJIT only)
+        if: startsWith(matrix.lua, 'luajit')
         run: |
-          export PATH=$PATH:$PWD/lua_install/bin
-          luarocks make
-      - name: Run tests
+          # hererocks bundles LuaRocks 3.8; its manifest exceeds LuaJIT's
+          # 65536 constants limit. Install LuaRocks 3.12 manually instead.
+          curl -fsSL https://luarocks.org/releases/luarocks-3.12.0.tar.gz | tar xz
+          cd luarocks-3.12.0
+          ./configure --with-lua="$LUA_ENV" --prefix="$LUA_ENV"
+          make build install
+
+      - name: Install dependencies
         run: |
-          export PATH=$PATH:$PWD/lua_install/bin
+          luarocks install lua-cjson2
           luarocks install busted
-          busted --verbose
+
+      - name: Build project
+        run: luarocks make
+
+      - name: Run tests
+        run: busted --verbose

lua-simdjson-pico.0.0.4-1.rockspec

@@ -0,0 +1,41 @@
+-- Rockspecs template for picodata version.
+-- Copy it, replace tag in source and change version.
+package="lua-simdjson"
+version="pico.0.0.4-1"
+source = {
+   url = "git://github.com/picodata/lua-simdjson",
+   tag = "0.0.4"
+}
+description = {
+   summary = "This is a simple Lua binding for simdjson",
+   detailed = [[
+      This is a c++ binding to simdjson for parsing JSON very quickly.
+   ]],
+   homepage = "https://github.com/picodata/lua-simdjson",
+   license = "Apache-2.0"
+}
+dependencies = {
+   "lua >= 5.1, < 5.5"
+}
+
+build = {
+   type = "make",
+   build_variables = {
+      CFLAGS="$(CFLAGS) -D TT_COMPAT",
+      -- add compat sources as well(normally they are excluded).
+      SRC="src/tt_compat.cpp",
+      LIBFLAG="$(LIBFLAG)",
+      LUA_LIBDIR="$(LUA_LIBDIR)",
+      LUA_BINDIR="$(LUA_BINDIR)",
+      LUA_INCDIR="$(LUA_INCDIR)",
+      LUA="$(LUA)",
+   },
+   install_variables = {
+      INST_PREFIX="$(PREFIX)",
+      INST_BINDIR="$(BINDIR)",
+      INST_LIBDIR="$(LIBDIR)",
+      INST_LUADIR="$(LUADIR)",
+      INST_CONFDIR="$(CONFDIR)",
+   },
+}
+

src/luasimdjson.cpp

@@ -86,11 +86,19 @@ void convert_element_to_table(lua_State *L, dom::element element) {
       break;
 
     case dom::element_type::INT64:
-      lua_pushinteger(L, int64_t(element));
+      #ifdef TT_COMPAT
+        tt_lua_pushinteger64(L, int64_t(element));
+      #else
+        lua_pushinteger(L, int64_t(element));
+      #endif
       break;
 
     case dom::element_type::UINT64:
-      lua_pushinteger(L, int64_t(element));
+      #ifdef TT_COMPAT
+        tt_lua_pushuint64(L, uint64_t(element));
+      #else
+        lua_pushinteger(L, int64_t(element));
+      #endif
       break;
 
     case dom::element_type::DOUBLE:

src/tt_compat.cpp

@@ -1,7 +1,11 @@
+#include <cstdint>
 #include <lua.hpp>
 #include <lauxlib.h>
+#include <module.h>
 #include "tt_compat.h"
 
+static constexpr int64_t TT_INT64_THRESHOLD = INT64_C(100000000000000);
+
 char* SER_MARKER_MAP = (char*)"map";
 char* SER_MARKER_SEQ = (char*)"seq";
 
@@ -21,3 +25,19 @@ void tt_compat_init(lua_State *L) {
   tt_lua_createnull(L);
 }
 
+void tt_lua_pushinteger64(lua_State *L, int64_t val) {
+  if (val >= TT_INT64_THRESHOLD || val <= -TT_INT64_THRESHOLD) {
+    luaL_pushint64(L, val);
+  } else {
+    lua_pushnumber(L, static_cast<lua_Number>(val));
+  }
+}
+
+void tt_lua_pushuint64(lua_State *L, uint64_t val) {
+  if (val > static_cast<uint64_t>(INT64_MAX) || val >= static_cast<uint64_t>(TT_INT64_THRESHOLD)) {
+    luaL_pushuint64(L, val);
+  } else {
+    lua_pushnumber(L, static_cast<lua_Number>(val));
+  }
+}
+

src/tt_compat.h

@@ -1,3 +1,4 @@
+#include <cstdint>
 #include <lua.hpp>
 #include <lauxlib.h>
 
@@ -33,3 +34,9 @@ inline void tt_lua_push_serialize_mt(lua_State *L, char* marker) {
   lua_pushstring(L, marker);
   lua_settable(L, -3);
 }
+
+/* Push 64-bit integers the same way as tarantool's json module:
+ * small values as lua numbers, large values as cdata (long/unsigned long).
+ */
+void tt_lua_pushinteger64(lua_State *L, int64_t val);
+void tt_lua_pushuint64(lua_State *L, uint64_t val);

tests/tarantool_test.lua

@@ -26,6 +26,20 @@ g.test_compat_with_tarantool_json = function()
   end
 end
 
+g.test_large_integers = function()
+  local cases = {
+    '42',
+    '100000000000000',
+    '123456789012345678',
+    '-100000000000000',
+    '9223372036854775808',
+  }
+  for _, num in ipairs(cases) do
+    local raw_json = '{"n":' .. num .. '}'
+    t.assert_equals(simdjson.parse(raw_json), tt_json.decode(raw_json), num)
+  end
+end
+
 g.test_serialize_markers = function ()
   local result = simdjson.parse('{"nested": {"emptyMap": {}}, "emptyArray": []}')
   t.assert_equals(getmetatable(result), {__serialize = "map"})