Add JSON encoding with simdjson 4.6.4

.github/workflows/ci.yml

@@ -70,3 +70,29 @@ jobs:
           luarocks install lua-cjson2
           luarocks install busted
           busted --verbose
+
+  sanitizers:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libreadline-dev
+          pip install git+https://github.com/luarocks/hererocks
+          hererocks lua_install -r^ --lua=5.4
+      - name: Build with AddressSanitizer and UndefinedBehaviorSanitizer
+        run: |
+          source lua_install/bin/activate
+          luarocks config variables.LIBFLAG -- "-shared -fsanitize=address,undefined"
+          CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer" \
+          luarocks make
+      - name: Run tests with sanitizers
+        run: |
+          source lua_install/bin/activate
+          luarocks install lua-cjson2
+          luarocks install busted
+          export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so):$(gcc -print-file-name=libubsan.so)"
+          export ASAN_OPTIONS="detect_leaks=1:halt_on_error=1"
+          export UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1"
+          busted --verbose

Makefile

@@ -1,6 +1,6 @@
-OBJ = src/luasimdjson.o src/simdjson.o
+OBJ = src/luasimdjson.o src/lua_encoder.o src/simdjson.o
 CPPFLAGS = -I$(LUA_INCDIR)
-CXXFLAGS = -std=c++11 -Wall -fvisibility=hidden $(CFLAGS)
+CXXFLAGS = -std=c++17 -Wall -fvisibility=hidden $(CFLAGS)
 LDFLAGS = $(LIBFLAG)
 LDLIBS = -lpthread
 
@@ -35,7 +35,7 @@ $(TARGET): $(OBJ)
 	$(CXX) $(LDFLAGS) $^ -o $@ $(LDLIBS)
 
 clean:
-	rm -f *.$(LIBEXT) src/*.{o,d}
+	rm -f *.$(LIBEXT) src/*.o src/*.d
 
 install: $(TARGET)
 	cp $(TARGET) $(INST_LIBDIR)

Makefile.win

@@ -1,4 +1,4 @@
-OBJ = src/luasimdjson.obj src/simdjson.obj
+OBJ = src/luasimdjson.obj src/lua_encoder.obj src/simdjson.obj
 CPPFLAGS = -I$(LUA_INCDIR)
 CXXFLAGS = -EHsc -std:c++17 $(CFLAGS)
 LDFLAGS = $(LIBFLAG)
@@ -12,6 +12,7 @@ TARGET = simdjson.dll
 all: $(TARGET)
 
 src/luasimdjson.obj: src/luasimdjson.h src/simdjson.h
+src/lua_encoder.obj: src/lua_encoder.h src/simdjson.h
 src/simdjson.obj: src/simdjson.h
 
 .cpp.obj::

README.md

@@ -3,7 +3,7 @@
 
 A basic Lua binding to [simdjson](https://simdjson.org). The simdjson library is an incredibly fast JSON parser that uses SIMD instructions and fancy algorithms to parse JSON very quickly. It's been tested with LuaJIT 2.0/2.1 and Lua 5.1 to 5.5 on linux/osx/windows. It has a general parsing mode and a lazy mode that uses a JSON pointer.
 
-Current simdjson version: 4.2.4
+Current simdjson version: 4.6.4
 
 ## Installation
 If all the requirements are met, lua-simdjson can be install via luarocks with:
@@ -15,7 +15,7 @@ Otherwise it can be installed manually by pulling the repo and running luarocks
 
 ## Requirements
  * lua-simdjson only works on 64bit systems.
- * a Lua build environment with support for C++11
+ * a Lua build environment with support for C++17
    * g++ version 7+ and clang++ version 6+ or newer should work!
 
 ## Parsing
@@ -93,6 +93,37 @@ The `open` and `parse` codeblocks should print out the same values. It's worth n
 
 This lazy style of using the simdjson data structure could also be used with array access in the future.
 
+## Encoding
+The `encode` method converts Lua values and tables into JSON strings. Its optional second argument is a table so that encoding options can be extended without changing the function signature.
+
+```lua
+local simdjson = require("simdjson")
+
+local data = {
+    name = "Alice",
+    active = true,
+    scores = {95, 87.5, 92}
+}
+
+local json = simdjson.encode(data)
+local limited = simdjson.encode(data, {
+    maxDepth = 10,
+    bufferSize = 8192
+})
+```
+
+The default maximum depth and initial buffer size can also be configured globally:
+
+```lua
+simdjson.setMaxEncodeDepth(128)
+simdjson.setEncodeBufferSize(32 * 1024)
+
+local maxDepth = simdjson.getMaxEncodeDepth()
+local bufferSize = simdjson.getEncodeBufferSize()
+```
+
+Tables containing consecutive positive integer keys from 1 through n are encoded as arrays. Sparse and mixed-key tables are encoded as objects, which prevents small tables with very large indices from expanding into enormous arrays. `simdjson.null` represents JSON `null`. Numbers and booleans are formatted by simdjson's string builder, encoded strings are validated as UTF-8, non-finite numbers are rejected, and cyclic tables produce an error. `maxDepth` is limited to 128 to protect the native stack, and the initial `bufferSize` is capped at 64 MiB.
+
 ## Error Handling
 lua-simdjson will error out with any errors from simdjson encountered while parsing. They are very good at helping identify what has gone wrong during parsing.
 
@@ -110,13 +141,12 @@ All tested files are in the [jsonexamples folder](jsonexamples/).
 lua-simdjson, like the simdjson library performs better on more modern hardware. These benchmarks were run on a ninth-gen i7 processor. On an older processor, rapidjson may perform better.
 
 ## Caveats & Alternatives
- * there is no encoding/dumping a Lua table to JSON (yet! Most other lua JSON libraries can handle this)
  * it only works on 64 bit systems
  * it builds a large binary. On a modern linux system, it ended up being \~200k (lua-cjson comes in at 42k)
  * since it's an external module, it's not quite as easy to just grab the file and go (dkjson has you covered here!)
 
 ## Philosophy
-I plan to keep it fairly inline with what the original simdjson library is capable of doing, which really means not adding too many additional options. The big _thing_ that's missing so far is encoding a lua table to JSON. I may add in an encoder at some point.
+I plan to keep it fairly inline with what the original simdjson library is capable of doing, which really means not adding too many additional options.
 
 ## Licenses
  * The jsonexamples, src/simdjson.cpp, src/simdjson.h are unmodified from the released version simdjson under the Apache License 2.0.

spec/encode_fuzz_spec.lua

@@ -0,0 +1,45 @@
+local simdjson = require("simdjson")
+
+local function random_string()
+    local bytes = {}
+    for _ = 1, math.random(0, 32) do
+        bytes[#bytes + 1] = string.char(math.random(0, 127))
+    end
+    return table.concat(bytes)
+end
+
+local function random_value(depth)
+    local kind = math.random(1, depth > 0 and 6 or 4)
+    if kind == 1 then
+        return math.random(-1000000, 1000000) / math.random(1, 100)
+    elseif kind == 2 then
+        return random_string()
+    elseif kind == 3 then
+        return math.random(0, 1) == 1
+    elseif kind == 4 then
+        return simdjson.null
+    elseif kind == 5 then
+        local array = {}
+        for i = 1, math.random(1, 8) do
+            array[i] = random_value(depth - 1)
+        end
+        return array
+    else
+        local object = {}
+        for _ = 1, math.random(1, 8) do
+            object[random_string()] = random_value(depth - 1)
+        end
+        return object
+    end
+end
+
+describe("simdjson.encode randomized inputs", function()
+    it("always emits parseable JSON for supported acyclic values", function()
+        math.randomseed(109)
+        for _ = 1, 1000 do
+            local encoded = simdjson.encode(random_value(4))
+            local ok, error_message = pcall(simdjson.parse, encoded)
+            assert.is_true(ok, error_message)
+        end
+    end)
+end)

spec/encode_spec.lua

@@ -0,0 +1,185 @@
+local simdjson = require("simdjson")
+
+describe("simdjson.encode", function()
+    local original_max_depth
+    local original_buffer_size
+
+    before_each(function()
+        original_max_depth = simdjson.getMaxEncodeDepth()
+        original_buffer_size = simdjson.getEncodeBufferSize()
+    end)
+
+    after_each(function()
+        simdjson.setMaxEncodeDepth(original_max_depth)
+        simdjson.setEncodeBufferSize(original_buffer_size)
+    end)
+
+    it("encodes scalar values", function()
+        assert.are.equal('"value"', simdjson.encode("value"))
+        assert.are.equal(42, simdjson.parse(simdjson.encode(42)))
+        assert.are.equal("1.5", simdjson.encode(1.5))
+        assert.are.equal("true", simdjson.encode(true))
+        assert.are.equal("false", simdjson.encode(false))
+        assert.are.equal("null", simdjson.encode(simdjson.null))
+    end)
+
+    it("uses simdjson's shortest number formatting", function()
+        local values = {0.0, 3.14159265358979, 1.23e-10, 1.23e10, -123.456}
+
+        for _, value in ipairs(values) do
+            local encoded = simdjson.encode(value)
+            assert.are.equal(value, simdjson.parse(encoded))
+        end
+
+        assert.are.equal("1.5", simdjson.encode(1.5))
+        assert.are.equal("2.7", simdjson.encode(2.7))
+    end)
+
+    it("preserves Lua integers on Lua 5.3 and newer", function()
+        if math.type then
+            local encoded = simdjson.encode(9223372036854775807)
+            assert.are.equal("9223372036854775807", encoded)
+            assert.are.equal("integer", math.type(simdjson.parse(encoded)))
+        end
+    end)
+
+    it("encodes booleans at every nesting position", function()
+        assert.are.equal("[true,false,true]", simdjson.encode({true, false, true}))
+
+        local decoded = simdjson.parse(simdjson.encode({
+            enabled = true,
+            nested = {disabled = false},
+            values = {false, true}
+        }))
+
+        assert.is_true(decoded.enabled)
+        assert.is_false(decoded.nested.disabled)
+        assert.is_false(decoded.values[1])
+        assert.is_true(decoded.values[2])
+    end)
+
+    it("encodes arrays, objects, and numeric object keys", function()
+        local decoded = simdjson.parse(simdjson.encode({
+            items = {1, "two", true},
+            object = {answer = 42},
+            keyed = {[0] = "zero", value = "other"}
+        }))
+
+        assert.are.equal(1, decoded.items[1])
+        assert.are.equal("two", decoded.items[2])
+        assert.is_true(decoded.items[3])
+        assert.are.equal(42, decoded.object.answer)
+        assert.are.equal("zero", decoded.keyed[simdjson.encode(0)])
+        assert.are.equal("{}", simdjson.encode({}))
+    end)
+
+    it("encodes sparse numeric tables without output amplification", function()
+        local encoded = simdjson.encode({[1] = "first", [1000000] = "last"})
+        local decoded = simdjson.parse(encoded)
+
+        assert.is_true(#encoded < 100)
+        assert.are.equal("first", decoded[simdjson.encode(1)])
+        assert.are.equal("last", decoded[simdjson.encode(1000000)])
+    end)
+
+    it("supports per-call options", function()
+        local value = {child = {value = true}}
+        assert.has_error(function()
+            simdjson.encode(value, {maxDepth = 1})
+        end)
+
+        local encoded = simdjson.encode(value, {maxDepth = 2, bufferSize = 8})
+        assert.is_true(simdjson.parse(encoded).child.value)
+    end)
+
+    it("supports global encode settings", function()
+        simdjson.setMaxEncodeDepth(64)
+        simdjson.setEncodeBufferSize(1024)
+        assert.are.equal(64, simdjson.getMaxEncodeDepth())
+        assert.are.equal(1024, simdjson.getEncodeBufferSize())
+    end)
+
+    it("grows beyond the initial string builder capacity", function()
+        local value = string.rep("abcdef", 4096)
+        local encoded = simdjson.encode({value = value}, {bufferSize = 8})
+        assert.are.equal(value, simdjson.parse(encoded).value)
+    end)
+
+    it("rejects unsupported values and invalid options", function()
+        assert.has_error(function()
+            simdjson.encode(function() end)
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {maxDepth = 0})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {bufferSize = 0})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {max_depth = 10})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {maxDepth = 1.5})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {bufferSize = 1.5})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {maxDepth = "10"})
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {bufferSize = 64 * 1024 * 1024 + 1})
+        end)
+    end)
+
+    it("preserves embedded NUL bytes", function()
+        local value = "before\0after"
+        assert.are.equal(value, simdjson.parse(simdjson.encode(value)))
+    end)
+
+    it("rejects non-finite numbers", function()
+        for _, value in ipairs({0 / 0, math.huge, -math.huge}) do
+            assert.has_error(function()
+                simdjson.encode(value)
+            end)
+        end
+    end)
+
+    it("rejects direct and indirect table cycles", function()
+        local direct = {}
+        direct.self = direct
+
+        local first = {}
+        local second = {first = first}
+        first.second = second
+
+        assert.has_error(function()
+            simdjson.encode(direct)
+        end)
+        assert.has_error(function()
+            simdjson.encode(first)
+        end)
+    end)
+
+    it("recovers after encoding errors", function()
+        local cyclic = {}
+        cyclic.self = cyclic
+
+        assert.has_error(function()
+            simdjson.encode(cyclic)
+        end)
+        assert.has_error(function()
+            simdjson.encode("\255")
+        end)
+        assert.are.equal('{"valid":true}', simdjson.encode({valid = true}))
+    end)
+
+    it("rejects overflowing configuration values", function()
+        assert.has_error(function()
+            simdjson.setMaxEncodeDepth(4294967297)
+        end)
+        assert.has_error(function()
+            simdjson.encode({}, {maxDepth = 4294967297})
+        end)
+    end)
+end)