Non-functional OpenZL prototype

Author: lemire

CMakeLists.txt

@@ -163,6 +163,11 @@ if(ENABLE_ROARING_TESTS AND BASH AND NOT EMSCRIPTEN)
 else()
   message(STATUS "Amalgamation tests disabled")
 endif()
+option(ROARING_OPENZL "Enable OpenZL integration" OFF)
+if(ROARING_OPENZL)
+  add_subdirectory(openzl)
+endif()
+
 option(ENABLE_ROARING_MICROBENCHMARKS "Enable microbenchmarks" OFF)
 if(ENABLE_ROARING_MICROBENCHMARKS)
     add_subdirectory(microbenchmarks)

README.md

@@ -40,6 +40,7 @@ Portable Roaring bitmaps in C (and C++) with full support for your favorite comp
 - [Mailing list/discussion group](#mailing-listdiscussion-group)
 - [Contributing](#contributing)
 - [References about Roaring](#references-about-roaring)
+- [OpenZL Integration](#openzl-integration)
 
 # Introduction
 
@@ -1189,6 +1190,27 @@ A compiler or static-analyzer warning is not a bug. Do not report such cases as
 
 [![Star History Chart](https://api.star-history.com/svg?repos=RoaringBitmap/CRoaring&type=Date)](https://www.star-history.com/#RoaringBitmap/CRoaring&Date)
 
+# OpenZL Integration
+
+The `openzl/` subdirectory provides an [OpenZL](https://github.com/facebook/openzl) integration with an SDDL description of the [Roaring Bitmap serialization format](https://github.com/RoaringBitmap/RoaringFormatSpec). This enables format-aware compression of serialized Roaring Bitmap data using OpenZL's specialized compressor.
+
+To enable OpenZL support, set the  `ROARING_OPENZL` flag during configuration:
+
+```
+cmake -DROARING_OPENZL=ON -B build
+```
+
+The SDDL file (`openzl/roaring.sddl`) describes the portable Roaring Bitmap binary format, including:
+
+- Cookie-based format detection (with and without run containers)
+- Container descriptors (key + cardinality)
+- All three container types: array, bitset, and run-length encoded
+- Offset headers for random access
+
+See the [SDDL documentation](https://openzl.org/sddl/) for more on the format description language.
+
+
+
 # References about Roaring
 
 - Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience Volume 48, Issue 4 April 2018 Pages 867-895 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821)

openzl/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(OPENZL_BUILD_TOOLS ON CACHE BOOL "Build OpenZL tools (needed for SDDL compiler)" FORCE)
+
+CPMAddPackage(
+  NAME openzl
+  GIT_REPOSITORY https://github.com/facebook/openzl.git
+  GIT_TAG 35300cebf0bca276fae7034aa511045df0de1936
+)
+
+if(ENABLE_ROARING_TESTS)
+  add_subdirectory(test)
+endif()
+
+add_subdirectory(compression_benchmark)

openzl/compression_benchmark/CMakeLists.txt

@@ -0,0 +1,34 @@
+set(SDDL_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../roaring.sddl")
+set(SDDL_COMPILED "${CMAKE_CURRENT_BINARY_DIR}/roaring_sddl.bin")
+set(SDDL_HEADER "${CMAKE_CURRENT_BINARY_DIR}/roaring_sddl.h")
+
+# Compile the SDDL source to binary using the sddl_compiler tool.
+add_custom_command(
+  OUTPUT ${SDDL_COMPILED}
+  COMMAND sddl_compiler < ${SDDL_SOURCE} > ${SDDL_COMPILED}
+  DEPENDS ${SDDL_SOURCE} sddl_compiler
+  COMMENT "Compiling roaring.sddl"
+)
+
+# Generate a C header embedding the compiled SDDL as a byte array.
+add_custom_command(
+  OUTPUT ${SDDL_HEADER}
+  COMMAND ${CMAKE_COMMAND}
+    -DINPUT_FILE=${SDDL_COMPILED}
+    -DOUTPUT_FILE=${SDDL_HEADER}
+    -DARRAY_NAME=roaring_sddl_compiled
+    -DSIZE_NAME=roaring_sddl_compiled_size
+    -DGUARD_NAME=ROARING_SDDL_H
+    -P ${CMAKE_CURRENT_SOURCE_DIR}/bin2header.cmake
+  DEPENDS ${SDDL_COMPILED}
+  COMMENT "Generating roaring_sddl.h"
+)
+
+add_executable(compression_benchmark compression_benchmark.c ${SDDL_HEADER})
+target_link_libraries(compression_benchmark roaring openzl sddl_profile)
+target_include_directories(compression_benchmark PRIVATE
+  ${PROJECT_SOURCE_DIR}/include
+  ${PROJECT_SOURCE_DIR}/cpp
+  ${CMAKE_CURRENT_BINARY_DIR}
+  ${openzl_SOURCE_DIR}
+)

openzl/compression_benchmark/bin2header.cmake

@@ -0,0 +1,38 @@
+file(READ "${INPUT_FILE}" CONTENT HEX)
+string(LENGTH "${CONTENT}" HEX_LEN)
+math(EXPR BYTE_COUNT "${HEX_LEN} / 2")
+
+set(OUTPUT "/* Auto-generated — do not edit. */\n")
+string(APPEND OUTPUT "#ifndef ${GUARD_NAME}\n")
+string(APPEND OUTPUT "#define ${GUARD_NAME}\n\n")
+string(APPEND OUTPUT "#include <stddef.h>\n\n")
+string(APPEND OUTPUT "static const unsigned char ${ARRAY_NAME}[] = {\n")
+
+set(POS 0)
+set(LINE "    ")
+set(COL 0)
+while(POS LESS HEX_LEN)
+  string(SUBSTRING "${CONTENT}" ${POS} 2 BYTE)
+  math(EXPR POS "${POS} + 2")
+  if(POS LESS HEX_LEN)
+    string(APPEND LINE "0x${BYTE}, ")
+  else()
+    string(APPEND LINE "0x${BYTE}")
+  endif()
+  math(EXPR COL "${COL} + 1")
+  if(COL EQUAL 12)
+    string(APPEND OUTPUT "${LINE}\n")
+    set(LINE "    ")
+    set(COL 0)
+  endif()
+endwhile()
+
+if(COL GREATER 0)
+  string(APPEND OUTPUT "${LINE}\n")
+endif()
+
+string(APPEND OUTPUT "};\n\n")
+string(APPEND OUTPUT "static const size_t ${SIZE_NAME} = ${BYTE_COUNT};\n\n")
+string(APPEND OUTPUT "#endif /* ${GUARD_NAME} */\n")
+
+file(WRITE "${OUTPUT_FILE}" "${OUTPUT}")