[Refactor] Split ORC format code into FormatOrc module (#74836)

alvin-phoenix-ai · web-flow · commit 53ba468de716 · 2026-06-15T18:41:57.000-07:00
Signed-off-by: alvin-phoenix-ai &lt;alvin.zhao@phoenixdata.ai&gt;
diff --git a/be/AGENTS.md b/be/AGENTS.md
@@ -177,7 +177,7 @@ Core runtime building blocks without full Runtime/Exec/Storage coupling.
 ### FormatCore (`formatcore`)
 Format-oriented core primitives above ComputeEnv, ExprCore, RuntimeCore, FSCore, ChunkCore, ColumnCore, and Types.
 - Targets: `FormatCore`
-- Allowed internal include prefixes: `formats/column_evaluator.h`, `formats/disk_range.hpp`, `formats/file_writer.h`, `formats/io/`, `formats/utils.h`, `exprs/`, `runtime/`, `fs/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/`
+- Allowed internal include prefixes: `formats/column_evaluator.h`, `formats/deletion_bitmap.h`, `formats/disk_range.hpp`, `formats/file_writer.h`, `formats/io/`, `formats/utils.h`, `exprs/`, `runtime/`, `fs/`, `column/`, `types/`, `common/`, `base/`, `gutil/`, `gen_cpp/`
 - Allowed target deps: `ComputeEnv`, `ExprCore`, `RuntimeCore`, `FSCore`, `ChunkCore`, `ColumnCore`, `Types`, `Common`, `Base`, `Gutil`, `StarRocksGen`
 - Core tests: `format_test`
 - Remediation: Keep FormatCore limited to reusable format primitives; move connector orchestration and higher execution policy upward.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -551,11 +551,11 @@ set(STARROCKS_LINK_LIBS
     Udf
     Util
     Script
-    orc
     ${WL_END_GROUP}
     FormatAvro
     FormatCsv
     FormatJson
+    FormatOrc
     FormatCore
     ExecRuntime
     ComputeEnv
diff --git a/be/module_boundary_manifest.json b/be/module_boundary_manifest.json
@@ -315,13 +315,15 @@
       "owned_targets": ["FormatCore"],
       "owned_globs": [
         "be/src/formats/column_evaluator.*",
+        "be/src/formats/deletion_bitmap.*",
         "be/src/formats/disk_range.hpp",
         "be/src/formats/file_writer.*",
         "be/src/formats/io/**",
         "be/src/formats/utils.h"
       ],
       "allowed_include_prefixes": [
         "formats/column_evaluator.h",
+        "formats/deletion_bitmap.h",
         "formats/disk_range.hpp",
         "formats/file_writer.h",
         "formats/io/",
@@ -351,6 +353,7 @@
       ],
       "allowed_test_targets": ["format_test"],
       "allowed_test_link_deps": [
+        "FormatOrc",
         "FormatAvro",
         "FormatJson",
         "FormatCsv",
diff --git a/be/src/connector/CMakeLists.txt b/be/src/connector/CMakeLists.txt
@@ -52,7 +52,6 @@ ADD_BE_LIB(Connector
         partition_chunk_writer.cpp
         connector_sink_executor.cpp
         deletion_vector/deletion_vector.cpp
-        deletion_vector/deletion_bitmap.cpp
         iceberg_delete_sink.cpp
         iceberg_row_delta_sink.cpp
 )
diff --git a/be/src/connector/deletion_vector/deletion_vector.cpp b/be/src/connector/deletion_vector/deletion_vector.cpp
@@ -19,7 +19,7 @@
 #include "base/string/base85.h"
 #include "base/uuid/uuid_generator.h"
 #include "common/config_scan_io_fwd.h"
-#include "deletion_bitmap.h"
+#include "formats/deletion_bitmap.h"
 
 namespace starrocks {
 
diff --git a/be/src/exec/hdfs_scanner/hdfs_scanner.h b/be/src/exec/hdfs_scanner/hdfs_scanner.h
@@ -22,13 +22,13 @@
 #include "cache/scan/shared_buffered_input_stream.h"
 #include "column/column_access_path.h"
 #include "common/runtime_profile.h"
-#include "connector/deletion_vector/deletion_bitmap.h"
 #include "exec/olap_scan_prepare.h"
 #include "exec/pipeline/scan/morsel.h"
 #include "exec/pipeline/scan/scan_morsel.h"
 #include "exec/runtime_filter/runtime_filter_probe.h"
 #include "exprs/expr.h"
 #include "exprs/expr_context.h"
+#include "formats/deletion_bitmap.h"
 #include "fs/fs.h"
 #include "runtime/descriptors.h"
 #include "runtime/runtime_state_fwd.h"
diff --git a/be/src/exec/paimon/paimon_delete_file_builder.cpp b/be/src/exec/paimon/paimon_delete_file_builder.cpp
@@ -20,7 +20,7 @@
 #include <bitset>
 
 #include "base/container/raw_container.h"
-#include "connector/deletion_vector/deletion_bitmap.h"
+#include "formats/deletion_bitmap.h"
 
 namespace starrocks {
 
diff --git a/be/src/formats/CMakeLists.txt b/be/src/formats/CMakeLists.txt
@@ -16,6 +16,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/formats")
 
 ADD_BE_LIB(FormatCore
         column_evaluator.cpp
+        deletion_bitmap.cpp
         disk_range.hpp
         file_writer.cpp
         io/async_flush_output_stream.cpp
@@ -90,7 +91,7 @@ ADD_BE_LIB(FormatAvro
 
 target_link_libraries(FormatAvro PUBLIC FormatCore)
 
-ADD_BE_LIB(Formats
+ADD_BE_LIB(FormatOrc
         orc/orc_chunk_reader.cpp
         orc/orc_file_writer.cpp
         orc/orc_input_stream.cpp
@@ -103,6 +104,11 @@ ADD_BE_LIB(Formats
         orc/column_reader.cpp
         orc/memory_stream/MemoryInputStream.cc
         orc/memory_stream/MemoryOutputStream.cc
+        )
+
+target_link_libraries(FormatOrc PUBLIC FormatCore Cache)
+
+ADD_BE_LIB(Formats
         parquet/arrow_memory_pool.cpp
         parquet/column_chunk_reader.cpp
         parquet/column_materializer.cpp
@@ -136,6 +142,7 @@ ADD_BE_LIB(Formats
         parquet/parquet_pos_reader.cpp
         )
 
-target_link_libraries(Formats PUBLIC FormatAvro FormatJson FormatCsv FormatCore Cache)
+target_link_libraries(Formats PUBLIC FormatOrc FormatAvro FormatJson FormatCsv FormatCore Cache)
 
 add_subdirectory(orc/apache-orc)
+target_link_libraries(FormatOrc PUBLIC orc)
diff --git a/be/src/formats/deletion_bitmap.cpp b/be/src/formats/deletion_bitmap.cpp
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "deletion_bitmap.h"
+#include "formats/deletion_bitmap.h"
 
 #include "base/utility/defer_op.h"
 #include "column/vectorized_fwd.h"
@@ -64,4 +64,4 @@ void DeletionBitmap::add_value(uint64_t val) {
     roaring64_bitmap_add(_bitmap, val);
 }
 
-} // namespace starrocks
+} // namespace starrocks
diff --git a/be/src/formats/deletion_bitmap.h b/be/src/formats/deletion_bitmap.h
@@ -17,6 +17,7 @@
 #include <roaring/roaring64.h>
 
 #include <memory>
+#include <vector>
 
 #include "column/vectorized_fwd.h"
 #include "common/statusor.h"
@@ -47,4 +48,4 @@ class DeletionBitmap {
 
 using DeletionBitmapPtr = std::shared_ptr<DeletionBitmap>;
 
-} // namespace starrocks
+} // namespace starrocks
diff --git a/be/src/starrocks_format/CMakeLists.txt b/be/src/starrocks_format/CMakeLists.txt
@@ -33,7 +33,6 @@ set(STARROCKS_LIBS
     FileSystem
     FSCore
     Formats
-    orc
     Gutil
     HttpCore
     IO
@@ -49,6 +48,7 @@ set(STARROCKS_LIBS
     Webserver
     Cache
     ${WL_END_GROUP}
+    FormatOrc
 )
 
 add_library(hdfs_so SHARED IMPORTED GLOBAL)
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
@@ -202,11 +202,7 @@ set(EXEC_FILES
         ./formats/avro/cpp/numeric_column_reader_test.cpp
         ./formats/avro/cpp/utils_test.cpp
         ./formats/orc/orc_chunk_reader_test.cpp
-        ./formats/orc/orc_column_reader_test.cpp
-        ./formats/orc/orc_emit_rejected_rows_test.cpp
         ./formats/orc/orc_file_writer_test.cpp
-        ./formats/orc/orc_lazy_load_test.cpp
-        ./formats/orc/utils_test.cpp
         ./formats/parquet/arrow_parquet_writer_test.cpp
         ./formats/parquet/parquet_schema_test.cpp
         ./formats/parquet/encoding_test.cpp
diff --git a/be/test/formats/CMakeLists.txt b/be/test/formats/CMakeLists.txt
@@ -57,7 +57,15 @@ set(FORMAT_AVRO_TEST_FILES
     avro/cpp/utils_test.cpp
 )
 
+set(FORMAT_ORC_TEST_FILES
+    orc/orc_column_reader_test.cpp
+    orc/orc_emit_rejected_rows_test.cpp
+    orc/orc_lazy_load_test.cpp
+    orc/utils_test.cpp
+)
+
 set(FORMAT_TEST_LINK_LIBS
+    FormatOrc
     FormatAvro
     FormatJson
     FormatCsv
@@ -97,11 +105,16 @@ add_library(starrocks_format_avro_test_objs OBJECT ${FORMAT_AVRO_TEST_FILES})
 target_link_libraries(starrocks_format_avro_test_objs PRIVATE StarRocksPCH)
 set_target_properties(starrocks_format_avro_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control")
 
+add_library(starrocks_format_orc_test_objs OBJECT ${FORMAT_ORC_TEST_FILES})
+target_link_libraries(starrocks_format_orc_test_objs PRIVATE StarRocksPCH)
+set_target_properties(starrocks_format_orc_test_objs PROPERTIES COMPILE_FLAGS "-fno-access-control")
+
 add_executable(format_test
     format_test_main.cpp
     $<TARGET_OBJECTS:starrocks_format_core_test_objs>
     $<TARGET_OBJECTS:starrocks_format_csv_test_objs>
     $<TARGET_OBJECTS:starrocks_format_json_test_objs>
     $<TARGET_OBJECTS:starrocks_format_avro_test_objs>
+    $<TARGET_OBJECTS:starrocks_format_orc_test_objs>
 )
 target_link_libraries(format_test ${FORMAT_TEST_LINK_LIBS})
diff --git a/be/test/formats/format_test_main.cpp b/be/test/formats/format_test_main.cpp
@@ -17,9 +17,16 @@
 #include <cstdio>
 
 #include "common/configbase.h"
+#include "formats/orc/lzo_decompressor_registration.h"
 #include "types/time_types.h"
 
 int main(int argc, char** argv) {
+    auto lzo_status = starrocks::register_orc_lzo_decompressor();
+    if (!lzo_status.ok()) {
+        std::fprintf(stderr, "fail to register ORC LZO decompressor: %s\n", lzo_status.to_string().c_str());
+        return 1;
+    }
+
     if (!starrocks::config::init(nullptr)) {
         std::fprintf(stderr, "failed to initialize config defaults\n");
         return 1;

Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,6 @@ ADD_BE_LIB(Connector`
`52`	`52`	`partition_chunk_writer.cpp`
`53`	`53`	`connector_sink_executor.cpp`
`54`	`54`	`deletion_vector/deletion_vector.cpp`
`55`		`- deletion_vector/deletion_bitmap.cpp`
`56`	`55`	`iceberg_delete_sink.cpp`
`57`	`56`	`iceberg_row_delta_sink.cpp`
`58`	`57`	`)`