Skip to content

Commit cda20eb

Browse files
jackylee-chjackylee-ch
committed
[VL][TableCache] Add default lazy per-column deserialization
Write V3 per-column cache bytes by default for Velox table cache. Partition stats now only controls the optional stats/pruning payload: stats off writes a no-stats V3 frame, stats on writes V3 with stats, and older native libraries still fall back to V2 stats or legacy bytes. Add the V3 no-stats JNI/native serializer, JVM parsing for statsLen=0, cross-language golden coverage, and GitHub Actions benchmark execution without committing local benchmark results. Change-Id: I2a8582f901fafd436cac1a1d16e0367e9330b336
1 parent 0dabb55 commit cda20eb

22 files changed

Lines changed: 2599 additions & 240 deletions

File tree

‎.github/workflows/velox_backend_x86.yml‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,33 @@
1616
name: Velox Backend (x86)
1717

1818
on:
19+
workflow_dispatch:
20+
inputs:
21+
table_cache_benchmark_rows:
22+
description: 'Rows for ColumnarTableCacheLazyDeserBenchmark manual evidence runs'
23+
required: false
24+
default: '5000000'
25+
type: string
26+
table_cache_benchmark_partitions:
27+
description: 'Partitions for ColumnarTableCacheLazyDeserBenchmark manual evidence runs'
28+
required: false
29+
default: '32'
30+
type: string
31+
table_cache_benchmark_iterations:
32+
description: 'Iterations for ColumnarTableCacheLazyDeserBenchmark manual evidence runs'
33+
required: false
34+
default: '3'
35+
type: string
36+
table_cache_benchmark_phases:
37+
description: 'Comma-separated phases for ColumnarTableCacheLazyDeserBenchmark manual evidence runs'
38+
required: false
39+
default: 'build,read1,read4,readAll,filter'
40+
type: string
41+
table_cache_benchmark_include_legacy_baseline:
42+
description: 'Whether to include old eager/raw no-stats baseline in benchmark evidence'
43+
required: false
44+
default: 'true'
45+
type: string
1946
pull_request:
2047
paths:
2148
- '.github/workflows/velox_backend_x86.yml'
@@ -1188,13 +1215,74 @@ jobs:
11881215
cd ./cpp/build && ctest -V
11891216
- name: Run CPP benchmark test
11901217
run: |
1191-
$MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \
1218+
$MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \
11921219
-DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip
11931220
# This test depends on files generated by the above mvn test.
11941221
./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \
11951222
--conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \
11961223
--plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \
11971224
--data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
1225+
- name: Run table cache lazy deserialization benchmark
1226+
if: github.event_name == 'workflow_dispatch'
1227+
timeout-minutes: 60
1228+
env:
1229+
TABLE_CACHE_BENCHMARK_ROWS: ${{ inputs.table_cache_benchmark_rows || '5000000' }}
1230+
TABLE_CACHE_BENCHMARK_PARTITIONS: ${{ inputs.table_cache_benchmark_partitions || '32' }}
1231+
TABLE_CACHE_BENCHMARK_ITERATIONS: ${{ inputs.table_cache_benchmark_iterations || '3' }}
1232+
TABLE_CACHE_BENCHMARK_PHASES: ${{ inputs.table_cache_benchmark_phases || 'build,read1,read4,readAll,filter' }}
1233+
TABLE_CACHE_BENCHMARK_INCLUDE_LEGACY_BASELINE: ${{ inputs.table_cache_benchmark_include_legacy_baseline || 'true' }}
1234+
run: |
1235+
set -o pipefail
1236+
mkdir -p benchmarks
1237+
RESULT_FILE=benchmarks/ColumnarTableCacheLazyDeserBenchmark-results.txt
1238+
{
1239+
echo "ColumnarTableCacheLazyDeserBenchmark GHA config"
1240+
echo "commit=${GITHUB_SHA}"
1241+
echo "event=${GITHUB_EVENT_NAME}"
1242+
echo "run_id=${GITHUB_RUN_ID}"
1243+
echo "run_attempt=${GITHUB_RUN_ATTEMPT}"
1244+
echo "run_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
1245+
echo "ref=${GITHUB_REF}"
1246+
echo "head_commit=$(git rev-parse HEAD)"
1247+
echo "head_parent=$(git rev-parse HEAD^ 2>/dev/null || true)"
1248+
echo "head_ref=${GITHUB_HEAD_REF}"
1249+
echo "base_ref=${GITHUB_BASE_REF}"
1250+
echo "workflow=${GITHUB_WORKFLOW}"
1251+
echo "job=${GITHUB_JOB}"
1252+
echo "runner_os=${RUNNER_OS}"
1253+
echo "runner_arch=${RUNNER_ARCH}"
1254+
echo "java_home=${JAVA_HOME:-}"
1255+
echo "uname=$(uname -a)"
1256+
echo "cpu_count=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo unknown)"
1257+
echo "spark_profile=spark-3.5"
1258+
echo "scala_profile=scala-2.12"
1259+
echo "backend=velox"
1260+
echo "rows=${TABLE_CACHE_BENCHMARK_ROWS}"
1261+
echo "partitions=${TABLE_CACHE_BENCHMARK_PARTITIONS}"
1262+
echo "iterations=${TABLE_CACHE_BENCHMARK_ITERATIONS}"
1263+
echo "phases=${TABLE_CACHE_BENCHMARK_PHASES}"
1264+
echo "includeLegacyBaseline=${TABLE_CACHE_BENCHMARK_INCLUDE_LEGACY_BASELINE}"
1265+
java -version 2>&1
1266+
} | tee "${RESULT_FILE}"
1267+
export MAVEN_OPTS="-Xss128m -Xmx8g -XX:ReservedCodeCacheSize=2g \
1268+
-Dspark.test.home=/opt/shims/spark35/spark_home/ \
1269+
-Dspark.gluten.benchmark.rows=${TABLE_CACHE_BENCHMARK_ROWS} \
1270+
-Dspark.gluten.benchmark.partitions=${TABLE_CACHE_BENCHMARK_PARTITIONS} \
1271+
-Dspark.gluten.benchmark.iterations=${TABLE_CACHE_BENCHMARK_ITERATIONS} \
1272+
-Dspark.gluten.benchmark.phases=${TABLE_CACHE_BENCHMARK_PHASES} \
1273+
-Dspark.gluten.benchmark.includeLegacyBaseline=${TABLE_CACHE_BENCHMARK_INCLUDE_LEGACY_BASELINE}"
1274+
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/cpp/build/releases \
1275+
$MVN_CMD test-compile exec:java -Pspark-3.5 -Pbackends-velox -pl backends-velox \
1276+
-Dexec.classpathScope=test \
1277+
-Dexec.mainClass=org.apache.spark.sql.execution.benchmark.ColumnarTableCacheLazyDeserBenchmark \
1278+
| tee -a "${RESULT_FILE}"
1279+
- name: Upload table cache lazy deserialization benchmark results
1280+
if: github.event_name == 'workflow_dispatch' && always()
1281+
uses: actions/upload-artifact@v4
1282+
with:
1283+
name: table-cache-lazy-deserialization-benchmark
1284+
path: benchmarks/ColumnarTableCacheLazyDeserBenchmark-results.txt
1285+
if-no-files-found: ignore
11981286
- name: Run UDF test
11991287
run: |
12001288
yum install -y java-17-openjdk-devel

0 commit comments

Comments
 (0)