Skip to content

Commit 626c8ed

Browse files
committed
Use configured copy buffer for Hadoop vectored reads
Signed-off-by: Hongbin Ma (Mahone) <mahongbin@apache.org>
1 parent 7557f16 commit 626c8ed

1 file changed

Lines changed: 20 additions & 3 deletions

File tree

sql-plugin/src/main/java/com/nvidia/spark/rapids/fileio/hadoop/HadoopInputFile.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2025, NVIDIA CORPORATION.
2+
* Copyright (c) 2025-2026, NVIDIA CORPORATION.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
1616

1717
package com.nvidia.spark.rapids.fileio.hadoop;
1818

19+
import ai.rapids.cudf.HostMemoryBuffer;
1920
import com.nvidia.spark.rapids.jni.fileio.RapidsInputFile;
2021
import com.nvidia.spark.rapids.jni.fileio.SeekableInputStream;
2122
import org.apache.hadoop.conf.Configuration;
@@ -24,6 +25,7 @@
2425
import org.apache.hadoop.fs.Path;
2526

2627
import java.io.IOException;
28+
import java.util.List;
2729
import java.util.Objects;
2830
import java.util.OptionalLong;
2931

@@ -34,21 +36,30 @@
3436
* for reading the file.
3537
*/
3638
public class HadoopInputFile implements RapidsInputFile {
39+
private static final String PARQUET_READ_ALLOCATION_SIZE = "parquet.read.allocation.size";
40+
3741
private final Path filePath;
3842
private final FileSystem fs;
43+
private final int copyBufferSize;
3944

4045
public static HadoopInputFile create(Path filePath, Configuration conf) throws IOException {
4146
Objects.requireNonNull(filePath, "filePath can't be null!");
4247
Objects.requireNonNull(conf, "Hadoop conf can't be null");
4348
FileSystem fs = filePath.getFileSystem(conf);
44-
return new HadoopInputFile(filePath, fs);
49+
int copyBufferSize = conf.getInt(PARQUET_READ_ALLOCATION_SIZE,
50+
RapidsInputFile.DEFAULT_READ_VECTORED_COPY_BUFFER_SIZE);
51+
return new HadoopInputFile(filePath, fs, copyBufferSize);
4552
}
4653

47-
private HadoopInputFile(Path filePath, FileSystem fs) {
54+
private HadoopInputFile(Path filePath, FileSystem fs, int copyBufferSize) {
4855
Objects.requireNonNull(filePath, "filePath can't be null!");
4956
Objects.requireNonNull(fs, "FileSystem can't be null");
57+
if (copyBufferSize <= 0) {
58+
throw new IllegalArgumentException(PARQUET_READ_ALLOCATION_SIZE + " must be positive");
59+
}
5060
this.filePath = filePath;
5161
this.fs = fs;
62+
this.copyBufferSize = copyBufferSize;
5263
}
5364

5465
@Override
@@ -70,4 +81,10 @@ public OptionalLong getLastModificationTime() throws IOException {
7081
public SeekableInputStream open() throws IOException {
7182
return new HadoopInputStream(fs.open(filePath));
7283
}
84+
85+
@Override
86+
public void readVectored(HostMemoryBuffer output, List<RapidsInputFile.CopyRange> copyRanges)
87+
throws IOException {
88+
RapidsInputFile.readVectoredUsingCopyBuffer(this, output, copyRanges, copyBufferSize);
89+
}
7390
}

0 commit comments

Comments
 (0)