Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[submodule "thirdparty/cudf"]
path = thirdparty/cudf
url = https://github.com/rapidsai/cudf.git
branch = release/26.08
branch = main
2 changes: 1 addition & 1 deletion build/env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@
#

set -ex
export sclCMD=${sclCMD:-"scl enable gcc-toolset-13"}
export sclCMD=${sclCMD:-"scl enable gcc-toolset-14"}
2 changes: 1 addition & 1 deletion ci/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ ARG CMAKE_ARCH=x86_64

### Install basic requirements
# pin urllib3<2.0 for https://github.com/psf/requests/issues/6432
RUN dnf --enablerepo=powertools install -y scl-utils gcc-toolset-13 gcc-toolset-${TOOLSET_VERSION} python39 \
RUN dnf --enablerepo=powertools install -y scl-utils gcc-toolset-14 gcc-toolset-${TOOLSET_VERSION} python39 \
zlib-devel maven tar wget patch ninja-build git zip && \
alternatives --set python /usr/bin/python3 && \
python -m pip install requests 'urllib3<2.0'
Expand Down
9 changes: 4 additions & 5 deletions src/main/cpp/src/shuffle_split.cu
Original file line number Diff line number Diff line change
Expand Up @@ -794,11 +794,10 @@ shuffle_split_metadata compute_metadata(cudf::table_view const& input,
/**
* @copydoc spark_rapids_jni::shuffle_split
*/
std::pair<shuffle_split_result, shuffle_split_metadata> shuffle_split(
cudf::table_view const& input,
std::vector<size_type> const& splits,
rmm::cuda_stream_view stream,
rmm::device_async_resource_ref mr)
shuffle_split_output shuffle_split(cudf::table_view const& input,
std::vector<size_type> const& splits,
rmm::cuda_stream_view stream,
rmm::device_async_resource_ref mr)
{
SRJ_FUNC_RANGE();

Expand Down
16 changes: 11 additions & 5 deletions src/main/cpp/src/shuffle_split.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,13 @@ struct shuffle_split_result {
rmm::device_uvector<size_t> offsets{0, cudf::get_default_stream()};
};

struct shuffle_split_output {
// Keep this as an aggregate instead of std::pair. CUDA 12.9 cudafe++ crashes when GCC 14's
// concepts-based std::pair constructor constraints are instantiated for these result types.
shuffle_split_result result;
shuffle_split_metadata metadata;
};

/**
* @brief Performs a split operation on a cudf table, returning a buffer of data containing
* all of the sub-tables as a contiguous buffer of anonymous bytes.
Expand Down Expand Up @@ -133,11 +140,10 @@ struct shuffle_split_result {
* partition, and a shuffle_split_metadata struct which contains the metadata needed to reconstruct
* a table using shuffle_assemble.
*/
std::pair<shuffle_split_result, shuffle_split_metadata> shuffle_split(
cudf::table_view const& input,
std::vector<cudf::size_type> const& splits,
rmm::cuda_stream_view stream,
rmm::device_async_resource_ref mr);
shuffle_split_output shuffle_split(cudf::table_view const& input,
std::vector<cudf::size_type> const& splits,
rmm::cuda_stream_view stream,
rmm::device_async_resource_ref mr);

/**
* @brief Buffer slice representing a portion of the shared allocated buffer
Expand Down
14 changes: 6 additions & 8 deletions src/main/cpp/tests/shuffle_split.cu
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025, NVIDIA CORPORATION.
* Copyright (c) 2025-2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -867,26 +867,24 @@ TEST_F(ShuffleSplitTests, MixedValidity)
cudf::table_view expected_t{{static_cast<cudf::column_view>(*expected)}};

// make the concatenated shuffle_split partitions
std::vector<
std::pair<spark_rapids_jni::shuffle_split_result, spark_rapids_jni::shuffle_split_metadata>>
shuf;
std::vector<spark_rapids_jni::shuffle_split_output> shuf;
size_t total_size = 0;
for (size_t idx = 0; idx < partition_views.size(); idx++) {
shuf.push_back(spark_rapids_jni::shuffle_split(
cudf::table_view{{partition_views[idx]}}, {}, stream, mr));
total_size += shuf.back().first.partitions->size();
total_size += shuf.back().result.partitions->size();
}
rmm::device_uvector<uint8_t> full{total_size, stream, mr};
rmm::device_uvector<size_t> full_offsets{partition_views.size() + 1, stream, mr};
std::vector<size_t> h_full_offsets(partition_views.size() + 1);
size_t pos = 0;
for (size_t idx = 0; idx < partition_views.size(); idx++) {
cudaMemcpy(static_cast<uint8_t*>(full.data()) + pos,
shuf[idx].first.partitions->data(),
shuf[idx].first.partitions->size(),
shuf[idx].result.partitions->data(),
shuf[idx].result.partitions->size(),
cudaMemcpyDeviceToDevice);
h_full_offsets[idx] = pos;
pos += shuf[idx].first.partitions->size();
pos += shuf[idx].result.partitions->size();
}
h_full_offsets[partition_views.size()] = pos;
cudaMemcpy(full_offsets.data(),
Expand Down
2 changes: 1 addition & 1 deletion thirdparty/cudf
Submodule cudf updated 395 files
2 changes: 1 addition & 1 deletion thirdparty/cudf-pins/rapids-cmake.sha
Original file line number Diff line number Diff line change
@@ -1 +1 @@
9c0829ec73702b3df8a5c2ec43f6aaabe5f1e5ec
7105d249b7ce31c7361238f096ccb53a6af72653
8 changes: 4 additions & 4 deletions thirdparty/cudf-pins/versions.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@
{
"always_download" : true,
"git_shallow" : false,
"git_tag" : "93606c074f3d863a7052af25afae569f72cb3304",
"git_tag" : "2e98113da4669b723f69bb21f87e8506f6210515",
"git_url" : "https://github.com/rapidsai/kvikio.git",
"source_subdir" : "cpp",
"version" : "26.08"
"version" : "26.10"
},
"nanoarrow" :
{
Expand Down Expand Up @@ -89,10 +89,10 @@
{
"always_download" : true,
"git_shallow" : false,
"git_tag" : "1a39f9e81c467b1a9522a4dcec8b5581ae165fef",
"git_tag" : "eeea5f73cca5abf20d88169156db6cbd40e1f6cc",
"git_url" : "https://github.com/rapidsai/rmm.git",
"source_subdir" : "cpp",
"version" : "26.08"
"version" : "26.10"
},
"rtcx" :
{
Expand Down