Skip to content

Commit d0e47d9

Browse files
bwyogatamabwyogatama
committed
Supporting string offset larger than cudf::size_type
1 parent f15c9b7 commit d0e47d9

3 files changed

Lines changed: 41 additions & 29 deletions

File tree

‎src/gpu_columns.cpp‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,12 @@ GPUColumn::convertToCudfColumn() {
9090
return column;
9191
} else if (data_wrapper.type.id() == GPUColumnTypeId::VARCHAR) {
9292
//convert offset to int32
93-
int32_t* new_offset = convertSiriusOffsetToCudfOffset();
93+
// int32_t* new_offset = convertSiriusOffsetToCudfOffset();
9494

9595
auto offsets_col = cudf::column_view(
96-
cudf::data_type{cudf::type_id::INT32},
96+
cudf::data_type{cudf::type_id::INT64},
9797
size + 1,
98-
reinterpret_cast<void*>(new_offset),
98+
reinterpret_cast<void*>(data_wrapper.offset),
9999
nullptr,
100100
0
101101
);
@@ -132,16 +132,28 @@ GPUColumn::setFromCudfColumn(cudf::column& cudf_column, bool _is_unique, int32_t
132132
is_unique = _is_unique;
133133
//add data to allocation table in gpu buffer manager
134134
if (col_type == cudf::data_type(cudf::type_id::STRING)) {
135-
cudf::column::contents child_cont = cont.children[0]->release();
136-
gpuBufferManager->rmm_stored_buffers.push_back(std::move(child_cont.data));
137-
data_wrapper.is_string_data = true;
138-
data_wrapper.type = GPUColumnType(GPUColumnTypeId::VARCHAR);
139-
int32_t* temp_offset = reinterpret_cast<int32_t*>(gpuBufferManager->rmm_stored_buffers.back()->data());
140-
convertCudfOffsetToSiriusOffset(temp_offset);
141-
//copy data from offset to num_bytes
142-
uint64_t* temp_num_bytes = gpuBufferManager->customCudaHostAlloc<uint64_t>(1);
143-
callCudaMemcpyDeviceToHost<uint64_t>(temp_num_bytes, data_wrapper.offset + column_length, 1, 0);
144-
data_wrapper.num_bytes = temp_num_bytes[0];
135+
if (cont.children[0]->type().id() == cudf::type_id::INT32) {
136+
cudf::column::contents child_cont = cont.children[0]->release();
137+
gpuBufferManager->rmm_stored_buffers.push_back(std::move(child_cont.data));
138+
data_wrapper.is_string_data = true;
139+
data_wrapper.type = GPUColumnType(GPUColumnTypeId::VARCHAR);
140+
int32_t* temp_offset = reinterpret_cast<int32_t*>(gpuBufferManager->rmm_stored_buffers.back()->data());
141+
convertCudfOffsetToSiriusOffset(temp_offset);
142+
//copy data from offset to num_bytes
143+
uint64_t* temp_num_bytes = gpuBufferManager->customCudaHostAlloc<uint64_t>(1);
144+
callCudaMemcpyDeviceToHost<uint64_t>(temp_num_bytes, data_wrapper.offset + column_length, 1, 0);
145+
data_wrapper.num_bytes = temp_num_bytes[0];
146+
} else if (cont.children[0]->type().id() == cudf::type_id::INT64) {
147+
cudf::column::contents child_cont = cont.children[0]->release();
148+
gpuBufferManager->rmm_stored_buffers.push_back(std::move(child_cont.data));
149+
data_wrapper.is_string_data = true;
150+
data_wrapper.type = GPUColumnType(GPUColumnTypeId::VARCHAR);
151+
data_wrapper.offset = reinterpret_cast<uint64_t*>(gpuBufferManager->rmm_stored_buffers.back()->data());
152+
//copy data from offset to num_bytes
153+
uint64_t* temp_num_bytes = gpuBufferManager->customCudaHostAlloc<uint64_t>(1);
154+
callCudaMemcpyDeviceToHost<uint64_t>(temp_num_bytes, data_wrapper.offset + column_length, 1, 0);
155+
data_wrapper.num_bytes = temp_num_bytes[0];
156+
}
145157
} else if (col_type == cudf::data_type(cudf::type_id::UINT64)) {
146158
data_wrapper.is_string_data = false;
147159
data_wrapper.type = GPUColumnType(GPUColumnTypeId::INT64);

‎src/operator/gpu_physical_grouped_aggregate.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -646,14 +646,14 @@ GPUPhysicalGroupedAggregate::Sink(GPUIntermediateRelation& input_relation) const
646646
HandleDuplicateElimination(group_by_column, gpuBufferManager, num_group_keys);
647647
} else {
648648
bool string_cudf_supported = true;
649-
for (int col = 0; col < num_group_keys; col++) {
650-
// if types is VARCHAR, check the number of bytes
651-
if (group_by_column[col]->data_wrapper.type.id() == GPUColumnTypeId::VARCHAR) {
652-
if (group_by_column[col]->data_wrapper.num_bytes > INT32_MAX) {
653-
string_cudf_supported = false;
654-
}
655-
}
656-
}
649+
// for (int col = 0; col < num_group_keys; col++) {
650+
// // if types is VARCHAR, check the number of bytes
651+
// if (group_by_column[col]->data_wrapper.type.id() == GPUColumnTypeId::VARCHAR) {
652+
// if (group_by_column[col]->data_wrapper.num_bytes > INT32_MAX) {
653+
// string_cudf_supported = false;
654+
// }
655+
// }
656+
// }
657657
if (group_by_column[0]->column_length > INT32_MAX || aggregate_column[0]->column_length > INT32_MAX || !string_cudf_supported) {
658658
HandleGroupByAggregateExpression(group_by_column, aggregate_column, gpuBufferManager, aggregates, num_group_keys);
659659
} else {

‎src/operator/gpu_physical_order.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,14 @@ GPUPhysicalOrder::Sink(GPUIntermediateRelation &input_relation) const {
163163
throw NotImplementedException("Order by with column length greater than INT32_MAX is not supported");
164164
}
165165

166-
for (int col = 0; col < projections.size(); col++) {
167-
// if types is VARCHAR, check the number of bytes
168-
if (projection_columns[col]->data_wrapper.type.id() == GPUColumnTypeId::VARCHAR) {
169-
if (projection_columns[col]->data_wrapper.num_bytes > INT32_MAX) {
170-
throw NotImplementedException("String column size greater than INT32_MAX is not supported");
171-
}
172-
}
173-
}
166+
// for (int col = 0; col < projections.size(); col++) {
167+
// // if types is VARCHAR, check the number of bytes
168+
// if (projection_columns[col]->data_wrapper.type.id() == GPUColumnTypeId::VARCHAR) {
169+
// if (projection_columns[col]->data_wrapper.num_bytes > INT32_MAX) {
170+
// throw NotImplementedException("String column size greater than INT32_MAX is not supported");
171+
// }
172+
// }
173+
// }
174174
HandleOrderBy(order_by_keys, projection_columns, orders, projections.size());
175175

176176
for (int col = 0; col < projections.size(); col++) {

0 commit comments

Comments
 (0)