Skip to content

Commit fc162be

Browse files
wjxiz1992wjxiz1992
committed
Clarify schema mismatch nullification flag
Signed-off-by: Allen Xu <allxu@nvidia.com>
1 parent 21ebd2b commit fc162be

1 file changed

Lines changed: 33 additions & 20 deletions

File tree

‎src/main/cpp/src/from_json_to_structs.cu‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ void nullify_rows(cudf::column& input,
226226
std::unique_ptr<cudf::column> child_column,
227227
cudf::size_type null_count,
228228
rmm::device_buffer&& null_mask,
229-
bool has_rows_nullified,
229+
bool did_nullify_schema_mismatch_rows,
230230
rmm::cuda_stream_view stream,
231231
rmm::device_async_resource_ref mr)
232232
{
@@ -240,7 +240,8 @@ void nullify_rows(cudf::column& input,
240240
null_count,
241241
std::move(children));
242242
// Row-level schema mismatch nulls can leave child data under null parents; sanitize it here.
243-
if (has_rows_nullified && null_count > 0 && cudf::has_nonempty_nulls(output->view(), stream)) {
243+
if (did_nullify_schema_mismatch_rows && null_count > 0 &&
244+
cudf::has_nonempty_nulls(output->view(), stream)) {
244245
output = cudf::purge_nonempty_nulls(output->view(), stream, mr);
245246
}
246247
return output;
@@ -251,11 +252,11 @@ void nullify_rows(cudf::column& input,
251252
std::vector<std::unique_ptr<cudf::column>>&& children,
252253
cudf::size_type null_count,
253254
rmm::device_buffer&& null_mask,
254-
bool has_rows_nullified,
255+
bool did_nullify_schema_mismatch_rows,
255256
rmm::cuda_stream_view stream,
256257
rmm::device_async_resource_ref mr)
257258
{
258-
if (has_rows_nullified && null_count > 0) {
259+
if (did_nullify_schema_mismatch_rows && null_count > 0) {
259260
// make_structs_column superimposes parent nulls onto children for a consistent nested column.
260261
return cudf::make_structs_column(
261262
num_rows, std::move(children), null_count, std::move(null_mask), stream, mr);
@@ -731,7 +732,7 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
731732
schema_element_with_precision const& schema,
732733
bool allow_nonnumeric_numbers,
733734
bool is_us_locale,
734-
bool has_rows_nullified,
735+
bool did_nullify_schema_mismatch_rows,
735736
rmm::cuda_stream_view stream,
736737
rmm::device_async_resource_ref mr)
737738
{
@@ -836,16 +837,21 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
836837
std::vector<std::unique_ptr<cudf::column>> new_children;
837838
new_children.emplace_back(
838839
std::move(input_content.children[cudf::lists_column_view::offsets_column_index]));
839-
new_children.emplace_back(convert_data_type(
840-
std::move(child), child_schema, allow_nonnumeric_numbers, is_us_locale, false, stream, mr));
840+
new_children.emplace_back(convert_data_type(std::move(child),
841+
child_schema,
842+
allow_nonnumeric_numbers,
843+
is_us_locale,
844+
/*did_nullify_schema_mismatch_rows=*/false,
845+
stream,
846+
mr));
841847

842848
return make_lists_column_with_null_sanitization(
843849
num_rows,
844850
std::move(new_children[cudf::lists_column_view::offsets_column_index]),
845851
std::move(new_children[cudf::lists_column_view::child_column_index]),
846852
null_count,
847853
std::move(*input_content.null_mask),
848-
has_rows_nullified,
854+
did_nullify_schema_mismatch_rows,
849855
stream,
850856
mr);
851857
}
@@ -858,7 +864,7 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
858864
schema.child_types[i].second,
859865
allow_nonnumeric_numbers,
860866
is_us_locale,
861-
false,
867+
/*did_nullify_schema_mismatch_rows=*/false,
862868
stream,
863869
mr));
864870
}
@@ -867,7 +873,7 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
867873
std::move(new_children),
868874
null_count,
869875
std::move(*input_content.null_mask),
870-
has_rows_nullified,
876+
did_nullify_schema_mismatch_rows,
871877
stream,
872878
mr);
873879
}
@@ -887,16 +893,21 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
887893
std::vector<std::unique_ptr<cudf::column>> new_children;
888894
new_children.emplace_back(
889895
std::make_unique<cudf::column>(input.child(cudf::lists_column_view::offsets_column_index)));
890-
new_children.emplace_back(convert_data_type(
891-
child, child_schema, allow_nonnumeric_numbers, is_us_locale, false, stream, mr));
896+
new_children.emplace_back(convert_data_type(child,
897+
child_schema,
898+
allow_nonnumeric_numbers,
899+
is_us_locale,
900+
/*did_nullify_schema_mismatch_rows=*/false,
901+
stream,
902+
mr));
892903

893904
return make_lists_column_with_null_sanitization(
894905
num_rows,
895906
std::move(new_children[cudf::lists_column_view::offsets_column_index]),
896907
std::move(new_children[cudf::lists_column_view::child_column_index]),
897908
null_count,
898909
cudf::copy_bitmask(input, stream, mr),
899-
has_rows_nullified,
910+
did_nullify_schema_mismatch_rows,
900911
stream,
901912
mr);
902913
}
@@ -909,7 +920,7 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
909920
schema.child_types[i].second,
910921
allow_nonnumeric_numbers,
911922
is_us_locale,
912-
false,
923+
/*did_nullify_schema_mismatch_rows=*/false,
913924
stream,
914925
mr));
915926
}
@@ -918,7 +929,7 @@ std::unique_ptr<cudf::column> convert_data_type(InputType&& input,
918929
std::move(new_children),
919930
null_count,
920931
cudf::copy_bitmask(input, stream, mr),
921-
has_rows_nullified,
932+
did_nullify_schema_mismatch_rows,
922933
stream,
923934
mr);
924935
}
@@ -994,14 +1005,16 @@ std::unique_ptr<cudf::column> from_json_to_structs(cudf::strings_column_view con
9941005
auto const& [col_name, col_schema] = schema_with_precision.child_types[i];
9951006
CUDF_EXPECTS(parsed_meta.schema_info[i].name == col_name, "Mismatched column name.");
9961007
auto const mismatch_rows = mismatch_rows_by_column.find(col_name);
997-
auto const has_rows_nullified =
1008+
auto const did_nullify_schema_mismatch_rows =
9981009
mismatch_rows != mismatch_rows_by_column.end() && !mismatch_rows->second.empty();
999-
if (has_rows_nullified) { nullify_rows(*parsed_columns[i], mismatch_rows->second, stream, mr); }
1010+
if (did_nullify_schema_mismatch_rows) {
1011+
nullify_rows(*parsed_columns[i], mismatch_rows->second, stream, mr);
1012+
}
10001013
converted_cols.emplace_back(convert_data_type(std::move(parsed_columns[i]),
10011014
col_schema,
10021015
allow_nonnumeric_numbers,
10031016
is_us_locale,
1004-
has_rows_nullified,
1017+
did_nullify_schema_mismatch_rows,
10051018
stream,
10061019
mr));
10071020
}
@@ -1015,7 +1028,7 @@ std::unique_ptr<cudf::column> from_json_to_structs(cudf::strings_column_view con
10151028
std::move(converted_cols),
10161029
null_count,
10171030
null_count > 0 ? std::move(null_mask) : rmm::device_buffer{0, stream, mr},
1018-
false,
1031+
/*did_nullify_schema_mismatch_rows=*/false,
10191032
stream,
10201033
mr);
10211034
}
@@ -1081,7 +1094,7 @@ std::unique_ptr<cudf::column> convert_from_strings(cudf::strings_column_view con
10811094
schema_with_precision.child_types.front().second,
10821095
allow_nonnumeric_numbers,
10831096
is_us_locale,
1084-
false,
1097+
/*did_nullify_schema_mismatch_rows=*/false,
10851098
stream,
10861099
mr);
10871100
}

0 commit comments

Comments
 (0)