Skip to content

Introduce lists_column_initializer to list column view #23688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nirandaperera wants to merge 29 commits into
base: main
Choose a base branch
from
Open

Introduce lists_column_initializer to list column view #23688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
b33186b
Add lists_column_initializer and opt-in harness failing-current helper
nirandaperera Aug 18, 2026
9da3e46
Fix Init nesting in segmented gather list test ports
nirandaperera Aug 15, 2026
e607cb1
reorder ctrs
nirandaperera Aug 17, 2026
8402bb6
precommit
nirandaperera Aug 18, 2026
60c8c7e
fix tests
nirandaperera Aug 18, 2026
cd0d68c
Merge branch 'main' into lists-column-initializer-memory-resources
nirandaperera Aug 20, 2026
e2235c5
Apply suggestions from code review
nirandaperera Aug 24, 2026
81a0096
addressing comments
nirandaperera Aug 24, 2026
a404703
Merge branch 'main' of github.com:rapidsai/cudf into lists-column-ini…
nirandaperera Aug 24, 2026
4e09b94
remove stray +
nirandaperera Aug 25, 2026
4d0ade6
malformed init
nirandaperera Aug 25, 2026
56af3f2
fix tests
nirandaperera Aug 25, 2026
df7481d
Merge branch 'main' into lists-column-initializer-memory-resources
nirandaperera Aug 25, 2026
a6d7108
Init to I32init
nirandaperera Aug 25, 2026
2d049f2
Merge branch 'main' of github.com:rapidsai/cudf into lists-column-ini…
nirandaperera Aug 25, 2026
431b965
Merge branch 'main' into lists-column-initializer-memory-resources
nirandaperera Aug 26, 2026
5f78584
Split lists column wrapper into separate header
bdice Aug 26, 2026
e18754f
Enable recursive lists column initialization
bdice Sep 2, 2026
dceab3b
Merge upstream main into lists-column-initializer-memory-resources
bdice Sep 2, 2026
585e735
Allow resource-aware empty lists column wrappers
bdice Sep 2, 2026
3016b01
Simplify list wrapper resource tests
bdice Sep 9, 2026
09e1f09
Refine recursive lists column initialization
bdice Sep 11, 2026
1efa2bd
Merge remote-tracking branch 'upstream/main' into lists-column-initia…
bdice Sep 16, 2026
f3cc7aa
Preserve list test data with inline initializers
bdice Sep 22, 2026
d77a221
Move lists wrapper back into column wrapper header
bdice Sep 22, 2026
c0de716
Merge remote-tracking branch 'upstream/main' into lists-column-initia…
bdice Sep 22, 2026
99b9de4
Revert copyright changes
bdice Sep 22, 2026
2413ce7
Simplify empty nested list initializers in tests
bdice Sep 23, 2026
9a48c66
Avoid nested calls
bdice Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,208 changes: 779 additions & 429 deletions cpp/include/cudf_test/column_wrapper.hpp
View file
Open in desktop

Large diffs are not rendered by default.

15 changes: 6 additions & 9 deletions cpp/tests/column/column_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ TYPED_TEST(ListsColumnTest, ListsSlicedNonNestedEmpty)
// Column of List<int>
LCW list{{1, 2}, {}, {3, 4}, {8, 9}};
// Column of 1 row, an empty List<int>
LCW expect{LCW{}};
LCW expect{{}};

auto sliced = cudf::slice(list, {1, 2}).front();
auto result = std::make_unique<cudf::column>(sliced);
Expand All @@ -553,9 +553,9 @@ TYPED_TEST(ListsColumnTest, ListsSlicedNestedEmpty)
using FWCW_SZ = cudf::test::fixed_width_column_wrapper<cudf::size_type>;

// Column of List<List<int>>, with incomplete hierarchy
LCW list{{LCW{1}, LCW{2}},
LCW list{{{1}, {2}},
{}, // < ----------- empty List<List<int>>, slice this
{LCW{3}, LCW{4, 5}}};
{{3}, {4, 5}}};

// Make 1-row column of type List<List<int>>, the row data contains 0 element.
// Well-formed memory layout:
Expand Down Expand Up @@ -585,7 +585,7 @@ TYPED_TEST(ListsColumnTest, ListsSlicedZeroSliceLengthNested)
using LCW = cudf::test::lists_column_wrapper<TypeParam>;

// Column of List<List<int>>, with incomplete hierarchy
LCW list{{LCW{1}, LCW{2}}, {}, {LCW{3}, LCW{4, 5}}};
LCW list{{{1}, {2}}, {}, {{3}, {4, 5}}};

auto expect = cudf::empty_like(list);

Expand Down Expand Up @@ -617,12 +617,9 @@ TYPED_TEST(ListsColumnTest, ListsSlicedColumnViewConstructorWithNulls)

using LCW = cudf::test::lists_column_wrapper<TypeParam>;

cudf::test::lists_column_wrapper<TypeParam> list{
{{{{1, 2}, {3, 4}}, valids}, LCW{}, {{{5, 6, 7}, LCW{}, {8, 9}}, valids}, LCW{}, LCW{}},
valids};
LCW list{{{{{1, 2}, {3, 4}}, valids}, {}, {{{5, 6, 7}, {}, {8, 9}}, valids}, {}, {}}, valids};

cudf::test::lists_column_wrapper<TypeParam> expect{
{LCW{}, {{{5, 6, 7}, LCW{}, {8, 9}}, valids}, LCW{}, LCW{}}, expect_valids};
LCW expect{{{}, {{{5, 6, 7}, {}, {8, 9}}, valids}, {}, {}}, expect_valids};

auto sliced = cudf::slice(list, {1, 5}).front();
auto result = std::make_unique<cudf::column>(sliced);
Expand Down
2 changes: 1 addition & 1 deletion cpp/tests/column/column_view_shallow_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down
57 changes: 26 additions & 31 deletions cpp/tests/column/factories_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,9 +404,9 @@ TYPED_TEST(ListsFixedWidthLeafTest, FromNonNested)
auto s = cudf::make_list_scalar(FCW({1, -1, 3}, {1, 0, 1}));
auto col = cudf::make_column_from_scalar(*s, 3);

auto expected = LCW{LCW({1, 2, 3}, valid_t{1, 0, 1}.begin()),
LCW({1, 2, 3}, valid_t{1, 0, 1}.begin()),
LCW({1, 2, 3}, valid_t{1, 0, 1}.begin())};
auto expected = LCW{{{1, 2, 3}, valid_t{1, 0, 1}.begin()},
{{1, 2, 3}, valid_t{1, 0, 1}.begin()},
{{1, 2, 3}, valid_t{1, 0, 1}.begin()}};
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*col, expected);
}

Expand All @@ -416,12 +416,12 @@ TYPED_TEST(ListsFixedWidthLeafTest, FromNested)
using valid_t = std::vector<cudf::valid_type>;

#define row_data \
LCW({LCW({-1, -1, 3}, valid_t{0, 0, 1}.begin()), LCW{}, LCW{}}, valid_t{1, 0, 1}.begin())
LCW::nested({{{-1, -1, 3}, valid_t{0, 0, 1}.begin()}, {}, {}}, valid_t{1, 0, 1}.begin())

auto s = cudf::make_list_scalar(row_data);
auto s = cudf::make_list_scalar(LCW(row_data));
auto col = cudf::make_column_from_scalar(*s, 5);

auto expected = LCW{row_data, row_data, row_data, row_data, row_data};
auto expected = LCW({row_data, row_data, row_data, row_data, row_data});
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*col, expected);

#undef row_data
Expand Down Expand Up @@ -496,10 +496,10 @@ TEST_F(ListsStringLeafTest, FromNonNested)
auto s = cudf::make_list_scalar(SCW({"xx", "", "z"}, {true, false, true}));
auto col = cudf::make_column_from_scalar(*s, 4);

auto expected = LCW{LCW({"xx", "", "z"}, valid_t{1, 0, 1}.begin()),
LCW({"xx", "", "z"}, valid_t{1, 0, 1}.begin()),
LCW({"xx", "", "z"}, valid_t{1, 0, 1}.begin()),
LCW({"xx", "", "z"}, valid_t{1, 0, 1}.begin())};
auto expected = LCW{{{"xx", "", "z"}, valid_t{1, 0, 1}.begin()},
{{"xx", "", "z"}, valid_t{1, 0, 1}.begin()},
{{"xx", "", "z"}, valid_t{1, 0, 1}.begin()},
{{"xx", "", "z"}, valid_t{1, 0, 1}.begin()}};
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*col, expected);
}

Expand All @@ -508,18 +508,18 @@ TEST_F(ListsStringLeafTest, FromNested)
using LCW = cudf::test::lists_column_wrapper<cudf::string_view>;
using valid_t = std::vector<cudf::valid_type>;

#define row_data \
LCW({LCW{}, \
LCW({"@@", "rapids", "", "四", "ら"}, valid_t{1, 1, 0, 1, 1}.begin()), \
LCW{}, \
LCW({"hello", ""}, valid_t{1, 0}.begin())}, \
valid_t{0, 1, 1, 1}.begin())
#define row_data \
LCW::nested({{}, \
{{"@@", "rapids", "", "四", "ら"}, valid_t{1, 1, 0, 1, 1}.begin()}, \
{}, \
{{"hello", ""}, valid_t{1, 0}.begin()}}, \
valid_t{0, 1, 1, 1}.begin())

auto s = cudf::make_list_scalar(row_data);
auto s = cudf::make_list_scalar(LCW(row_data));

auto col = cudf::make_column_from_scalar(*s, 3);

auto expected = LCW{row_data, row_data, row_data};
auto expected = LCW({row_data, row_data, row_data});
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*col, expected);
#undef row_data
}
Expand Down Expand Up @@ -580,12 +580,11 @@ TYPED_TEST(ListsStructsLeafTest, FromNested)
using StringCW = cudf::test::strings_column_wrapper;
using offset_t = cudf::test::fixed_width_column_wrapper<cudf::size_type>;
using valid_t = std::vector<cudf::valid_type>;
auto leaf = this->make_test_structs_column(
{{1, 2}, {0, 1}},
StringCW({"étoile", "星"}, {true, true}),
LCWinner_t({LCWinner_t{}, LCWinner_t{42}}, valid_t{1, 1}.begin()),
valid_t{0, 1}.begin());
auto mask = cudf::create_null_mask(3, cudf::mask_state::ALL_VALID);
auto leaf = this->make_test_structs_column({{1, 2}, {0, 1}},
StringCW({"étoile", "星"}, {true, true}),
LCWinner_t({{}, {42}}, valid_t{1, 1}.begin()),
valid_t{0, 1}.begin());
auto mask = cudf::create_null_mask(3, cudf::mask_state::ALL_VALID);
cudf::set_null_mask(static_cast<cudf::bitmask_type*>(mask.data()), 0, 1, false);
auto data =
cudf::make_lists_column(3, offset_t{0, 0, 1, 2}.release(), leaf.release(), 1, std::move(mask));
Expand All @@ -597,9 +596,7 @@ TYPED_TEST(ListsStructsLeafTest, FromNested)
{{1, 2, 1, 2, 1, 2}, {0, 1, 0, 1, 0, 1}},
StringCW({"étoile", "星", "étoile", "星", "étoile", "星"},
{true, true, true, true, true, true}),
LCWinner_t(
{LCWinner_t{}, LCWinner_t{42}, LCWinner_t{}, LCWinner_t{42}, LCWinner_t{}, LCWinner_t{42}},
valid_t{1, 1, 1, 1, 1, 1}.begin()),
LCWinner_t({{}, {42}, {}, {42}, {}, {42}}, valid_t{1, 1, 1, 1, 1, 1}.begin()),
valid_t{0, 1, 0, 1, 0, 1}.begin());
auto mask2 = cudf::create_null_mask(9, cudf::mask_state::ALL_VALID);
cudf::set_null_mask(static_cast<cudf::bitmask_type*>(mask2.data()), 0, 1, false);
Expand Down Expand Up @@ -649,7 +646,7 @@ TEST_F(ListsZeroLengthColumnTest, MixedTypes)
}

{
auto s = cudf::make_list_scalar(LCW{LCW{1, 2, 3}, LCW{}, LCW{5, 6}});
auto s = cudf::make_list_scalar(LCW{{1, 2, 3}, {}, {5, 6}});
auto got = cudf::make_column_from_scalar(*s, 0);
auto nested = cudf::make_lists_column(0,
offset_t{}.release(),
Expand Down Expand Up @@ -725,12 +722,10 @@ TEST_F(ListsZeroLengthColumnTest, SuperimposeNulls)

void struct_from_scalar(bool is_valid)
{
using LCW = cudf::test::lists_column_wrapper<int>;

cudf::test::fixed_width_column_wrapper<int> col0{1};
cudf::test::strings_column_wrapper col1{"abc"};
cudf::test::lists_column_wrapper<int> col2{{1, 2, 3}};
cudf::test::lists_column_wrapper<int> col3{LCW{}};
cudf::test::lists_column_wrapper<int> col3{{}};

std::vector<cudf::column_view> src_children({col0, col1, col2, col3});
auto value = cudf::struct_scalar(src_children, is_valid);
Expand Down
Loading
Loading