Skip to content

Fix doris 26821 map lazy fill #65403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HappenLee wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix doris 26821 map lazy fill #65403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
75fe1db
[fix](be) Fill skipped map side during lazy pruned reads
HappenLee Jul 7, 2026
bec1331
[improvement](be) Add native ZoneMap serialization
HappenLee Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
717 changes: 691 additions & 26 deletions be/src/storage/index/zone_map/zone_map_index.cpp
View file
Open in desktop

Large diffs are not rendered by default.

43 changes: 37 additions & 6 deletions be/src/storage/index/zone_map/zone_map_index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ struct ZoneMap {

static Status from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_type,
ZoneMap& zone_map_info);

void to_entry_pb_native(ZoneMapEntryPB* dst, const DataTypePtr& data_type) const;

static Status from_entry_pb_native(const ZoneMapEntryPB& zone_map, const DataTypePtr& data_type,
ZoneMap& zone_map_info);
};

class ZoneMapIndexWriter {
Expand Down Expand Up @@ -166,8 +171,8 @@ class TypedZoneMapIndexWriter final : public ZoneMapIndexWriter {
std::string _page_min_storage;
std::string _page_max_storage;

// serialized ZoneMapPB for each data page
std::vector<std::string> _values;
// ZoneMap for each data page.
std::vector<ZoneMap> _page_zone_maps;
uint64_t _estimated_size = 0;
};

Expand All @@ -179,29 +184,55 @@ class ZoneMapIndexReader : public MetadataAdder<ZoneMapIndexReader> {
_page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps));
}

explicit ZoneMapIndexReader(io::FileReaderSPtr file_reader,
const ZoneMapIndexPB& zone_map_index, DataTypePtr data_type)
: _file_reader(std::move(file_reader)), _data_type(std::move(data_type)) {
if (zone_map_index.has_page_zone_maps_v2()) {
_page_zone_maps_v2_meta.reset(
new ZoneMapPageIndexPB(zone_map_index.page_zone_maps_v2()));
} else if (zone_map_index.has_page_zone_maps()) {
_page_zone_maps_meta.reset(new IndexedColumnMetaPB(zone_map_index.page_zone_maps()));
}
}

virtual ~ZoneMapIndexReader();

// load all page zone maps into memory
Status load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats = nullptr,
const io::IOContext* io_ctx = nullptr);

const std::vector<ZoneMapPB>& page_zone_maps() const { return _page_zone_maps; }
const std::vector<ZoneMapPB>& page_zone_maps() const;

const std::vector<ZoneMap>& page_zone_map_infos() const { return _page_zone_map_infos; }

size_t num_pages() const { return _page_zone_maps.size(); }
size_t num_pages() const { return _num_pages; }

private:
Status _load(bool use_page_cache, bool kept_in_memory, std::unique_ptr<IndexedColumnMetaPB>,
Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB> page_zone_maps_meta,
std::unique_ptr<ZoneMapPageIndexPB> page_zone_maps_v2_meta,
OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx);

Status _load_legacy(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB> page_zone_maps_meta,
OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx);

Status _load_v2(std::unique_ptr<ZoneMapPageIndexPB> page_zone_maps_v2_meta,
OlapReaderStatistics* index_load_stats, const io::IOContext* io_ctx);

int64_t get_metadata_size() const override;

private:
DorisCallOnce<Status> _load_once;
// TODO: yyq, we shoud remove file_reader from here.
io::FileReaderSPtr _file_reader;
DataTypePtr _data_type;
std::unique_ptr<IndexedColumnMetaPB> _page_zone_maps_meta;
std::vector<ZoneMapPB> _page_zone_maps;
std::unique_ptr<ZoneMapPageIndexPB> _page_zone_maps_v2_meta;
mutable std::vector<ZoneMapPB> _page_zone_maps;
std::vector<ZoneMap> _page_zone_map_infos;
size_t _num_pages = 0;
int64_t _pb_meta_size {0};
};

Expand Down
138 changes: 92 additions & 46 deletions be/src/storage/segment/column_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,43 @@ void remove_current_level_meta_access_paths(TColumnAccessPaths& paths) {
paths.erase(removed.begin(), removed.end());
}

Status read_or_fill_map_item_column(ColumnIterator* iterator, bool seek_to_start,
ordinal_t start_ordinal, size_t item_count,
MutableColumnPtr& dst) {
if (item_count == 0) {
return Status::OK();
}

if (iterator->need_to_read()) {
if (seek_to_start) {
RETURN_IF_ERROR(iterator->seek_to_ordinal(start_ordinal));
}
size_t num_read = item_count;
bool has_null = false;
RETURN_IF_ERROR(iterator->next_batch(&num_read, dst, &has_null));
DCHECK_EQ(num_read, item_count);
return Status::OK();
}

if (iterator->read_requirement() == ColumnIterator::ReadRequirement::SKIP) {
dst->insert_many_defaults(item_count);
return Status::OK();
}

if (iterator->read_requirement() == ColumnIterator::ReadRequirement::PREDICATE) {
return Status::OK();
}

if (seek_to_start) {
RETURN_IF_ERROR(iterator->seek_to_ordinal(start_ordinal));
}
size_t num_read = item_count;
bool has_null = false;
RETURN_IF_ERROR(iterator->next_batch(&num_read, dst, &has_null));
DCHECK_EQ(num_read, item_count);
return Status::OK();
}

Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
const io::FileReaderSPtr& file_reader,
std::shared_ptr<ColumnReader>* reader) {
Expand Down Expand Up @@ -319,15 +356,16 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
ColumnReader::~ColumnReader() = default;

int64_t ColumnReader::get_metadata_size() const {
return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0);
return sizeof(ColumnReader) + (_segment_zone_map ? _segment_zone_map->ByteSizeLong() : 0) +
(_segment_zone_map_v2 ? _segment_zone_map_v2->ByteSizeLong() : 0);
}

#ifdef BE_TEST
/// This function is only used in UT to verify the correctness of data read from zone map
/// See UT case 'SegCompactionMoWTest.SegCompactionInterleaveWithBig_ooooOOoOooooooooO'
/// be/test/olap/segcompaction_mow_test.cpp
void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) const {
if (!_segment_zone_map) {
if (!_segment_zone_map && !_segment_zone_map_v2) {
return;
}

Expand All @@ -353,7 +391,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst)
}

ZoneMap zone_map;
THROW_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));
THROW_IF_ERROR(_get_segment_zone_map(&zone_map));

if (zone_map.has_null) {
return;
Expand Down Expand Up @@ -392,10 +430,15 @@ Status ColumnReader::init(const ColumnMetaPB* meta) {
new OrdinalIndexReader(_file_reader, _num_rows, index_meta.ordinal_index()));
break;
case ZONE_MAP_INDEX:
_segment_zone_map =
std::make_unique<ZoneMapPB>(index_meta.zone_map_index().segment_zone_map());
_zone_map_index.reset(new ZoneMapIndexReader(
_file_reader, index_meta.zone_map_index().page_zone_maps()));
if (index_meta.zone_map_index().has_segment_zone_map_v2()) {
_segment_zone_map_v2 = std::make_unique<ZoneMapEntryPB>(
index_meta.zone_map_index().segment_zone_map_v2());
} else {
_segment_zone_map =
std::make_unique<ZoneMapPB>(index_meta.zone_map_index().segment_zone_map());
}
_zone_map_index.reset(
new ZoneMapIndexReader(_file_reader, index_meta.zone_map_index(), _data_type));
break;
case BLOOM_FILTER_INDEX:
_bloom_filter_index.reset(
Expand Down Expand Up @@ -469,12 +512,12 @@ Status ColumnReader::get_row_ranges_by_zone_map(
}

Status ColumnReader::next_batch_of_zone_map(size_t* n, MutableColumnPtr& dst) const {
if (_segment_zone_map == nullptr) {
if (_segment_zone_map == nullptr && _segment_zone_map_v2 == nullptr) {
return Status::InternalError("segment zonemap not exist");
}
// TODO: this work to get min/max value seems should only do once
ZoneMap zone_map;
RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));
RETURN_IF_ERROR(_get_segment_zone_map(&zone_map));

dst->reserve(*n);
if (!zone_map.has_not_null) {
Expand All @@ -495,7 +538,7 @@ Status ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicat
return Status::OK();
}
ZoneMap zone_map;
RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));
RETURN_IF_ERROR(_get_segment_zone_map(&zone_map));

*matched = _zone_map_match_condition(zone_map, col_predicates);
return Status::OK();
Expand All @@ -522,7 +565,7 @@ Status ColumnReader::prune_predicates_by_zone_map(
}

ZoneMap zone_map;
RETURN_IF_ERROR(ZoneMap::from_proto(*_segment_zone_map, _data_type, zone_map));
RETURN_IF_ERROR(_get_segment_zone_map(&zone_map));
if (zone_map.pass_all) {
return Status::OK();
}
Expand Down Expand Up @@ -554,21 +597,19 @@ Status ColumnReader::_get_filtered_pages(
std::vector<uint32_t>* page_indexes, const ColumnIteratorOptions& iter_opts) {
RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));

const std::vector<ZoneMapPB>& zone_maps = _zone_map_index->page_zone_maps();
const std::vector<ZoneMap>& zone_maps = _zone_map_index->page_zone_map_infos();
size_t page_size = _zone_map_index->num_pages();
for (size_t i = 0; i < page_size; ++i) {
if (zone_maps[i].pass_all()) {
if (zone_maps[i].pass_all) {
page_indexes->push_back(cast_set<uint32_t>(i));
} else {
segment_v2::ZoneMap zone_map;
RETURN_IF_ERROR(ZoneMap::from_proto(zone_maps[i], _data_type, zone_map));
if (_zone_map_match_condition(zone_map, col_predicates)) {
if (_zone_map_match_condition(zone_maps[i], col_predicates)) {
bool should_read = true;
if (delete_predicates != nullptr) {
for (auto del_pred : *delete_predicates) {
// TODO: Both `min_value` and `max_value` should be 0 or neither should be 0.
// So nullable only need to judge once.
if (del_pred->evaluate_del(zone_map)) {
if (del_pred->evaluate_del(zone_maps[i])) {
should_read = false;
break;
}
Expand Down Expand Up @@ -656,8 +697,7 @@ Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memo

Status ColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const {
DORIS_CHECK(zone_map != nullptr);
DORIS_CHECK(_segment_zone_map != nullptr);
return ZoneMap::from_proto(*_segment_zone_map, _data_type, *zone_map);
return _get_segment_zone_map(zone_map);
}

Status ConstantColumnReader::get_segment_zone_map(segment_v2::ZoneMap* zone_map) const {
Expand All @@ -679,6 +719,27 @@ Status ColumnReader::get_page_zone_maps(const ColumnIteratorOptions& iter_opts,
return Status::OK();
}

Status ColumnReader::get_page_zone_map_infos(const ColumnIteratorOptions& iter_opts,
const std::vector<segment_v2::ZoneMap>** zone_maps) {
DORIS_CHECK(zone_maps != nullptr);
if (_zone_map_index == nullptr) {
*zone_maps = nullptr;
return Status::OK();
}
RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
*zone_maps = &_zone_map_index->page_zone_map_infos();
return Status::OK();
}

Status ColumnReader::_get_segment_zone_map(segment_v2::ZoneMap* zone_map) const {
DORIS_CHECK(zone_map != nullptr);
if (_segment_zone_map_v2 != nullptr) {
return ZoneMap::from_entry_pb_native(*_segment_zone_map_v2, _data_type, *zone_map);
}
DORIS_CHECK(_segment_zone_map != nullptr);
return ZoneMap::from_proto(*_segment_zone_map, _data_type, *zone_map);
}

Status ColumnReader::get_row_range_for_page(uint32_t page_index,
const ColumnIteratorOptions& iter_opts,
RowRange* row_range) {
Expand Down Expand Up @@ -1171,12 +1232,10 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool*
key_ptr->insert_many_defaults(num_items);
val_ptr->insert_many_defaults(num_items);
} else {
size_t num_read = num_items;
bool key_has_null = false;
bool val_has_null = false;
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null));
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null));
DCHECK(num_read == num_items);
RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), false, 0, num_items,
key_ptr));
RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), false, 0, num_items,
val_ptr));
}
}

Expand Down Expand Up @@ -1377,18 +1436,11 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t
}
auto start = static_cast<ordinal_t>(starts_data[i]);
if (start != last_idx) {
size_t n = this_run;
bool dummy_has_null = false;

if (this_run != 0) {
RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null));
DCHECK(n == this_run);

n = this_run;
RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null));
DCHECK(n == this_run);
RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), true, start_idx,
this_run, keys_ptr));
RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), true, start_idx,
this_run, vals_ptr));
}
start_idx = start;
this_run = sz;
Expand All @@ -1400,17 +1452,11 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t
last_idx += sz;
}

size_t n = this_run;
bool dummy_has_null = false;
if (this_run != 0) {
RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx));
RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null));
DCHECK(n == this_run);

n = this_run;
RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx));
RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null));
DCHECK(n == this_run);
RETURN_IF_ERROR(read_or_fill_map_item_column(_key_iterator.get(), true, start_idx, this_run,
keys_ptr));
RETURN_IF_ERROR(read_or_fill_map_item_column(_val_iterator.get(), true, start_idx, this_run,
vals_ptr));
}
return Status::OK();
}
Expand Down
4 changes: 4 additions & 0 deletions be/src/storage/segment/column_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,8 @@ class ColumnReader : public MetadataAdder<ColumnReader>,
virtual Status get_segment_zone_map(segment_v2::ZoneMap* zone_map) const;
Status get_page_zone_maps(const ColumnIteratorOptions& iter_opts,
const std::vector<ZoneMapPB>** zone_maps);
Status get_page_zone_map_infos(const ColumnIteratorOptions& iter_opts,
const std::vector<segment_v2::ZoneMap>** zone_maps);
Status get_row_range_for_page(uint32_t page_index, const ColumnIteratorOptions& iter_opts,
RowRange* row_range);

Expand Down Expand Up @@ -275,6 +277,7 @@ class ColumnReader : public MetadataAdder<ColumnReader>,

bool _zone_map_match_condition(const segment_v2::ZoneMap& zone_map,
const AndBlockColumnPredicate* col_predicates) const;
Status _get_segment_zone_map(segment_v2::ZoneMap* zone_map) const;

Status _get_filtered_pages(
const AndBlockColumnPredicate* col_predicates,
Expand Down Expand Up @@ -310,6 +313,7 @@ class ColumnReader : public MetadataAdder<ColumnReader>,

// meta for various column indexes (null if the index is absent)
std::unique_ptr<ZoneMapPB> _segment_zone_map;
std::unique_ptr<ZoneMapEntryPB> _segment_zone_map_v2;

mutable std::shared_mutex _load_index_lock;
std::unique_ptr<ZoneMapIndexReader> _zone_map_index;
Expand Down
9 changes: 3 additions & 6 deletions be/src/storage/segment/segment_iterator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3427,8 +3427,8 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr
if (reader == nullptr || !reader->has_zone_map()) {
continue;
}
const std::vector<ZoneMapPB>* page_zone_maps = nullptr;
RETURN_IF_ERROR(reader->get_page_zone_maps(iter_opts, &page_zone_maps));
const std::vector<ZoneMap>* page_zone_maps = nullptr;
RETURN_IF_ERROR(reader->get_page_zone_map_infos(iter_opts, &page_zone_maps));
if (page_zone_maps == nullptr || page_zone_maps->empty()) {
continue;
}
Expand All @@ -3448,10 +3448,7 @@ Status SegmentIterator::_apply_expr_zonemap_to_row_ranges(const VExprContextSPtr
ZoneMapEvalContext ctx;
ZoneMapEvalContext::SlotZoneMap slot_zone_map;
slot_zone_map.data_type = data_type;
ZoneMap zone_map;
RETURN_IF_ERROR(
ZoneMap::from_proto((*page_zone_maps)[page_index], data_type, zone_map));
slot_zone_map.zone_map = std::make_shared<ZoneMap>(std::move(zone_map));
slot_zone_map.zone_map = std::make_shared<ZoneMap>((*page_zone_maps)[page_index]);
ctx.slots.emplace(slot_index, std::move(slot_zone_map));
const auto result = VExprContext::evaluate_zonemap_filter(slot_conjuncts, ctx);
page_stats.merge_page_eval_stats(ctx.stats);
Expand Down
Loading
Loading