diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index 0a49789660096d..9bcdedb01e6196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -87,9 +87,9 @@ public ProbeContext(SlotReference slot, Set requiredMaterializedSlots) { public Optional visitPhysicalFilter(PhysicalFilter filter, ProbeContext context) { if (SessionVariable.getTopNLazyMaterializationUsingIndex() && filter.child() instanceof PhysicalOlapScan) { - // agg table do not support lazy materialize + // agg table / non-light-schema-change table do not support lazy materialize OlapTable table = ((PhysicalOlapScan) filter.child()).getTable(); - if (KeysType.AGG_KEYS.equals(table.getKeysType())) { + if (!supportOlapTopnLazyMaterialize(table)) { return Optional.empty(); } if (filter.getInputSlots().contains(context.slot)) { @@ -133,6 +133,34 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { return (hmsExternalTable.getDlaType() == DLAType.HIVE && hmsExternalTable.supportedHiveTopNLazyTable()) || hmsExternalTable.getDlaType() == DLAType.ICEBERG; } + if (relation.getTable() instanceof OlapTable) { + return supportOlapTopnLazyMaterialize((OlapTable) relation.getTable()); + } + return true; + } + + /** + * Whether an OLAP table can perform topn lazy materialization. + * + *

Two hard requirements: + *

+ */ + private boolean supportOlapTopnLazyMaterialize(OlapTable table) { + if (KeysType.AGG_KEYS.equals(table.getKeysType())) { + return false; + } + if (!table.getEnableLightSchemaChange()) { + return false; + } return true; } @@ -155,9 +183,9 @@ public Optional visitPhysicalOlapScan(PhysicalOlapScan scan, if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()) { return Optional.empty(); } - // agg table do not support lazy materialize + // agg table / non-light-schema-change table do not support lazy materialize OlapTable table = scan.getTable(); - if (KeysType.AGG_KEYS.equals(table.getKeysType())) { + if (!supportOlapTopnLazyMaterialize(table)) { return Optional.empty(); } if (context.requiredMaterializedSlots.contains(context.slot)) { diff --git a/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out new file mode 100644 index 00000000000000..d8d82278ed8810 --- /dev/null +++ b/regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !shape_lsc_false_not_lazy -- +PhysicalResultSink +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------PhysicalProject +------------filter((topn_lazy_lsc_false.__DORIS_DELETE_SIGN__ = 0)) +--------------PhysicalOlapScan[topn_lazy_lsc_false] + +-- !result_lsc_false -- +5 eee +4 ddd +3 ccc + +-- !shape_lsc_true_lazy -- +PhysicalResultSink +--PhysicalProject +----PhysicalLazyMaterialize[materializedSlots:(topn_lazy_lsc_true.createdate) lazySlots:(topn_lazy_lsc_true.name)] +------PhysicalTopN[MERGE_SORT] +--------PhysicalDistribute[DistributionSpecGather] +----------PhysicalTopN[LOCAL_SORT] +------------PhysicalProject +--------------filter((topn_lazy_lsc_true.__DORIS_DELETE_SIGN__ = 0)) +----------------PhysicalLazyMaterializeOlapScan[topn_lazy_lsc_true lazySlots:(topn_lazy_lsc_true.name)] + +-- !result_lsc_true -- +5 eee +4 ddd +3 ccc + diff --git a/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy new file mode 100644 index 00000000000000..f46a8ebb87ba54 --- /dev/null +++ b/regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Regression test for topn lazy materialization on non-light-schema-change tables. +// +// TopN lazy materialization appends a synthetic global row-id column +// (__DORIS_GLOBAL_ROWID_COL__) to the OLAP scan. The BE only rebuilds the +// tablet schema from FE's columns_desc when columns_desc[0].col_unique_id >= 0, +// which is only true for light_schema_change tables. On a non-light-schema-change +// table every column's uniqueId is -1, so the synthetic row-id column never enters +// the BE tablet schema and the scan fails with +// "field name is invalid. field=__DORIS_GLOBAL_ROWID_COL__
" +// (or, if the scan schema is forced, the second-phase fetch silently returns NULL +// for the lazily-materialized columns). +// +// The fix disables topn lazy materialization for non-light-schema-change OLAP tables +// in MaterializeProbeVisitor, falling back to normal topn. This test verifies both +// the plan shape (no lazy materialization) and correct results on such a table, and +// verifies that a light_schema_change=true table still uses lazy materialization. +suite("topn_lazy_light_schema_change") { + // ---- light_schema_change = false : lazy materialization must be disabled ---- + sql """ drop table if exists topn_lazy_lsc_false """ + sql """ + CREATE TABLE topn_lazy_lsc_false ( + `id` INT NOT NULL, + `name` VARCHAR(64), + `createdate` DATETIME + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "false", + "enable_unique_key_merge_on_write" = "false" + ); + """ + sql """ + insert into topn_lazy_lsc_false values + (1, 'aaa', '2024-01-01 10:00:00'), + (2, 'bbb', '2024-01-02 10:00:00'), + (3, 'ccc', '2024-01-03 10:00:00'), + (4, 'ddd', '2024-01-04 10:00:00'), + (5, 'eee', '2024-01-05 10:00:00'); + """ + + // Plan shape: a non-light-schema-change table must fall back to a plain + // PhysicalOlapScan with no PhysicalLazyMaterialize / PhysicalLazyMaterializeOlapScan. + qt_shape_lsc_false_not_lazy """ + explain shape plan + select name from topn_lazy_lsc_false order by createdate desc limit 3 + """ + + // Correctness: the query used to error with "field name is invalid" (or return NULL + // for name). It must now run and return the real values ordered by createdate desc. + qt_result_lsc_false """ + select id, name from topn_lazy_lsc_false order by createdate desc limit 3 + """ + + // ---- light_schema_change = true : lazy materialization must still apply ---- + sql """ drop table if exists topn_lazy_lsc_true """ + sql """ + CREATE TABLE topn_lazy_lsc_true ( + `id` INT NOT NULL, + `name` VARCHAR(64), + `createdate` DATETIME + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "true", + "enable_unique_key_merge_on_write" = "false" + ); + """ + sql """ + insert into topn_lazy_lsc_true values + (1, 'aaa', '2024-01-01 10:00:00'), + (2, 'bbb', '2024-01-02 10:00:00'), + (3, 'ccc', '2024-01-03 10:00:00'), + (4, 'ddd', '2024-01-04 10:00:00'), + (5, 'eee', '2024-01-05 10:00:00'); + """ + + // Plan shape: a light_schema_change table keeps lazy materialization + // (PhysicalLazyMaterialize present). + qt_shape_lsc_true_lazy """ + explain shape plan + select name from topn_lazy_lsc_true order by createdate desc limit 3 + """ + + // Correctness: lazy materialization returns the same real values. + qt_result_lsc_true """ + select id, name from topn_lazy_lsc_true order by createdate desc limit 3 + """ +}