# [fix](topn lazy materialization) Disable topn lazy materialization on non-light-schema-change tables

fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java

@@ -87,9 +87,9 @@ public ProbeContext(SlotReference slot, Set<Slot> requiredMaterializedSlots) {
     public Optional<MaterializeSource> visitPhysicalFilter(PhysicalFilter<? extends Plan> filter,
                                                            ProbeContext context) {
         if (SessionVariable.getTopNLazyMaterializationUsingIndex() && filter.child() instanceof PhysicalOlapScan) {
-            // agg table do not support lazy materialize
+            // agg table / non-light-schema-change table do not support lazy materialize
             OlapTable table = ((PhysicalOlapScan) filter.child()).getTable();
-            if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+            if (!supportOlapTopnLazyMaterialize(table)) {
                 return Optional.empty();
             }
             if (filter.getInputSlots().contains(context.slot)) {
@@ -133,6 +133,34 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) {
             return (hmsExternalTable.getDlaType() == DLAType.HIVE && hmsExternalTable.supportedHiveTopNLazyTable())
                     || hmsExternalTable.getDlaType() == DLAType.ICEBERG;
         }
+        if (relation.getTable() instanceof OlapTable) {
+            return supportOlapTopnLazyMaterialize((OlapTable) relation.getTable());
+        }
+        return true;
+    }
+
+    /**
+     * Whether an OLAP table can perform topn lazy materialization.
+     *
+     * <p>Two hard requirements:
+     * <ul>
+     *   <li>Not an AGG_KEYS table: aggregate tables cannot locate a single source row for a value.</li>
+     *   <li>light_schema_change is enabled: lazy materialization appends a synthetic global row-id
+     *       column to the scan and relies on the BE rebuilding the tablet schema from FE's
+     *       columns_desc (keyed by column uniqueId). For non-light-schema-change tables every
+     *       column's uniqueId is -1, so the BE keeps its own on-disk schema and never installs the
+     *       synthetic row-id column. That path either fails with "field name is invalid" during the
+     *       scan or silently returns NULL for the lazily-fetched columns. Disable the optimization
+     *       for such tables and fall back to normal topn.</li>
+     * </ul>
+     */
+    private boolean supportOlapTopnLazyMaterialize(OlapTable table) {
+        if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+            return false;
+        }
+        if (!table.getEnableLightSchemaChange()) {
+            return false;
+        }
         return true;
     }
 
@@ -155,9 +183,9 @@ public Optional<MaterializeSource> visitPhysicalOlapScan(PhysicalOlapScan scan,
         if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()) {
             return Optional.empty();
         }
-        // agg table do not support lazy materialize
+        // agg table / non-light-schema-change table do not support lazy materialize
         OlapTable table = scan.getTable();
-        if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
+        if (!supportOlapTopnLazyMaterialize(table)) {
             return Optional.empty();
         }
         if (context.requiredMaterializedSlots.contains(context.slot)) {

regression-test/data/query_p0/topn_lazy/topn_lazy_light_schema_change.out

@@ -0,0 +1,32 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !shape_lsc_false_not_lazy --
+PhysicalResultSink
+--PhysicalProject
+----PhysicalTopN[MERGE_SORT]
+------PhysicalDistribute[DistributionSpecGather]
+--------PhysicalTopN[LOCAL_SORT]
+----------PhysicalProject
+------------filter((topn_lazy_lsc_false.__DORIS_DELETE_SIGN__ = 0))
+--------------PhysicalOlapScan[topn_lazy_lsc_false]
+
+-- !result_lsc_false --
+5	eee
+4	ddd
+3	ccc
+
+-- !shape_lsc_true_lazy --
+PhysicalResultSink
+--PhysicalProject
+----PhysicalLazyMaterialize[materializedSlots:(topn_lazy_lsc_true.createdate) lazySlots:(topn_lazy_lsc_true.name)]
+------PhysicalTopN[MERGE_SORT]
+--------PhysicalDistribute[DistributionSpecGather]
+----------PhysicalTopN[LOCAL_SORT]
+------------PhysicalProject
+--------------filter((topn_lazy_lsc_true.__DORIS_DELETE_SIGN__ = 0))
+----------------PhysicalLazyMaterializeOlapScan[topn_lazy_lsc_true lazySlots:(topn_lazy_lsc_true.name)]
+
+-- !result_lsc_true --
+5	eee
+4	ddd
+3	ccc
+

regression-test/suites/query_p0/topn_lazy/topn_lazy_light_schema_change.groovy

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression test for topn lazy materialization on non-light-schema-change tables.
+//
+// TopN lazy materialization appends a synthetic global row-id column
+// (__DORIS_GLOBAL_ROWID_COL__<table>) to the OLAP scan. The BE only rebuilds the
+// tablet schema from FE's columns_desc when columns_desc[0].col_unique_id >= 0,
+// which is only true for light_schema_change tables. On a non-light-schema-change
+// table every column's uniqueId is -1, so the synthetic row-id column never enters
+// the BE tablet schema and the scan fails with
+//   "field name is invalid. field=__DORIS_GLOBAL_ROWID_COL__<table>"
+// (or, if the scan schema is forced, the second-phase fetch silently returns NULL
+// for the lazily-materialized columns).
+//
+// The fix disables topn lazy materialization for non-light-schema-change OLAP tables
+// in MaterializeProbeVisitor, falling back to normal topn. This test verifies both
+// the plan shape (no lazy materialization) and correct results on such a table, and
+// verifies that a light_schema_change=true table still uses lazy materialization.
+suite("topn_lazy_light_schema_change") {
+    // ---- light_schema_change = false : lazy materialization must be disabled ----
+    sql """ drop table if exists topn_lazy_lsc_false """
+    sql """
+        CREATE TABLE topn_lazy_lsc_false (
+            `id` INT NOT NULL,
+            `name` VARCHAR(64),
+            `createdate` DATETIME
+        )
+        UNIQUE KEY(`id`)
+        DISTRIBUTED BY HASH(`id`) BUCKETS 3
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "light_schema_change" = "false",
+            "enable_unique_key_merge_on_write" = "false"
+        );
+    """
+    sql """
+        insert into topn_lazy_lsc_false values
+            (1, 'aaa', '2024-01-01 10:00:00'),
+            (2, 'bbb', '2024-01-02 10:00:00'),
+            (3, 'ccc', '2024-01-03 10:00:00'),
+            (4, 'ddd', '2024-01-04 10:00:00'),
+            (5, 'eee', '2024-01-05 10:00:00');
+    """
+
+    // Plan shape: a non-light-schema-change table must fall back to a plain
+    // PhysicalOlapScan with no PhysicalLazyMaterialize / PhysicalLazyMaterializeOlapScan.
+    qt_shape_lsc_false_not_lazy """
+        explain shape plan
+        select name from topn_lazy_lsc_false order by createdate desc limit 3
+    """
+
+    // Correctness: the query used to error with "field name is invalid" (or return NULL
+    // for name). It must now run and return the real values ordered by createdate desc.
+    qt_result_lsc_false """
+        select id, name from topn_lazy_lsc_false order by createdate desc limit 3
+    """
+
+    // ---- light_schema_change = true : lazy materialization must still apply ----
+    sql """ drop table if exists topn_lazy_lsc_true """
+    sql """
+        CREATE TABLE topn_lazy_lsc_true (
+            `id` INT NOT NULL,
+            `name` VARCHAR(64),
+            `createdate` DATETIME
+        )
+        UNIQUE KEY(`id`)
+        DISTRIBUTED BY HASH(`id`) BUCKETS 3
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "light_schema_change" = "true",
+            "enable_unique_key_merge_on_write" = "false"
+        );
+    """
+    sql """
+        insert into topn_lazy_lsc_true values
+            (1, 'aaa', '2024-01-01 10:00:00'),
+            (2, 'bbb', '2024-01-02 10:00:00'),
+            (3, 'ccc', '2024-01-03 10:00:00'),
+            (4, 'ddd', '2024-01-04 10:00:00'),
+            (5, 'eee', '2024-01-05 10:00:00');
+    """
+
+    // Plan shape: a light_schema_change table keeps lazy materialization
+    // (PhysicalLazyMaterialize present).
+    qt_shape_lsc_true_lazy """
+        explain shape plan
+        select name from topn_lazy_lsc_true order by createdate desc limit 3
+    """
+
+    // Correctness: lazy materialization returns the same real values.
+    qt_result_lsc_true """
+        select id, name from topn_lazy_lsc_true order by createdate desc limit 3
+    """
+}