From 303804dd1c09accb6ad5991629f43ddd1bafa866 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:30:48 +0800 Subject: [PATCH] [fix](regression) Stabilize base compaction file size limit case --- ...on_with_dup_key_max_file_size_limit.groovy | 95 ++++++++++++------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy b/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy index a006ea406d62eb..ad9a1e44008ddf 100644 --- a/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy +++ b/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy @@ -17,7 +17,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { +suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2,nonConcurrent") { def tableName = "test_base_compaction_with_dup_key_max_file_size_limit" // use customer table of tpch_sf100 @@ -65,27 +65,54 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { } } } - try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - } + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_params = [string: [:]] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + def set_be_param = { paramName, paramValue -> + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def (rcode, rout, rerr) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) + assertTrue(rout.contains("OK")) + } + } + + def reset_be_param = { paramName -> + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def originalValue = backendId_to_params.get(id).get(paramName) + def (rcode, rout, rerr) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, originalValue)) + assertTrue(rout.contains("OK")) } + } + + def get_be_param = { paramName -> + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def (rcode, rout, rerr) = curl("GET", String.format("http://%s:%s/api/show_config?conf_item=%s", beIp, bePort, paramName)) + assertEquals(rcode, 0) + assertTrue(rout.contains(paramName)) + def resultList = parseJson(rout)[0] + assertTrue(resultList.size() == 4) + backendId_to_params.get(id, [:]).put(paramName, resultList[2]) + } + } + + get_be_param("disable_auto_compaction") + get_be_param("base_compaction_dup_key_max_file_size_mbytes") + + try { + // This test expects manual base compaction to be rejected with E-808 after building a + // single large base rowset. Background auto compaction can reshape those rowsets, so keep + // it disabled cluster-wide while the case is running. + set_be_param("disable_auto_compaction", "true") + // The encoded size of the 15M-row TPCH customer load can be below the default 1GB limit. + // Lower the limit so the generated base rowset deterministically hits the size gate. + set_be_param("base_compaction_dup_key_max_file_size_mbytes", "100") def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> // trigger compactions for all tablets in ${tableName} @@ -98,15 +125,12 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { String command = sb.toString() logger.info(command) def process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success, " + out - } - assertEquals(code, 0) - return out + def ccode = process.waitFor() + def cerr = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + def cout = process.getText() + logger.info("Run compaction: code=" + ccode + ", out=" + cout + ", err=" + cerr) + assertEquals(ccode, 0) + return cout } sql """ DROP TABLE IF EXISTS ${tableName}; """ @@ -177,11 +201,12 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { // cp: 5 trigger_and_wait_compaction(tableName, "cumulative") - // Due to the limit of config::base_compaction_dup_key_max_file_size_mbytes(1G), - // can not do base compaction, return E-808 + // Due to the limit of config::base_compaction_dup_key_max_file_size_mbytes (100MB, set + // above), the first input rowset exceeds it, so base compaction can not run and returns + // E-808. // rowsets: - // [0-3] 2G nooverlapping - // [4-4] 1G nooverlapping + // [0-3] nooverlapping + // [4-4] nooverlapping // cp: 5 // WHAT: replace with plugin and handle fail? assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], @@ -190,5 +215,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { def rowCount = sql "select count(*) from ${tableName}" assertTrue(rowCount[0][0] != rows) } finally { + reset_be_param("disable_auto_compaction") + reset_be_param("base_compaction_dup_key_max_file_size_mbytes") } }