From 582594cc41b1469a721142aaff5bbc0051251543 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 30 Jun 2026 16:14:58 +0800 Subject: [PATCH 01/18] [fix](be) Validate task executor scan handles Issue Number: None Related PR: None Problem Summary: Task-executor scan scheduling could pass a null or invalid task handle into TimeSharingTaskExecutor. enqueue_splits and related paths cast the base TaskHandle to TimeSharingTaskHandle and immediately dereferenced the result, so a broken ScannerContext task-handle invariant caused BE to crash with SIGSEGV instead of returning a diagnostic error. This change validates scanner context, scan task, and task handle before submitting scan splits, and validates the task handle type at TimeSharingTaskExecutor entry points before dereferencing it. None - Test: Unit Test - Added TimeSharingTaskExecutorTest coverage for null and invalid task handles. - Tried: JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home ./run-be-ut.sh --run --filter='TimeSharingTaskExecutorTest.*'; build failed during CMake configure because thirdparty/installed is missing Snappy. - Behavior changed: Yes. Invalid task-executor scan handles now return InternalError instead of dereferencing a null cast result. - Does this need documentation: No (cherry picked from commit 82cb6bfaa0e13ee563b68bf81f09b63701041c0a) --- .../time_sharing_task_executor.cpp | 28 ++++++- be/src/vec/exec/scan/scanner_scheduler.h | 22 ++++- .../time_sharing_task_executor_test.cpp | 83 ++++++++++++++++++- 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/be/src/vec/exec/executor/time_sharing/time_sharing_task_executor.cpp b/be/src/vec/exec/executor/time_sharing/time_sharing_task_executor.cpp index 4459f20037babe..876645927fa0f9 100644 --- a/be/src/vec/exec/executor/time_sharing/time_sharing_task_executor.cpp +++ b/be/src/vec/exec/executor/time_sharing/time_sharing_task_executor.cpp @@ -58,6 +58,24 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(thread_pool_task_wait_worker_time_ns_total, MetricUnit::NANOSECONDS); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(thread_pool_task_wait_worker_count_total, MetricUnit::NOUNIT); +namespace { + +Result> get_time_sharing_task_handle( + const std::shared_ptr& task_handle, const char* operation) { + if (task_handle == nullptr) { + return ResultError(Status::InternalError("{} got null task handle", operation)); + } + + auto handle = std::dynamic_pointer_cast(task_handle); + if (handle == nullptr) { + return ResultError(Status::InternalError("{} got invalid task handle type, task id: {}", + operation, task_handle->task_id().to_string())); + } + return handle; +} + +} // namespace + SplitThreadPoolToken::SplitThreadPoolToken(TimeSharingTaskExecutor* pool, TimeSharingTaskExecutor::ExecutionMode mode, std::shared_ptr split_queue, @@ -755,7 +773,7 @@ Status TimeSharingTaskExecutor::add_task(const TaskId& task_id, } Status TimeSharingTaskExecutor::remove_task(std::shared_ptr task_handle) { - auto handle = std::dynamic_pointer_cast(task_handle); + auto handle = DORIS_TRY(get_time_sharing_task_handle(task_handle, "remove_task")); std::vector> splits_to_destroy; { @@ -818,7 +836,11 @@ Result>> TimeSharingTaskExecutor::enque } }}; std::vector> finished_futures; - auto handle = std::dynamic_pointer_cast(task_handle); + auto handle_result = get_time_sharing_task_handle(task_handle, "enqueue_splits"); + if (!handle_result.has_value()) { + return ResultError(handle_result.error()); + } + auto handle = handle_result.value(); { std::unique_lock lock(_mutex); for (const auto& task_split : splits) { @@ -851,7 +873,7 @@ Result>> TimeSharingTaskExecutor::enque Status TimeSharingTaskExecutor::re_enqueue_split(std::shared_ptr task_handle, bool intermediate, const std::shared_ptr& split) { - auto handle = std::dynamic_pointer_cast(task_handle); + auto handle = DORIS_TRY(get_time_sharing_task_handle(task_handle, "re_enqueue_split")); std::shared_ptr prioritized_split = handle->get_split(split, intermediate); prioritized_split->reset_level_priority(); diff --git a/be/src/vec/exec/scan/scanner_scheduler.h b/be/src/vec/exec/scan/scanner_scheduler.h index 089f3e1e5b75e7..1790c30faa8670 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.h +++ b/be/src/vec/exec/scan/scanner_scheduler.h @@ -294,13 +294,28 @@ class TaskExecutorSimplifiedScanScheduler final : public ScannerScheduler { Status submit_scan_task(SimplifiedScanTask scan_task) override { if (!_is_stop) { + if (scan_task.scanner_context == nullptr) { + return Status::InternalError("scanner pool {} got null scanner context.", + _sched_name); + } + if (scan_task.scan_task == nullptr) { + return Status::InternalError("scanner pool {} got null scan task.", + _sched_name); + } + auto task_handle = scan_task.scanner_context->task_handle(); + if (task_handle == nullptr) { + return Status::InternalError( + "scanner pool {} got null task handle, scan task first schedule: {}, " + "scanner context: {}", + _sched_name, scan_task.scan_task->is_first_schedule, + scan_task.scanner_context->debug_string()); + } std::shared_ptr split_runner; if (scan_task.scan_task->is_first_schedule) { split_runner = std::make_shared("scanner_split_runner", scan_task.scan_func); RETURN_IF_ERROR(split_runner->init()); - auto result = _task_executor->enqueue_splits( - scan_task.scanner_context->task_handle(), false, {split_runner}); + auto result = _task_executor->enqueue_splits(task_handle, false, {split_runner}); if (!result.has_value()) { LOG(WARNING) << "enqueue_splits failed: " << result.error(); return result.error(); @@ -311,8 +326,7 @@ class TaskExecutorSimplifiedScanScheduler final : public ScannerScheduler { if (split_runner == nullptr) { return Status::OK(); } - RETURN_IF_ERROR(_task_executor->re_enqueue_split( - scan_task.scanner_context->task_handle(), false, split_runner)); + RETURN_IF_ERROR(_task_executor->re_enqueue_split(task_handle, false, split_runner)); } scan_task.scan_task->split_runner = split_runner; return Status::OK(); diff --git a/be/test/vec/exec/executor/time_sharing/time_sharing_task_executor_test.cpp b/be/test/vec/exec/executor/time_sharing/time_sharing_task_executor_test.cpp index a1e604cb6f92bd..70187f9ef66e45 100644 --- a/be/test/vec/exec/executor/time_sharing/time_sharing_task_executor_test.cpp +++ b/be/test/vec/exec/executor/time_sharing/time_sharing_task_executor_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "vec/exec/executor/ticker.h" @@ -290,13 +291,51 @@ class TestingSplitRunner : public SplitRunner { ListenableFuture _completion_future {}; }; +class QueueOnlySplitRunner : public SplitRunner { +public: + Status init() override { return Status::OK(); } + + Result> process_for(std::chrono::nanoseconds) override { + _started = true; + _finished = true; + return SharedListenableFuture::create_ready(); + } + + void close(const Status& status) override {} + + bool is_finished() override { return _finished.load(); } + + Status finished_status() override { return Status::OK(); } + + std::string get_info() const override { return "queue_only_split"; } + + bool is_started() const { return _started.load(); } + +private: + std::atomic _started {false}; + std::atomic _finished {false}; +}; + +class TestingTaskHandle final : public TaskHandle { +public: + explicit TestingTaskHandle(std::string task_id) : _task_id(std::move(task_id)) {} + + Status init() override { return Status::OK(); } + + bool is_closed() const override { return false; } + + TaskId task_id() const override { return _task_id; } + +private: + TaskId _task_id; +}; + class TimeSharingTaskExecutorTest : public testing::Test { protected: void SetUp() override {} void TearDown() override {} -private: template void assert_split_states(int end_index, const Container& splits) { for (int i = 0; i <= end_index; ++i) { @@ -324,6 +363,48 @@ class TimeSharingTaskExecutorTest : public testing::Test { } }; +TEST_F(TimeSharingTaskExecutorTest, test_invalid_task_handle_returns_error) { + auto ticker = std::make_shared(); + + TimeSharingTaskExecutor::ThreadConfig thread_config; + thread_config.thread_name = "invalid_task_handle"; + thread_config.workload_group = "normal"; + TimeSharingTaskExecutor executor(thread_config, 0, 1, 1, ticker); + ASSERT_TRUE(executor.init().ok()); + + auto split = std::make_shared(); + + auto null_enqueue_result = executor.enqueue_splits(nullptr, false, {split}); + ASSERT_FALSE(null_enqueue_result.has_value()); + EXPECT_NE(std::string(null_enqueue_result.error().msg()).find("null task handle"), + std::string::npos); + + Status null_re_enqueue_status = executor.re_enqueue_split(nullptr, false, split); + ASSERT_FALSE(null_re_enqueue_status.ok()); + EXPECT_NE(std::string(null_re_enqueue_status.msg()).find("null task handle"), + std::string::npos); + + Status null_remove_status = executor.remove_task(nullptr); + ASSERT_FALSE(null_remove_status.ok()); + EXPECT_NE(std::string(null_remove_status.msg()).find("null task handle"), std::string::npos); + + auto invalid_task_handle = std::make_shared("invalid_task"); + auto invalid_enqueue_result = executor.enqueue_splits(invalid_task_handle, false, {split}); + ASSERT_FALSE(invalid_enqueue_result.has_value()); + EXPECT_NE(std::string(invalid_enqueue_result.error().msg()).find("invalid task handle type"), + std::string::npos); + + Status invalid_re_enqueue_status = executor.re_enqueue_split(invalid_task_handle, false, split); + ASSERT_FALSE(invalid_re_enqueue_status.ok()); + EXPECT_NE(std::string(invalid_re_enqueue_status.msg()).find("invalid task handle type"), + std::string::npos); + + Status invalid_remove_status = executor.remove_task(invalid_task_handle); + ASSERT_FALSE(invalid_remove_status.ok()); + EXPECT_NE(std::string(invalid_remove_status.msg()).find("invalid task handle type"), + std::string::npos); +} + TEST_F(TimeSharingTaskExecutorTest, test_tasks_complete) { auto ticker = std::make_shared(); From b342308dd538d6c8ce516ffc744b5c1df56b3249 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 2 Jul 2026 14:59:04 +0800 Subject: [PATCH 02/18] [fix](regression) Isolate S3 TVF insert paths Issue Number: None Related PR: None Problem Summary: The S3 TVF insert regression case wrote all runs under a fixed regression/insert_tvf_test prefix while also using delete_existing_files=true and wildcard reads. Concurrent regression jobs could delete each other's directories or infer schema from another run's files, causing flaky mismatches or missing columns. This change gives each suite run a UUID-scoped S3 prefix and uses that same prefix for both writes and reads, so deletes and wildcard reads are isolated to the current run. None - Test: Regression test attempted: ./run-regression-test.sh --run -d external_table_p0/tvf/insert -s test_insert_into_s3_tvf. It could not complete because local FE at 127.0.0.1:9030 refused the MySQL connection before the first SQL statement. - Behavior changed: No, test-only path isolation. - Does this need documentation: No (cherry picked from commit 7a0116c8aa29e3034fba7e377f860bd4ec47ba26) --- .../tvf/insert/test_insert_into_s3_tvf.groovy | 567 ++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy diff --git a/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy b/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy new file mode 100644 index 00000000000000..c6b00182852393 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy @@ -0,0 +1,567 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_insert_into_s3_tvf", "p0,external") { + + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName") + + if (ak == null || ak == "" || sk == null || sk == "") { + logger.info("S3 not configured, skip") + return + } + + def s3BasePrefix = "regression/insert_tvf_test/${UUID.randomUUID().toString()}" + def s3BasePath = "${bucket}/${s3BasePrefix}" + + // file_path is now a prefix; BE generates: {prefix}{query_id}_{idx}.{ext} + def s3WriteProps = { String path, String format -> + return """ + "file_path" = "s3://${s3BasePath}/${path}", + "format" = "${format}", + "s3.endpoint" = "${s3_endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "s3.region" = "${region}" + """ + } + + // Read uses wildcard to match generated file names + def s3ReadProps = { String path, String format -> + return """ + "uri" = "https://${bucket}.${s3_endpoint}/${s3BasePrefix}/${path}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "format" = "${format}", + "region" = "${region}" + """ + } + + // ============ Source tables ============ + + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_src """ + sql """ + CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_src ( + c_bool BOOLEAN, + c_tinyint TINYINT, + c_smallint SMALLINT, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE, + c_decimal DECIMAL(10,2), + c_date DATE, + c_datetime DATETIME, + c_varchar VARCHAR(100), + c_string STRING + ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + + sql """ + INSERT INTO test_insert_into_s3_tvf_src VALUES + (true, 1, 100, 1000, 100000, 1.1, 2.2, 123.45, '2024-01-01', '2024-01-01 10:00:00', 'hello', 'world'), + (false, 2, 200, 2000, 200000, 3.3, 4.4, 678.90, '2024-06-15', '2024-06-15 12:30:00', 'foo', 'bar'), + (true, 3, 300, 3000, 300000, 5.5, 6.6, 999.99, '2024-12-31', '2024-12-31 23:59:59', 'test', 'data'), + (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL), + (false, -1, -100, -1000, -100000, -1.1, -2.2, -123.45,'2020-02-29', '2020-02-29 00:00:00', '', 'special_chars'); + """ + + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_complex_src """ + sql """ + CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_complex_src ( + c_int INT, + c_array ARRAY, + c_map MAP, + c_struct STRUCT + ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + + sql """ + INSERT INTO test_insert_into_s3_tvf_complex_src VALUES + (1, [1, 2, 3], {'a': 1, 'b': 2}, {1, 'hello'}), + (2, [4, 5], {'x': 10}, {2, 'world'}), + (3, [], {}, {3, ''}), + (4, NULL, NULL, NULL); + """ + + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_join_src """ + sql """ + CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_join_src ( + c_int INT, + c_label STRING + ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + + sql """ INSERT INTO test_insert_into_s3_tvf_join_src VALUES (1000, 'label_a'), (2000, 'label_b'), (3000, 'label_c'); """ + + // ============ 1. S3 CSV basic types ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("basic_csv/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; + """ + + order_qt_s3_csv_basic_types """ + SELECT * FROM s3( + ${s3ReadProps("basic_csv/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 2. S3 Parquet basic types ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("basic_parquet/data_", "parquet")}, + "delete_existing_files" = "true" + ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; + """ + + order_qt_s3_parquet_basic_types """ + SELECT * FROM s3( + ${s3ReadProps("basic_parquet/*", "parquet")} + ) ORDER BY c_int; + """ + + // ============ 3. S3 ORC basic types ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("basic_orc/data_", "orc")}, + "delete_existing_files" = "true" + ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; + """ + + order_qt_s3_orc_basic_types """ + SELECT * FROM s3( + ${s3ReadProps("basic_orc/*", "orc")} + ) ORDER BY c_int; + """ + + // ============ 4. S3 Parquet complex types ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("complex_parquet/data_", "parquet")}, + "delete_existing_files" = "true" + ) SELECT * FROM test_insert_into_s3_tvf_complex_src ORDER BY c_int; + """ + + order_qt_s3_parquet_complex_types """ + SELECT * FROM s3( + ${s3ReadProps("complex_parquet/*", "parquet")} + ) ORDER BY c_int; + """ + + // ============ 5. S3 ORC complex types ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("complex_orc/data_", "orc")}, + "delete_existing_files" = "true" + ) SELECT * FROM test_insert_into_s3_tvf_complex_src ORDER BY c_int; + """ + + order_qt_s3_orc_complex_types """ + SELECT * FROM s3( + ${s3ReadProps("complex_orc/*", "orc")} + ) ORDER BY c_int; + """ + + // ============ 6. S3 CSV separator: comma ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("sep_comma/data_", "csv")}, + "column_separator" = ",", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_sep_comma """ + SELECT * FROM s3( + ${s3ReadProps("sep_comma/*", "csv")}, + "column_separator" = "," + ) ORDER BY c1; + """ + + // ============ 7. S3 CSV separator: tab ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("sep_tab/data_", "csv")}, + "column_separator" = "\t", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_sep_tab """ + SELECT * FROM s3( + ${s3ReadProps("sep_tab/*", "csv")}, + "column_separator" = "\t" + ) ORDER BY c1; + """ + + // ============ 8. S3 CSV separator: pipe ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("sep_pipe/data_", "csv")}, + "column_separator" = "|", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_sep_pipe """ + SELECT * FROM s3( + ${s3ReadProps("sep_pipe/*", "csv")}, + "column_separator" = "|" + ) ORDER BY c1; + """ + + // ============ 9. S3 CSV separator: multi-char ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("sep_multi/data_", "csv")}, + "column_separator" = ";;", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_sep_multi """ + SELECT * FROM s3( + ${s3ReadProps("sep_multi/*", "csv")}, + "column_separator" = ";;" + ) ORDER BY c1; + """ + + // ============ 10. S3 CSV line delimiter: CRLF ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("line_crlf/data_", "csv")}, + "line_delimiter" = "\r\n", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_line_crlf """ + SELECT * FROM s3( + ${s3ReadProps("line_crlf/*", "csv")}, + "line_delimiter" = "\r\n" + ) ORDER BY c1; + """ + + // ============ 11. S3 CSV compress: gz ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("compress_gz/data_", "csv")}, + "compression_type" = "gz", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_compress_gz """ + SELECT * FROM s3( + ${s3ReadProps("compress_gz/*", "csv")}, + "compress_type" = "gz" + ) ORDER BY c1; + """ + + // ============ 12. S3 CSV compress: zstd ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("compress_zstd/data_", "csv")}, + "compression_type" = "zstd", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_compress_zstd """ + SELECT * FROM s3( + ${s3ReadProps("compress_zstd/*", "csv")}, + "compress_type" = "zstd" + ) ORDER BY c1; + """ + + // ============ 13. S3 CSV compress: lz4 ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("compress_lz4/data_", "csv")}, + "compression_type" = "lz4block", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_compress_lz4 """ + SELECT * FROM s3( + ${s3ReadProps("compress_lz4/*", "csv")}, + "compress_type" = "lz4block" + ) ORDER BY c1; + """ + + // ============ 14. S3 CSV compress: snappy ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("compress_snappy/data_", "csv")}, + "compression_type" = "snappyblock", + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_csv_compress_snappy """ + SELECT * FROM s3( + ${s3ReadProps("compress_snappy/*", "csv")}, + "compress_type" = "snappyblock" + ) ORDER BY c1; + """ + + // ============ 15. S3 Overwrite mode ============ + // delete_existing_files=true is handled by FE for S3 + + // First write: 5 rows + sql """ + INSERT INTO s3( + ${s3WriteProps("overwrite/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src ORDER BY c_int; + """ + + order_qt_s3_overwrite_first """ + SELECT * FROM s3( + ${s3ReadProps("overwrite/*", "csv")} + ) ORDER BY c1; + """ + + // Second write: 2 rows with overwrite (FE deletes the directory first) + sql """ + INSERT INTO s3( + ${s3WriteProps("overwrite/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int > 0 ORDER BY c_int LIMIT 2; + """ + + order_qt_s3_overwrite_second """ + SELECT * FROM s3( + ${s3ReadProps("overwrite/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 16. S3 Append mode ============ + + // First write + sql """ + INSERT INTO s3( + ${s3WriteProps("append/data_", "parquet")}, + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 1000; + """ + + order_qt_s3_append_first """ + SELECT * FROM s3( + ${s3ReadProps("append/*", "parquet")} + ) ORDER BY c_int; + """ + + // Second write (append — different query_id produces different file name) + sql """ + INSERT INTO s3( + ${s3WriteProps("append/data_", "parquet")} + ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 2000; + """ + + order_qt_s3_append_second """ + SELECT * FROM s3( + ${s3ReadProps("append/*", "parquet")} + ) ORDER BY c_int; + """ + + // ============ 17. S3 Complex SELECT: constant expressions ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("const_expr/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT 1, 'hello', 3.14, CAST('2024-01-01' AS DATE); + """ + + order_qt_s3_const_expr """ + SELECT * FROM s3( + ${s3ReadProps("const_expr/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 18. S3 Complex SELECT: WHERE + GROUP BY ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("where_groupby/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT c_bool, COUNT(*), SUM(c_int) FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL GROUP BY c_bool ORDER BY c_bool; + """ + + order_qt_s3_where_groupby """ + SELECT * FROM s3( + ${s3ReadProps("where_groupby/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 19. S3 Complex SELECT: JOIN ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("join_query/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT a.c_int, a.c_varchar, b.c_label + FROM test_insert_into_s3_tvf_src a INNER JOIN test_insert_into_s3_tvf_join_src b ON a.c_int = b.c_int + ORDER BY a.c_int; + """ + + order_qt_s3_join_query """ + SELECT * FROM s3( + ${s3ReadProps("join_query/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 20. S3 Complex SELECT: subquery ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("subquery/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT * FROM (SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int) sub; + """ + + order_qt_s3_subquery """ + SELECT * FROM s3( + ${s3ReadProps("subquery/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 21. S3 Complex SELECT: type cast ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("type_cast/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT CAST(c_int AS BIGINT), CAST(c_float AS DOUBLE), CAST(c_date AS STRING) + FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; + """ + + order_qt_s3_type_cast """ + SELECT * FROM s3( + ${s3ReadProps("type_cast/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 22. S3 Complex SELECT: UNION ALL ============ + + sql """ + INSERT INTO s3( + ${s3WriteProps("union_query/data_", "csv")}, + "delete_existing_files" = "true" + ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 1000 + UNION ALL + SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 2000; + """ + + order_qt_s3_union_query """ + SELECT * FROM s3( + ${s3ReadProps("union_query/*", "csv")} + ) ORDER BY c1; + """ + + // ============ 23. Error: missing file_path ============ + + test { + sql """ + INSERT INTO s3( + "format" = "csv", + "s3.endpoint" = "${s3_endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "s3.region" = "${region}" + ) SELECT 1; + """ + exception "file_path" + } + + // ============ 24. Error: missing format ============ + + test { + sql """ + INSERT INTO s3( + "file_path" = "s3://${s3BasePath}/err/data_", + "s3.endpoint" = "${s3_endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "s3.region" = "${region}" + ) SELECT 1; + """ + exception "format" + } + + // ============ 25. Error: unsupported format ============ + + test { + sql """ + INSERT INTO s3( + "file_path" = "s3://${s3BasePath}/err/data_", + "format" = "json", + "s3.endpoint" = "${s3_endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "s3.region" = "${region}" + ) SELECT 1; + """ + exception "Unsupported" + } + + // ============ 26. Error: wildcard in file_path ============ + + test { + sql """ + INSERT INTO s3( + "file_path" = "s3://${s3BasePath}/wildcard/*.csv", + "format" = "csv", + "s3.endpoint" = "${s3_endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "s3.region" = "${region}" + ) SELECT 1; + """ + exception "wildcards" + } + + // ============ Cleanup ============ + + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_src """ + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_complex_src """ + sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_join_src """ +} From 539cfb2a79641197bcbaceaf1dbe0fcff6a882a7 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 7 Jul 2026 19:08:15 +0800 Subject: [PATCH 03/18] [fix](paimon) Enable IOManager for Paimon JNI reads Issue Number: None Related PR: None Problem Summary: Paimon JNI reads did not attach Paimon IOManager when creating TableRead, so primary-key merge reads in the Java scanner could not spill through Paimon IOManager. This change forwards catalog-level Paimon JNI IOManager options from FE to BE, creates and closes the IOManager in the Java scanner, supports an optional custom IOManager implementation class that is already available on the scanner classpath, and injects a default IOManager temp directory for both legacy and Format V2 JNI scan paths when the user enables IOManager without specifying a temp dir. The default temp directory is `${storage_root_path}/paimon_jni_scanner_io_tmp` for each BE storage root. Paimon JNI scanner can enable Paimon IOManager spill with catalog property `paimon.doris.enable_jni_io_manager=true`; `paimon.doris.jni_io_manager.tmp_dir` can optionally override the spill temp directory, and `paimon.doris.jni_io_manager.impl_class` can optionally select a custom `org.apache.paimon.disk.IOManager` implementation already available on the scanner classpath. - Test: Unit Test / Manual test - `mvn test -Dmaven.build.cache.enabled=false -pl be-java-extensions/paimon-scanner -Dtest=PaimonJniScannerTest` passed on `gabriel@10.26.20.3:/mnt/disk3/gabriel/Workspace/dev2/doris`. - `python3 build-support/run_clang_format.py ...` check-only passed for modified BE C++ files. - `git diff --check` passed. - `ninja -C be/ut_build_ASAN src/format/CMakeFiles/Format.dir/table/paimon_jni_reader.cpp.o src/format/CMakeFiles/Format.dir/__/format_v2/jni/paimon_jni_reader.cpp.o` compiled the modified reader objects on the remote validation host. - `./run-be-ut.sh --run --filter="PaimonJniReaderTest.*"` was attempted on the remote validation host but stopped before target tests due to unrelated missing header `faiss/invlists/PreadInvertedLists.h` while compiling `be/src/storage/index/ann/faiss_ann_index.cpp`. - `./run-fe-ut.sh --run org.apache.doris.datasource.paimon.source.PaimonScanNodeTest` was attempted on the remote validation host but stopped before target tests due to unrelated generated `ScalarBuiltins` compile errors referencing missing `FunctionSet`. - Behavior changed: Yes. When enabled, Paimon JNI reads now create a Paimon IOManager and use a storage-root scoped temp directory by default. Users can optionally choose a preinstalled custom IOManager implementation class. - Does this need documentation: No. The PR description documents the catalog properties, default temp directory, and custom IOManager implementation behavior. (cherry picked from commit c6bca61a45652fb229c554fd01d8f01b578c684c) --- .../exec/format/table/paimon_jni_reader.cpp | 27 ++++ .../vec/exec/format/table/paimon_jni_reader.h | 2 + .../apache/doris/paimon/PaimonJniScanner.java | 139 ++++++++++++++++- .../doris/paimon/PaimonJniScannerTest.java | 146 ++++++++++++++++++ .../paimon/source/PaimonScanNode.java | 48 ++++++ .../paimon/source/PaimonScanNodeTest.java | 32 ++++ 6 files changed, 391 insertions(+), 3 deletions(-) create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp b/be/src/vec/exec/format/table/paimon_jni_reader.cpp index 3c9afe93eb36b0..6e3f7d14f4c6cd 100644 --- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp +++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp @@ -18,10 +18,14 @@ #include "paimon_jni_reader.h" #include +#include +#include +#include "runtime/exec_env.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "runtime/types.h" +#include "util/string_util.h" #include "vec/core/block.h" #include "vec/core/types.h" namespace doris { @@ -35,8 +39,14 @@ class Block; namespace doris::vectorized { #include "common/compile_check_begin.h" +namespace { +constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; +} // namespace + const std::string PaimonJniReader::PAIMON_OPTION_PREFIX = "paimon."; const std::string PaimonJniReader::HADOOP_OPTION_PREFIX = "hadoop."; +const std::string PaimonJniReader::DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; +const std::string PaimonJniReader::DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; PaimonJniReader::PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, @@ -73,6 +83,23 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d for (const auto& kv : range.table_format_params.paimon_params.paimon_options) { params[PAIMON_OPTION_PREFIX + kv.first] = kv.second; } + const std::string enable_io_manager_key = + PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; + const std::string io_manager_tmp_dir_key = + PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; + auto enable_io_manager_it = params.find(enable_io_manager_key); + if (enable_io_manager_it != params.end() && iequal(enable_io_manager_it->second, "true") && + params.find(io_manager_tmp_dir_key) == params.end()) { + std::vector tmp_dirs; + for (const auto& store_path : state->exec_env()->store_paths()) { + tmp_dirs.push_back(store_path.path + "/" + + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + } + DORIS_CHECK(!tmp_dirs.empty()); + // Paimon's IOManager creates and later removes its own paimon-* child + // directory under these Doris storage-root scoped parent directories. + params[io_manager_tmp_dir_key] = join(tmp_dirs, ":"); + } // Prefer hadoop conf from scan node level (range_params->properties) over split level // to avoid redundant configuration in each split if (range_params->__isset.properties && !range_params->properties.empty()) { diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.h b/be/src/vec/exec/format/table/paimon_jni_reader.h index 81b5bd68d29a4d..78c17f9f2dbc80 100644 --- a/be/src/vec/exec/format/table/paimon_jni_reader.h +++ b/be/src/vec/exec/format/table/paimon_jni_reader.h @@ -50,6 +50,8 @@ class PaimonJniReader : public JniReader { public: static const std::string PAIMON_OPTION_PREFIX; static const std::string HADOOP_OPTION_PREFIX; + static const std::string DORIS_ENABLE_JNI_IO_MANAGER; + static const std::string DORIS_JNI_IO_MANAGER_TMP_DIR; PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, const TFileRangeDesc& range, const TFileScanRangeParams* range_params); diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 5690b7f65054c3..4adce6d2594fb8 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -25,18 +25,26 @@ import com.google.common.base.Preconditions; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; import org.apache.paimon.types.DataType; import org.apache.paimon.types.TimestampType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -45,6 +53,9 @@ public class PaimonJniScanner extends JniScanner { private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class); private static final String HADOOP_OPTION_PREFIX = "hadoop."; + static final String ENABLE_JNI_IO_MANAGER = "paimon.doris.enable_jni_io_manager"; + static final String JNI_IO_MANAGER_TMP_DIR = "paimon.doris.jni_io_manager.tmp_dir"; + static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.doris.jni_io_manager.impl_class"; private final Map params; private final Map hadoopOptionParams; @@ -52,6 +63,8 @@ public class PaimonJniScanner extends JniScanner { private final String paimonPredicate; private Table table; private RecordReader reader; + private IOManager ioManager; + private String ioManagerTempDirs; private final PaimonColumnValue columnValue = new PaimonColumnValue(); private List paimonAllFieldNames; private List paimonDataTypeList; @@ -99,6 +112,11 @@ public void open() throws IOException { resetDatetimeV2Precision(); } catch (Throwable e) { + try { + close(); + } catch (IOException closeException) { + e.addSuppressed(closeException); + } LOG.warn("Failed to open paimon_scanner: " + e.getMessage(), e); throw new RuntimeException(e); } @@ -116,11 +134,96 @@ private void initReader() throws IOException { int[] projected = getProjected(); readBuilder.withProjection(projected); readBuilder.withFilter(getPredicates()); - reader = readBuilder.newRead().executeFilter().createReader(getSplit()); + reader = newReadWithOptionalIOManager(readBuilder).executeFilter().createReader(getSplit()); paimonDataTypeList = Arrays.stream(projected).mapToObj(i -> table.rowType().getTypeAt(i)).collect(Collectors.toList()); } + private TableRead newReadWithOptionalIOManager(ReadBuilder readBuilder) throws IOException { + TableRead tableRead = readBuilder.newRead(); + if (!isIOManagerEnabled(params)) { + return tableRead; + } + ioManagerTempDirs = getIOManagerTempDirs(params); + ioManager = createIOManager(ioManagerTempDirs, getIOManagerImplClass(params)); + LOG.info("Enable Paimon JNI IOManager with temp dirs: {}, implementation: {}", + ioManagerTempDirs, ioManager.getClass().getName()); + return tableRead.withIOManager(ioManager); + } + + static boolean isIOManagerEnabled(Map params) { + return Boolean.parseBoolean(params.getOrDefault(ENABLE_JNI_IO_MANAGER, "false")); + } + + static String getIOManagerTempDirs(Map params) throws IOException { + String tempDirs = params.get(JNI_IO_MANAGER_TMP_DIR); + if (tempDirs == null || tempDirs.trim().isEmpty()) { + throw new IOException("Paimon JNI IOManager is enabled but " + JNI_IO_MANAGER_TMP_DIR + " is not set"); + } + return tempDirs.trim(); + } + + static String getIOManagerImplClass(Map params) { + String implClass = params.get(JNI_IO_MANAGER_IMPL_CLASS); + return implClass == null || implClass.trim().isEmpty() ? null : implClass.trim(); + } + + static IOManager createIOManager(String tempDirs) throws IOException { + return createIOManager(tempDirs, null); + } + + static IOManager createIOManager(String tempDirs, String implClassName) throws IOException { + String[] splitDirs = IOManagerImpl.splitPaths(tempDirs); + if (splitDirs.length == 0) { + throw new IOException("Paimon JNI IOManager temp dirs are empty"); + } + for (String splitDir : splitDirs) { + Files.createDirectories(Paths.get(splitDir)); + } + if (implClassName == null) { + return IOManager.create(splitDirs); + } + return createCustomIOManager(implClassName, splitDirs, tempDirs); + } + + private static IOManager createCustomIOManager(String implClassName, String[] splitDirs, String tempDirs) + throws IOException { + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + if (loader == null) { + loader = PaimonJniScanner.class.getClassLoader(); + } + try { + Class implClass = Class.forName(implClassName, true, loader); + if (!IOManager.class.isAssignableFrom(implClass)) { + throw new IOException("Paimon JNI IOManager implementation " + implClassName + + " does not implement " + IOManager.class.getName()); + } + return (IOManager) instantiateCustomIOManager(implClass, splitDirs, tempDirs); + } catch (ClassNotFoundException e) { + throw new IOException("Failed to find Paimon JNI IOManager implementation: " + implClassName, e); + } catch (ReflectiveOperationException e) { + throw new IOException("Failed to create Paimon JNI IOManager implementation: " + implClassName, e); + } + } + + private static Object instantiateCustomIOManager(Class implClass, String[] splitDirs, String tempDirs) + throws ReflectiveOperationException { + try { + Constructor constructor = implClass.getConstructor(String[].class); + return constructor.newInstance((Object) splitDirs); + } catch (NoSuchMethodException e) { + try { + Constructor constructor = implClass.getConstructor(String.class); + return constructor.newInstance(tempDirs); + } catch (NoSuchMethodException stringConstructorMissing) { + Constructor constructor = implClass.getConstructor(); + return constructor.newInstance(); + } + } catch (InvocationTargetException e) { + throw e; + } + } + private int[] getProjected() { return Arrays.stream(fields).mapToInt(paimonAllFieldNames::indexOf).toArray(); } @@ -159,8 +262,32 @@ private void resetDatetimeV2Precision() { @Override public void close() throws IOException { + IOException exception = null; if (reader != null) { - reader.close(); + try { + reader.close(); + } catch (IOException e) { + exception = e; + } finally { + reader = null; + } + } + if (ioManager != null) { + try { + ioManager.close(); + } catch (Exception e) { + LOG.warn("Failed to close Paimon JNI IOManager, temp dirs: {}", ioManagerTempDirs, e); + if (exception == null) { + exception = new IOException(e); + } else { + exception.addSuppressed(e); + } + } finally { + ioManager = null; + } + } + if (exception != null) { + throw exception; } } @@ -217,6 +344,13 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { return null; } + @Override + public Map getStatistics() { + Map statistics = new HashMap<>(); + statistics.put("counter:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); + return statistics; + } + private void initTable() { Preconditions.checkState(params.containsKey("serialized_table")); table = PaimonUtils.deserialize(params.get("serialized_table")); @@ -227,4 +361,3 @@ private void initTable() { } } - diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java new file mode 100644 index 00000000000000..1355d5f9844d9c --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.disk.BufferFileReader; +import org.apache.paimon.disk.BufferFileWriter; +import org.apache.paimon.disk.FileIOChannel; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +public class PaimonJniScannerTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testConstructorAcceptsEmptyProjection() { + new PaimonJniScanner(128, createBaseParams()); + } + + @Test + public void testIOManagerOptionHelpers() throws Exception { + Map params = createBaseParams(); + Assert.assertFalse(PaimonJniScanner.isIOManagerEnabled(params)); + + params.put(PaimonJniScanner.ENABLE_JNI_IO_MANAGER, "true"); + File tempDir = new File(temporaryFolder.getRoot(), "paimon-io-manager"); + params.put(PaimonJniScanner.JNI_IO_MANAGER_TMP_DIR, tempDir.getAbsolutePath()); + + Assert.assertTrue(PaimonJniScanner.isIOManagerEnabled(params)); + Assert.assertEquals(tempDir.getAbsolutePath(), PaimonJniScanner.getIOManagerTempDirs(params)); + Assert.assertNull(PaimonJniScanner.getIOManagerImplClass(params)); + PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()).close(); + Assert.assertTrue(tempDir.exists()); + } + + @Test + public void testCreateDefaultAndCustomIOManager() throws Exception { + File tempDir = new File(temporaryFolder.getRoot(), "paimon-io-manager-impl"); + IOManager defaultIOManager = PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()); + Assert.assertTrue(defaultIOManager instanceof IOManagerImpl); + defaultIOManager.close(); + + Map params = createBaseParams(); + params.put(PaimonJniScanner.JNI_IO_MANAGER_IMPL_CLASS, TestIOManager.class.getName()); + Assert.assertEquals(TestIOManager.class.getName(), PaimonJniScanner.getIOManagerImplClass(params)); + IOManager customIOManager = PaimonJniScanner.createIOManager( + tempDir.getAbsolutePath(), PaimonJniScanner.getIOManagerImplClass(params)); + Assert.assertTrue(customIOManager instanceof TestIOManager); + Assert.assertArrayEquals(new String[] {tempDir.getAbsolutePath()}, customIOManager.tempDirs()); + } + + @Test + public void testCloseCleansIOManagerTempDirectory() throws Exception { + File tempDir = temporaryFolder.newFolder("paimon-io-manager-clean"); + IOManager ioManager = PaimonJniScanner.createIOManager(tempDir.getAbsolutePath()); + FileIOChannel.ID channel = ioManager.createChannel(); + File spillFile = channel.getPathFile(); + Assert.assertTrue(spillFile.createNewFile()); + File spillDir = spillFile.getParentFile(); + Assert.assertTrue(spillDir.exists()); + + PaimonJniScanner scanner = new PaimonJniScanner(128, createBaseParams()); + Field ioManagerField = PaimonJniScanner.class.getDeclaredField("ioManager"); + ioManagerField.setAccessible(true); + ioManagerField.set(scanner, ioManager); + Assert.assertEquals("1", scanner.getStatistics().get("counter:PaimonJniIOManagerEnabled")); + + scanner.close(); + Assert.assertFalse(spillDir.exists()); + } + + private Map createBaseParams() { + Map params = new HashMap<>(); + params.put("required_fields", ""); + params.put("columns_types", ""); + params.put("paimon_split", ""); + params.put("paimon_predicate", ""); + return params; + } + + public static class TestIOManager implements IOManager { + private final String[] tempDirs; + + public TestIOManager(String[] tempDirs) { + this.tempDirs = tempDirs; + } + + @Override + public FileIOChannel.ID createChannel() { + throw new UnsupportedOperationException(); + } + + @Override + public FileIOChannel.ID createChannel(String channelName) { + throw new UnsupportedOperationException(); + } + + @Override + public String[] tempDirs() { + return tempDirs; + } + + @Override + public FileIOChannel.Enumerator createChannelEnumerator() { + throw new UnsupportedOperationException(); + } + + @Override + public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channel) { + throw new UnsupportedOperationException(); + } + + @Override + public BufferFileReader createBufferFileReader(FileIOChannel.ID channel) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 00359b5a11c90d..9c18c2aeeb7eaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.paimon.PaimonUtils; import org.apache.doris.datasource.paimon.profile.PaimonMetricRegistry; import org.apache.doris.datasource.paimon.profile.PaimonScanMetricsReporter; +import org.apache.doris.datasource.property.metastore.PaimonJdbcMetaStoreProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -65,6 +66,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -89,6 +91,14 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_START_TIMESTAMP = "startTimestamp"; private static final String DORIS_END_TIMESTAMP = "endTimestamp"; private static final String DORIS_INCREMENTAL_BETWEEN_SCAN_MODE = "incrementalBetweenScanMode"; + private static final String PAIMON_PROPERTY_PREFIX = "paimon."; + private static final String DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; + private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; + private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "doris.jni_io_manager.impl_class"; + private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( + DORIS_ENABLE_JNI_IO_MANAGER, + DORIS_JNI_IO_MANAGER_TMP_DIR, + DORIS_JNI_IO_MANAGER_IMPL_CLASS); private enum SplitReadType { JNI, @@ -142,6 +152,7 @@ public String toString() { // get them in doInitialize() to ensure internal consistency of ScanNode private Map storagePropertiesMap; private Map backendStorageProperties; + private Map backendPaimonOptions = Collections.emptyMap(); // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); @@ -169,6 +180,7 @@ protected void doInitialize() throws UserException { source.getPaimonTable() ); backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + backendPaimonOptions = getBackendPaimonOptions(); } @VisibleForTesting @@ -201,6 +213,13 @@ public void createScanRangeLocations() throws UserException { // Set paimon_predicate at ScanNode level to avoid redundant serialization in each split String serializedPredicate = PaimonUtil.encodeObjectToString(predicates); params.setPaimonPredicate(serializedPredicate); + setScanLevelPaimonOptions(); + } + + private void setScanLevelPaimonOptions() { + if (!backendPaimonOptions.isEmpty()) { + params.setPaimonOptions(backendPaimonOptions); + } } private void putHistorySchemaInfo(Long schemaId) { @@ -428,6 +447,35 @@ public List getSplits(int numBackends) throws UserException { return splits; } + @VisibleForTesting + Map getBackendPaimonOptions() { + if (source == null) { + return Collections.emptyMap(); + } + if (!(source.getCatalog() instanceof PaimonExternalCatalog)) { + return Collections.emptyMap(); + } + PaimonExternalCatalog catalog = (PaimonExternalCatalog) source.getCatalog(); + Map backendOptions = new HashMap<>(); + Map catalogProperties = catalog.getCatalogProperty().getProperties(); + if (catalogProperties == null) { + catalogProperties = Collections.emptyMap(); + } + for (String option : BACKEND_PAIMON_OPTIONS) { + String catalogProperty = PAIMON_PROPERTY_PREFIX + option; + if (catalogProperties.containsKey(catalogProperty)) { + backendOptions.put(option, catalogProperties.get(catalogProperty)); + } + } + if (!(catalog.getCatalogProperty().getMetastoreProperties() instanceof PaimonJdbcMetaStoreProperties)) { + return backendOptions; + } + PaimonJdbcMetaStoreProperties jdbcMetaStoreProperties = + (PaimonJdbcMetaStoreProperties) catalog.getCatalogProperty().getMetastoreProperties(); + backendOptions.putAll(jdbcMetaStoreProperties.getBackendPaimonOptions()); + return backendOptions; + } + private long determineTargetFileSplitSize(List dataSplits, boolean isBatchMode) { if (sessionVariable.getFileSplitSize() > 0) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 454f2d42d9e9f3..0ddf49b16dbea9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -21,9 +21,12 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.FileSplitter; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; @@ -408,6 +411,35 @@ public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Excep Assert.assertEquals(100L * 1024L * 1024L, target); } + @Test + public void testGetBackendPaimonOptionsForJniIOManager() { + Map props = new HashMap<>(); + props.put("paimon.doris.enable_jni_io_manager", "true"); + props.put("paimon.doris.jni_io_manager.tmp_dir", "/tmp/doris-paimon"); + props.put("paimon.doris.jni_io_manager.impl_class", "org.example.CustomIOManager"); + + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(catalogProperty.getProperties()).thenReturn(props); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(Mockito.mock(MetastoreProperties.class)); + + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getCatalog()).thenReturn(catalog); + + PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), + new TupleDescriptor(new TupleId(0)), false, sv, ScanContext.EMPTY); + node.setSource(source); + + Map backendOptions = node.getBackendPaimonOptions(); + Assert.assertEquals("true", backendOptions.get("doris.enable_jni_io_manager")); + Assert.assertEquals("/tmp/doris-paimon", backendOptions.get("doris.jni_io_manager.tmp_dir")); + Assert.assertEquals("org.example.CustomIOManager", + backendOptions.get("doris.jni_io_manager.impl_class")); + Assert.assertEquals(3, backendOptions.size()); + } + private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } From 453a4b8f1df16c4f1935ee122ece6cb8f45ff4f1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 11:56:07 +0800 Subject: [PATCH 04/18] [improvement](paimon) Add Paimon JNI scanner observability Issue Number: None Related PR: None Problem Summary: Paimon JNI scans can create Paimon SDK internal async file readers, especially for primary-key merge reads over large ORC files. Doris scanner concurrency limits do not directly expose those internal async reader threads or Java heap pressure in the query profile. This change adds lightweight profile metrics from the Paimon JNI scanner for active scanner counts, async reader thread counts, JVM heap/non-heap usage, split and predicate diagnostic sizes, async threshold configuration visibility, and readBatch/open timing counters. None - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - Behavior changed: No - Does this need documentation: No (cherry picked from commit 961726d3b76f80572d6967b5e7a9142b0143045b) --- .../apache/doris/paimon/PaimonJniScanner.java | 232 ++++++++++++++++-- .../doris/paimon/PaimonJniScannerTest.java | 61 +++++ 2 files changed, 272 insertions(+), 21 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 4adce6d2594fb8..86feaec1545035 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -39,6 +39,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; @@ -46,16 +49,25 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; public class PaimonJniScanner extends JniScanner { private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class); private static final String HADOOP_OPTION_PREFIX = "hadoop."; + private static final String PAIMON_OPTION_PREFIX = "paimon."; + private static final String ASYNC_READER_THREAD_NAME_PREFIX = "paimon-reader-async-thread"; + private static final String FILE_READER_ASYNC_THRESHOLD = "file-reader-async-threshold"; static final String ENABLE_JNI_IO_MANAGER = "paimon.doris.enable_jni_io_manager"; static final String JNI_IO_MANAGER_TMP_DIR = "paimon.doris.jni_io_manager.tmp_dir"; static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.doris.jni_io_manager.impl_class"; + private static final AtomicInteger ACTIVE_SCANNERS = new AtomicInteger(); + private static final AtomicInteger PEAK_ACTIVE_SCANNERS = new AtomicInteger(); + private static final AtomicInteger PEAK_ASYNC_READER_THREADS = new AtomicInteger(); private final Map params; private final Map hadoopOptionParams; @@ -71,6 +83,12 @@ public class PaimonJniScanner extends JniScanner { private RecordReader.RecordIterator recordIterator = null; private final ClassLoader classLoader; private PreExecutionAuthenticator preExecutionAuthenticator; + private boolean scannerCounted; + private long openTimeNanos; + private long readBatchTimeNanos; + private long readBatchCalls; + private long emptyReadBatchCalls; + private long rowsRead; public PaimonJniScanner(int batchSize, Map params) { this.classLoader = this.getClass().getClassLoader(); @@ -98,6 +116,8 @@ public PaimonJniScanner(int batchSize, Map params) { @Override public void open() throws IOException { + markScannerOpenedForMetrics(); + long startTime = System.nanoTime(); try { // When the user does not specify hive-site.xml, Paimon will look for the file from the classpath: // org.apache.paimon.hive.HiveCatalog.createHiveConf: @@ -119,6 +139,8 @@ public void open() throws IOException { } LOG.warn("Failed to open paimon_scanner: " + e.getMessage(), e); throw new RuntimeException(e); + } finally { + openTimeNanos += System.nanoTime() - startTime; } } @@ -263,28 +285,32 @@ private void resetDatetimeV2Precision() { @Override public void close() throws IOException { IOException exception = null; - if (reader != null) { - try { - reader.close(); - } catch (IOException e) { - exception = e; - } finally { - reader = null; + try { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + exception = e; + } finally { + reader = null; + } } - } - if (ioManager != null) { - try { - ioManager.close(); - } catch (Exception e) { - LOG.warn("Failed to close Paimon JNI IOManager, temp dirs: {}", ioManagerTempDirs, e); - if (exception == null) { - exception = new IOException(e); - } else { - exception.addSuppressed(e); + if (ioManager != null) { + try { + ioManager.close(); + } catch (Exception e) { + LOG.warn("Failed to close Paimon JNI IOManager, temp dirs: {}", ioManagerTempDirs, e); + if (exception == null) { + exception = new IOException(e); + } else { + exception.addSuppressed(e); + } + } finally { + ioManager = null; } - } finally { - ioManager = null; } + } finally { + markScannerClosedForMetrics(); } if (exception != null) { throw exception; @@ -295,7 +321,7 @@ private int readAndProcessNextBatch() throws IOException { int rows = 0; try { if (recordIterator == null) { - recordIterator = reader.readBatch(); + recordIterator = readBatchWithMetrics(); } while (recordIterator != null) { @@ -310,15 +336,23 @@ private int readAndProcessNextBatch() throws IOException { appendData(i, columnValue); } if (rows >= batchSize) { + if (fields.length == 0) { + vectorTable.appendVirtualData(rows); + } appendDataTime += System.nanoTime() - startTime; + rowsRead += rows; return rows; } } appendDataTime += System.nanoTime() - startTime; recordIterator.releaseBatch(); - recordIterator = reader.readBatch(); + recordIterator = readBatchWithMetrics(); } + if (fields.length == 0 && rows > 0) { + vectorTable.appendVirtualData(rows); + } + rowsRead += rows; } catch (Exception e) { close(); LOG.warn("Failed to get the next batch of paimon. " @@ -329,6 +363,20 @@ private int readAndProcessNextBatch() throws IOException { return rows; } + private RecordReader.RecordIterator readBatchWithMetrics() throws IOException { + long startTime = System.nanoTime(); + try { + RecordReader.RecordIterator iterator = reader.readBatch(); + if (iterator == null) { + emptyReadBatchCalls++; + } + return iterator; + } finally { + readBatchCalls++; + readBatchTimeNanos += System.nanoTime() - startTime; + } + } + @Override protected int getNext() { try { @@ -348,9 +396,151 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { public Map getStatistics() { Map statistics = new HashMap<>(); statistics.put("counter:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); + statistics.put("counter:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get())); + statistics.put("counter:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get())); + statistics.put("counter:PaimonJniAsyncReaderThreadCount", + String.valueOf(currentAsyncReaderThreadCount())); + statistics.put("counter:PaimonJniAsyncReaderThreadPeakCount", + String.valueOf(PEAK_ASYNC_READER_THREADS.get())); + statistics.put("counter:PaimonJniRequiredFieldCount", String.valueOf(fields.length)); + statistics.put("counter:PaimonJniSplitEncodedLength", String.valueOf(lengthOfParam("paimon_split"))); + statistics.put("counter:PaimonJniPredicateEncodedLength", String.valueOf(lengthOfParam("paimon_predicate"))); + statistics.put("counter:PaimonJniAsyncThresholdConfigured", + hasPaimonOption(FILE_READER_ASYNC_THRESHOLD) ? "1" : "0"); + parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD)) + .ifPresent(bytes -> statistics.put("bytes:PaimonJniAsyncThresholdBytes", String.valueOf(bytes))); + statistics.put("counter:PaimonJniReadBatchCalls", String.valueOf(readBatchCalls)); + statistics.put("counter:PaimonJniEmptyReadBatchCalls", String.valueOf(emptyReadBatchCalls)); + statistics.put("counter:PaimonJniRowsRead", String.valueOf(rowsRead)); + statistics.put("timer:PaimonJniScannerOpenTime", String.valueOf(openTimeNanos)); + statistics.put("timer:PaimonJniReadBatchTime", String.valueOf(readBatchTimeNanos)); + putMemoryStatistics(statistics); return statistics; } + private int lengthOfParam(String key) { + String value = params.get(key); + return value == null ? 0 : value.length(); + } + + private boolean hasPaimonOption(String key) { + return paimonOption(key) != null; + } + + private String paimonOption(String key) { + return params.get(PAIMON_OPTION_PREFIX + key); + } + + private static void putMemoryStatistics(Map statistics) { + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage(); + MemoryUsage nonHeapUsage = memoryMXBean.getNonHeapMemoryUsage(); + statistics.put("bytes:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed()))); + statistics.put("bytes:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted()))); + statistics.put("bytes:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax()))); + statistics.put("bytes:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed()))); + statistics.put("bytes:PaimonJniJvmNonHeapCommitted", String.valueOf(nonNegative(nonHeapUsage.getCommitted()))); + statistics.put("bytes:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax()))); + } + + private static long nonNegative(long value) { + return Math.max(value, 0L); + } + + private static int currentAsyncReaderThreadCount() { + int currentCount = countThreadsByNamePrefix(ASYNC_READER_THREAD_NAME_PREFIX); + updatePeak(PEAK_ASYNC_READER_THREADS, currentCount); + return currentCount; + } + + static int countThreadsByNamePrefix(String threadNamePrefix) { + int count = 0; + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (thread.getName().startsWith(threadNamePrefix)) { + count++; + } + } + return count; + } + + private void markScannerOpenedForMetrics() { + if (!scannerCounted) { + scannerCounted = true; + int currentCount = ACTIVE_SCANNERS.incrementAndGet(); + updatePeak(PEAK_ACTIVE_SCANNERS, currentCount); + } + } + + private void markScannerClosedForMetrics() { + if (scannerCounted) { + scannerCounted = false; + ACTIVE_SCANNERS.decrementAndGet(); + } + } + + private static void updatePeak(AtomicInteger peak, int value) { + int previous; + do { + previous = peak.get(); + if (value <= previous) { + return; + } + } while (!peak.compareAndSet(previous, value)); + } + + static Optional parseDataSizeBytes(String value) { + if (value == null || value.trim().isEmpty()) { + return Optional.empty(); + } + String normalized = value.trim().toLowerCase(Locale.ROOT).replace("_", "").replace(" ", ""); + int unitStart = 0; + while (unitStart < normalized.length() + && (Character.isDigit(normalized.charAt(unitStart)) || normalized.charAt(unitStart) == '.')) { + unitStart++; + } + if (unitStart == 0) { + return Optional.empty(); + } + try { + double number = Double.parseDouble(normalized.substring(0, unitStart)); + String unit = normalized.substring(unitStart); + long multiplier; + switch (unit) { + case "": + case "b": + case "byte": + case "bytes": + multiplier = 1L; + break; + case "k": + case "kb": + case "kib": + multiplier = 1024L; + break; + case "m": + case "mb": + case "mib": + multiplier = 1024L * 1024L; + break; + case "g": + case "gb": + case "gib": + multiplier = 1024L * 1024L * 1024L; + break; + case "t": + case "tb": + case "tib": + multiplier = 1024L * 1024L * 1024L * 1024L; + break; + default: + return Optional.empty(); + } + return Optional.of((long) (number * multiplier)); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } + private void initTable() { Preconditions.checkState(params.containsKey("serialized_table")); table = PaimonUtils.deserialize(params.get("serialized_table")); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 1355d5f9844d9c..24461b14b541bf 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -31,6 +31,8 @@ import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; public class PaimonJniScannerTest { @Rule @@ -93,6 +95,65 @@ public void testCloseCleansIOManagerTempDirectory() throws Exception { Assert.assertFalse(spillDir.exists()); } + @Test + public void testStatisticsIncludePaimonDiagnostics() { + Map params = createBaseParams(); + params.put("paimon_split", "encoded-split"); + params.put("paimon_predicate", "encoded-predicate"); + params.put("paimon.file-reader-async-threshold", "10 MiB"); + PaimonJniScanner scanner = new PaimonJniScanner(128, params); + + Map statistics = scanner.getStatistics(); + + Assert.assertEquals("0", statistics.get("counter:PaimonJniIOManagerEnabled")); + Assert.assertEquals("0", statistics.get("counter:PaimonJniRequiredFieldCount")); + Assert.assertEquals("13", statistics.get("counter:PaimonJniSplitEncodedLength")); + Assert.assertEquals("17", statistics.get("counter:PaimonJniPredicateEncodedLength")); + Assert.assertEquals("1", statistics.get("counter:PaimonJniAsyncThresholdConfigured")); + Assert.assertEquals(String.valueOf(10L * 1024L * 1024L), + statistics.get("bytes:PaimonJniAsyncThresholdBytes")); + Assert.assertTrue(statistics.containsKey("counter:PaimonJniAsyncReaderThreadCount")); + Assert.assertTrue(statistics.containsKey("counter:PaimonJniActiveScannerCount")); + Assert.assertTrue(statistics.containsKey("counter:PaimonJniReadBatchCalls")); + Assert.assertTrue(statistics.containsKey("timer:PaimonJniScannerOpenTime")); + Assert.assertTrue(statistics.containsKey("timer:PaimonJniReadBatchTime")); + Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapUsed")) > 0); + Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapCommitted")) > 0); + } + + @Test + public void testCountThreadsByNamePrefix() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Thread thread = new Thread(() -> { + started.countDown(); + try { + release.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }, "paimon-reader-async-thread-test"); + + thread.start(); + try { + Assert.assertTrue(started.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(PaimonJniScanner.countThreadsByNamePrefix("paimon-reader-async-thread") >= 1); + } finally { + release.countDown(); + thread.join(5000); + } + } + + @Test + public void testParseDataSizeBytes() { + Assert.assertEquals(Long.valueOf(1024L), PaimonJniScanner.parseDataSizeBytes("1 KiB").get()); + Assert.assertEquals(Long.valueOf(10L * 1024L * 1024L), + PaimonJniScanner.parseDataSizeBytes("10 MiB").get()); + Assert.assertEquals(Long.valueOf(2L * 1024L * 1024L * 1024L), + PaimonJniScanner.parseDataSizeBytes("2GB").get()); + Assert.assertFalse(PaimonJniScanner.parseDataSizeBytes("unknown").isPresent()); + } + private Map createBaseParams() { Map params = new HashMap<>(); params.put("required_fields", ""); From 5517611e3558f761ce985baaf1155d84aab55590 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 12:49:22 +0800 Subject: [PATCH 05/18] [improvement](paimon) Surface Paimon JNI stats in v2 reader Issue Number: None Related PR: None Problem Summary: Paimon JNI scanner observability was only collected by the legacy JNI reader path. FileScannerV2 uses the format-v2 JNI table reader, which registered the Java getStatistics method but never called it before closing the scanner, so default v2 Paimon JNI scans missed the added profile counters. The async threshold diagnostic also read only prefixed JNI params, while normal Paimon catalog/table configuration is available from the deserialized table options. This change collects Java scanner statistics in the v2 JNI reader before scanner close and reads the async threshold from Paimon table options with the existing params fallback. None - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - python3 build-support/run_clang_format.py --clang-format-executable $(brew --prefix llvm@16)/bin/clang-format --style file --inplace false be/src/format_v2/jni/jni_table_reader.cpp be/src/format_v2/jni/jni_table_reader.h - git diff --check - ./run-be-ut.sh --run --filter="PaimonJniReaderTest*" (failed before running tests: missing thirdparty/installed/bin/protoc and Snappy) - Behavior changed: No - Does this need documentation: No (cherry picked from commit 61eb6bf6e8249556a2fd5086dd2d704e33eca3d5) --- .../apache/doris/paimon/PaimonJniScanner.java | 6 +++++ .../doris/paimon/PaimonJniScannerTest.java | 23 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 86feaec1545035..8ec643392b252c 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -428,6 +428,12 @@ private boolean hasPaimonOption(String key) { } private String paimonOption(String key) { + if (table != null) { + String tableOption = table.options().get(key); + if (tableOption != null) { + return tableOption; + } + } return params.get(PAIMON_OPTION_PREFIX + key); } diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 24461b14b541bf..1cbe2cae488978 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.disk.FileIOChannel; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; +import org.apache.paimon.table.Table; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -29,6 +30,8 @@ import java.io.File; import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -96,12 +99,12 @@ public void testCloseCleansIOManagerTempDirectory() throws Exception { } @Test - public void testStatisticsIncludePaimonDiagnostics() { + public void testStatisticsIncludePaimonDiagnostics() throws Exception { Map params = createBaseParams(); params.put("paimon_split", "encoded-split"); params.put("paimon_predicate", "encoded-predicate"); - params.put("paimon.file-reader-async-threshold", "10 MiB"); PaimonJniScanner scanner = new PaimonJniScanner(128, params); + setTableOptions(scanner, Collections.singletonMap("file-reader-async-threshold", "10 MiB")); Map statistics = scanner.getStatistics(); @@ -163,6 +166,22 @@ private Map createBaseParams() { return params; } + private void setTableOptions(PaimonJniScanner scanner, Map options) throws Exception { + Table table = (Table) Proxy.newProxyInstance( + Table.class.getClassLoader(), new Class[] {Table.class}, (proxy, method, args) -> { + if ("options".equals(method.getName())) { + return options; + } + if ("toString".equals(method.getName())) { + return "TestPaimonTable"; + } + throw new UnsupportedOperationException(method.getName()); + }); + Field tableField = PaimonJniScanner.class.getDeclaredField("table"); + tableField.setAccessible(true); + tableField.set(scanner, table); + } + public static class TestIOManager implements IOManager { private final String[] tempDirs; From 1e7a3f3dc74fe38db9b92c95e389d0c22a9856d0 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 13:16:18 +0800 Subject: [PATCH 06/18] [improvement](paimon) Fix Paimon JNI profile snapshot aggregation Issue Number: None Related PR: None Problem Summary: Paimon JNI scanner statistics include both scanner-local cumulative values and process-wide snapshot values such as active scanner counts, async reader thread counts, and JVM memory usage. Exporting all of them through the additive JNI profile counter path made multi-split scans multiply snapshot values by the number of collected splits. This change extends the JNI statistics profile collectors to support non-additive gauge and peak metric types, then reports process-wide snapshots as set/max metrics while keeping scanner-local rows, calls, encoded lengths, and timers additive. None - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - python3 build-support/run_clang_format.py --clang-format-executable $(brew --prefix llvm@16)/bin/clang-format --style file --inplace true be/src/format/jni/jni_reader.cpp be/src/format_v2/jni/jni_table_reader.cpp be/src/format_v2/jni/jni_table_reader.h - git diff --check - Behavior changed: No - Does this need documentation: No (cherry picked from commit a08d695bb10e5ec25ba835524926a9a5dbc9d1b1) --- be/src/vec/exec/jni_connector.cpp | 36 +++++++++++++++++-- .../apache/doris/paimon/PaimonJniScanner.java | 31 ++++++++-------- .../doris/paimon/PaimonJniScannerTest.java | 18 +++++----- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index 97bd476289f1f2..36e733f8b5cafb 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -867,6 +867,9 @@ void JniConnector::_collect_profile_before_close() { return; } + const auto update_peak = [](int64_t previous, int64_t current) { + return current > previous; + }; for (const auto& metric : statistics_result) { std::vector type_and_name = split(metric.first, ":"); if (type_and_name.size() != 2) { @@ -874,22 +877,49 @@ void JniConnector::_collect_profile_before_close() { << "'metricType:metricName'"; continue; } - long metric_value = std::stol(metric.second); + int64_t metric_value = std::stoll(metric.second); RuntimeProfile::Counter* scanner_counter; if (type_and_name[0] == "timer") { scanner_counter = ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); } else if (type_and_name[0] == "counter") { scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT, _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); } else if (type_and_name[0] == "bytes") { scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES, _connector_name.c_str()); + COUNTER_UPDATE(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_gauge") { + scanner_counter = + ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "gauge") { + scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT, + _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "bytes_gauge") { + scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES, + _connector_name.c_str()); + COUNTER_SET(scanner_counter, metric_value); + } else if (type_and_name[0] == "timer_peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::TIME_NS, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::UNIT, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); + } else if (type_and_name[0] == "bytes_peak") { + auto* scanner_peak_counter = _profile->add_conditition_counter( + type_and_name[1], TUnit::BYTES, update_peak, _connector_name.c_str()); + scanner_peak_counter->conditional_update(metric_value, metric_value); } else { - LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter or bytes"; + LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter, bytes, " + << "timer_gauge, gauge, bytes_gauge, timer_peak, peak or bytes_peak"; continue; } - COUNTER_UPDATE(scanner_counter, metric_value); } } } diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 8ec643392b252c..15802dd6a4b9ef 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -395,20 +395,20 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { @Override public Map getStatistics() { Map statistics = new HashMap<>(); - statistics.put("counter:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); - statistics.put("counter:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get())); - statistics.put("counter:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get())); - statistics.put("counter:PaimonJniAsyncReaderThreadCount", + statistics.put("gauge:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); + statistics.put("gauge:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get())); + statistics.put("peak:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get())); + statistics.put("gauge:PaimonJniAsyncReaderThreadCount", String.valueOf(currentAsyncReaderThreadCount())); - statistics.put("counter:PaimonJniAsyncReaderThreadPeakCount", + statistics.put("peak:PaimonJniAsyncReaderThreadPeakCount", String.valueOf(PEAK_ASYNC_READER_THREADS.get())); - statistics.put("counter:PaimonJniRequiredFieldCount", String.valueOf(fields.length)); + statistics.put("gauge:PaimonJniRequiredFieldCount", String.valueOf(fields.length)); statistics.put("counter:PaimonJniSplitEncodedLength", String.valueOf(lengthOfParam("paimon_split"))); statistics.put("counter:PaimonJniPredicateEncodedLength", String.valueOf(lengthOfParam("paimon_predicate"))); - statistics.put("counter:PaimonJniAsyncThresholdConfigured", + statistics.put("gauge:PaimonJniAsyncThresholdConfigured", hasPaimonOption(FILE_READER_ASYNC_THRESHOLD) ? "1" : "0"); - parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD)) - .ifPresent(bytes -> statistics.put("bytes:PaimonJniAsyncThresholdBytes", String.valueOf(bytes))); + parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD)).ifPresent( + bytes -> statistics.put("bytes_gauge:PaimonJniAsyncThresholdBytes", String.valueOf(bytes))); statistics.put("counter:PaimonJniReadBatchCalls", String.valueOf(readBatchCalls)); statistics.put("counter:PaimonJniEmptyReadBatchCalls", String.valueOf(emptyReadBatchCalls)); statistics.put("counter:PaimonJniRowsRead", String.valueOf(rowsRead)); @@ -441,12 +441,13 @@ private static void putMemoryStatistics(Map statistics) { MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage(); MemoryUsage nonHeapUsage = memoryMXBean.getNonHeapMemoryUsage(); - statistics.put("bytes:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed()))); - statistics.put("bytes:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted()))); - statistics.put("bytes:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax()))); - statistics.put("bytes:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed()))); - statistics.put("bytes:PaimonJniJvmNonHeapCommitted", String.valueOf(nonNegative(nonHeapUsage.getCommitted()))); - statistics.put("bytes:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax()))); + statistics.put("bytes_gauge:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed()))); + statistics.put("bytes_gauge:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted()))); + statistics.put("bytes_gauge:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapCommitted", + String.valueOf(nonNegative(nonHeapUsage.getCommitted()))); + statistics.put("bytes_gauge:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax()))); } private static long nonNegative(long value) { diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 1cbe2cae488978..49f8a69dc9ac2f 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -92,7 +92,7 @@ public void testCloseCleansIOManagerTempDirectory() throws Exception { Field ioManagerField = PaimonJniScanner.class.getDeclaredField("ioManager"); ioManagerField.setAccessible(true); ioManagerField.set(scanner, ioManager); - Assert.assertEquals("1", scanner.getStatistics().get("counter:PaimonJniIOManagerEnabled")); + Assert.assertEquals("1", scanner.getStatistics().get("gauge:PaimonJniIOManagerEnabled")); scanner.close(); Assert.assertFalse(spillDir.exists()); @@ -108,20 +108,20 @@ public void testStatisticsIncludePaimonDiagnostics() throws Exception { Map statistics = scanner.getStatistics(); - Assert.assertEquals("0", statistics.get("counter:PaimonJniIOManagerEnabled")); - Assert.assertEquals("0", statistics.get("counter:PaimonJniRequiredFieldCount")); + Assert.assertEquals("0", statistics.get("gauge:PaimonJniIOManagerEnabled")); + Assert.assertEquals("0", statistics.get("gauge:PaimonJniRequiredFieldCount")); Assert.assertEquals("13", statistics.get("counter:PaimonJniSplitEncodedLength")); Assert.assertEquals("17", statistics.get("counter:PaimonJniPredicateEncodedLength")); - Assert.assertEquals("1", statistics.get("counter:PaimonJniAsyncThresholdConfigured")); + Assert.assertEquals("1", statistics.get("gauge:PaimonJniAsyncThresholdConfigured")); Assert.assertEquals(String.valueOf(10L * 1024L * 1024L), - statistics.get("bytes:PaimonJniAsyncThresholdBytes")); - Assert.assertTrue(statistics.containsKey("counter:PaimonJniAsyncReaderThreadCount")); - Assert.assertTrue(statistics.containsKey("counter:PaimonJniActiveScannerCount")); + statistics.get("bytes_gauge:PaimonJniAsyncThresholdBytes")); + Assert.assertTrue(statistics.containsKey("gauge:PaimonJniAsyncReaderThreadCount")); + Assert.assertTrue(statistics.containsKey("gauge:PaimonJniActiveScannerCount")); Assert.assertTrue(statistics.containsKey("counter:PaimonJniReadBatchCalls")); Assert.assertTrue(statistics.containsKey("timer:PaimonJniScannerOpenTime")); Assert.assertTrue(statistics.containsKey("timer:PaimonJniReadBatchTime")); - Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapUsed")) > 0); - Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapCommitted")) > 0); + Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapUsed")) > 0); + Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapCommitted")) > 0); } @Test From 25fafbd63d368ca106cba1e106fbcd5c47831e04 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 13:40:36 +0800 Subject: [PATCH 07/18] [improvement](paimon) Avoid stack trace collection for async thread metrics ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Paimon JNI async reader thread diagnostics counted matching threads with Thread.getAllStackTraces(), which builds stack traces for every live JVM thread even though only thread names are needed. Since scanner statistics are collected on scanner close, multi-split scans can repeat that expensive JVM-wide stack trace collection. This change uses ThreadMXBean.getAllThreadIds() and getThreadInfo(ids, 0) to inspect thread names without collecting stack traces. ### Release note None ### Check List (For Author) - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - git diff --check - Behavior changed: No - Does this need documentation: No (cherry picked from commit c2b34885ea1b27a5dddcabddf2e6a5c05b889dd4) --- .../java/org/apache/doris/paimon/PaimonJniScanner.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 15802dd6a4b9ef..4bccf3ea1daa4f 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -42,6 +42,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; @@ -462,8 +464,10 @@ private static int currentAsyncReaderThreadCount() { static int countThreadsByNamePrefix(String threadNamePrefix) { int count = 0; - for (Thread thread : Thread.getAllStackTraces().keySet()) { - if (thread.getName().startsWith(threadNamePrefix)) { + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds(), 0); + for (ThreadInfo threadInfo : threadInfos) { + if (threadInfo != null && threadInfo.getThreadName().startsWith(threadNamePrefix)) { count++; } } From 81d8b205f51359358ed09c9bf7d212d4ddae4a89 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 8 Jul 2026 13:58:43 +0800 Subject: [PATCH 08/18] [improvement](paimon) Remove JVM lifetime peaks from scanner profile ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Paimon JNI active scanner and async reader thread peak values were tracked with static JVM-lifetime high-water marks. Even with non-additive profile aggregation, later query profiles could report stale BE-wide historical peaks as if they belonged to the current query. This change removes those JVM-lifetime peak values from the per-query scanner statistics and keeps the current active scanner and async reader thread counts as snapshot gauges. ### Release note None ### Check List (For Author) - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - git diff --check - Behavior changed: No - Does this need documentation: No (cherry picked from commit 94554218feb37352995b756464b5932dab89fd90) --- .../apache/doris/paimon/PaimonJniScanner.java | 22 ++----------------- .../doris/paimon/PaimonJniScannerTest.java | 2 ++ 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 4bccf3ea1daa4f..3dba563f853a6d 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -68,8 +68,6 @@ public class PaimonJniScanner extends JniScanner { static final String JNI_IO_MANAGER_TMP_DIR = "paimon.doris.jni_io_manager.tmp_dir"; static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.doris.jni_io_manager.impl_class"; private static final AtomicInteger ACTIVE_SCANNERS = new AtomicInteger(); - private static final AtomicInteger PEAK_ACTIVE_SCANNERS = new AtomicInteger(); - private static final AtomicInteger PEAK_ASYNC_READER_THREADS = new AtomicInteger(); private final Map params; private final Map hadoopOptionParams; @@ -399,11 +397,8 @@ public Map getStatistics() { Map statistics = new HashMap<>(); statistics.put("gauge:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0"); statistics.put("gauge:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get())); - statistics.put("peak:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get())); statistics.put("gauge:PaimonJniAsyncReaderThreadCount", String.valueOf(currentAsyncReaderThreadCount())); - statistics.put("peak:PaimonJniAsyncReaderThreadPeakCount", - String.valueOf(PEAK_ASYNC_READER_THREADS.get())); statistics.put("gauge:PaimonJniRequiredFieldCount", String.valueOf(fields.length)); statistics.put("counter:PaimonJniSplitEncodedLength", String.valueOf(lengthOfParam("paimon_split"))); statistics.put("counter:PaimonJniPredicateEncodedLength", String.valueOf(lengthOfParam("paimon_predicate"))); @@ -457,9 +452,7 @@ private static long nonNegative(long value) { } private static int currentAsyncReaderThreadCount() { - int currentCount = countThreadsByNamePrefix(ASYNC_READER_THREAD_NAME_PREFIX); - updatePeak(PEAK_ASYNC_READER_THREADS, currentCount); - return currentCount; + return countThreadsByNamePrefix(ASYNC_READER_THREAD_NAME_PREFIX); } static int countThreadsByNamePrefix(String threadNamePrefix) { @@ -477,8 +470,7 @@ static int countThreadsByNamePrefix(String threadNamePrefix) { private void markScannerOpenedForMetrics() { if (!scannerCounted) { scannerCounted = true; - int currentCount = ACTIVE_SCANNERS.incrementAndGet(); - updatePeak(PEAK_ACTIVE_SCANNERS, currentCount); + ACTIVE_SCANNERS.incrementAndGet(); } } @@ -489,16 +481,6 @@ private void markScannerClosedForMetrics() { } } - private static void updatePeak(AtomicInteger peak, int value) { - int previous; - do { - previous = peak.get(); - if (value <= previous) { - return; - } - } while (!peak.compareAndSet(previous, value)); - } - static Optional parseDataSizeBytes(String value) { if (value == null || value.trim().isEmpty()) { return Optional.empty(); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 49f8a69dc9ac2f..4344b89ec402a3 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -117,6 +117,8 @@ public void testStatisticsIncludePaimonDiagnostics() throws Exception { statistics.get("bytes_gauge:PaimonJniAsyncThresholdBytes")); Assert.assertTrue(statistics.containsKey("gauge:PaimonJniAsyncReaderThreadCount")); Assert.assertTrue(statistics.containsKey("gauge:PaimonJniActiveScannerCount")); + Assert.assertFalse(statistics.containsKey("peak:PaimonJniActiveScannerPeakCount")); + Assert.assertFalse(statistics.containsKey("peak:PaimonJniAsyncReaderThreadPeakCount")); Assert.assertTrue(statistics.containsKey("counter:PaimonJniReadBatchCalls")); Assert.assertTrue(statistics.containsKey("timer:PaimonJniScannerOpenTime")); Assert.assertTrue(statistics.containsKey("timer:PaimonJniReadBatchTime")); From 0cb1b72631483df6fc755176da6b78cc61597213 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 1 Jul 2026 15:42:12 +0800 Subject: [PATCH 09/18] [fix](fe) Preserve external table column name case Issue Number: None Related PR: None Problem Summary: Creating Iceberg or Paimon external tables with mixed-case partition columns could fail because Doris converted top-level external column names to lower case while building external schemas and partition specs. Reading external table schemas and partition metadata also normalized some Paimon and Iceberg column names to lower case, so SHOW CREATE and partition helpers could lose the original external column spelling. This change preserves the original top-level external field names when converting Doris columns to Iceberg/Paimon schemas, resolves partition and primary key names case-insensitively back to the external canonical names, and stops schema/partition parsing paths from lowercasing external column names. Fix Iceberg and Paimon external table column name casing for mixed-case partition columns. - Test: Unit Test - Maven focused FE test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl fe-core -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=CreateIcebergTableTest,PaimonMetadataOpsTest,IcebergUtilsTest#testParseSchemaPreservesNonLowercaseColumnNames,PaimonUtilTest#testParseSchemaPreservesNonLowercaseColumnNames - git diff --check - A broader focused run including two existing Mockito-based IcebergUtilsTest methods compiled successfully but those two methods failed locally because Mockito inline Byte Buddy could not self-attach to the Homebrew JDK 17 VM. - Behavior changed: Yes. Iceberg and Paimon external schemas, partition specs, and partition metadata now preserve external column name casing. - Does this need documentation: No (cherry picked from commit 374e8d3d05a5b02b2140a636d583877a7fee1dfb) --- .../iceberg/DorisTypeToIcebergType.java | 13 +++++- .../iceberg/IcebergMetadataOps.java | 4 +- .../datasource/iceberg/IcebergUtils.java | 26 +++++++----- .../paimon/PaimonExternalTable.java | 2 +- .../doris/datasource/paimon/PaimonUtil.java | 2 +- .../source/PaimonPredicateConverter.java | 15 +++++-- .../iceberg/CreateIcebergTableTest.java | 22 ++++++++++ .../datasource/iceberg/IcebergUtilsTest.java | 13 ++++++ .../datasource/paimon/PaimonUtilTest.java | 40 +++++++++++++++++++ 9 files changed, 119 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java index 56fa03120d9f37..de19d90728d606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java @@ -29,6 +29,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import java.util.Collections; import java.util.List; @@ -37,14 +38,21 @@ */ public class DorisTypeToIcebergType extends DorisTypeVisitor { private final StructType root; + private final List rootFieldNames; private int nextId = 0; public DorisTypeToIcebergType() { this.root = null; + this.rootFieldNames = Collections.emptyList(); } public DorisTypeToIcebergType(StructType root) { + this(root, Collections.emptyList()); + } + + public DorisTypeToIcebergType(StructType root, List rootFieldNames) { this.root = root; + this.rootFieldNames = rootFieldNames; // the root struct's fields use the first ids this.nextId = root.getFields().size(); } @@ -65,10 +73,11 @@ public Type struct(StructType struct, List types) { Type type = types.get(i); int id = isRoot ? i : getNextId(); + String fieldName = isRoot && !rootFieldNames.isEmpty() ? rootFieldNames.get(i) : field.getName(); if (field.getContainsNull()) { - newFields.add(Types.NestedField.optional(id, field.getName(), type, field.getComment())); + newFields.add(Types.NestedField.optional(id, fieldName, type, field.getComment())); } else { - newFields.add(Types.NestedField.required(id, field.getName(), type, field.getComment())); + newFields.add(Types.NestedField.required(id, fieldName, type, field.getComment())); } } return Types.StructType.of(newFields); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 545237cea27edf..3aa26f3572c46d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -353,8 +353,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx .map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull())) .collect(Collectors.toList()); StructType structType = new StructType(new ArrayList<>(collect)); - Type visit = - DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType)); + List rootFieldNames = columns.stream().map(Column::getName).collect(Collectors.toList()); + Type visit = DorisTypeVisitor.visit(structType, new DorisTypeToIcebergType(structType, rootFieldNames)); Schema schema = new Schema(visit.asNestedType().asStructType().fields()); Map properties = createTableInfo.getProperties(); properties.put(ExternalCatalog.DORIS_VERSION, ExternalCatalog.DORIS_VERSION_VALUE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 9895ceb7023ba0..222809ca6a41df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -133,7 +133,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -520,34 +519,38 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes PartitionSpec.Builder builder = PartitionSpec.builderFor(schema); for (Expr expr : partitionExprs) { if (expr instanceof SlotRef) { - builder.identity(((SlotRef) expr).getColumnName()); + builder.identity(getIcebergColumnName(schema, ((SlotRef) expr).getColumnName())); } else if (expr instanceof FunctionCallExpr) { String exprName = expr.getExprName(); List params = ((FunctionCallExpr) expr).getParams().exprs(); switch (exprName.toLowerCase()) { case "bucket": - builder.bucket(params.get(1).getExprName(), Integer.parseInt(params.get(0).getStringValue())); + builder.bucket( + getIcebergColumnName(schema, params.get(1).getExprName()), + Integer.parseInt(params.get(0).getStringValue())); break; case "year": case "years": - builder.year(params.get(0).getExprName()); + builder.year(getIcebergColumnName(schema, params.get(0).getExprName())); break; case "month": case "months": - builder.month(params.get(0).getExprName()); + builder.month(getIcebergColumnName(schema, params.get(0).getExprName())); break; case "date": case "day": case "days": - builder.day(params.get(0).getExprName()); + builder.day(getIcebergColumnName(schema, params.get(0).getExprName())); break; case "date_hour": case "hour": case "hours": - builder.hour(params.get(0).getExprName()); + builder.hour(getIcebergColumnName(schema, params.get(0).getExprName())); break; case "truncate": - builder.truncate(params.get(1).getExprName(), Integer.parseInt(params.get(0).getStringValue())); + builder.truncate( + getIcebergColumnName(schema, params.get(1).getExprName()), + Integer.parseInt(params.get(0).getStringValue())); break; default: throw new UserException("unsupported partition for " + exprName); @@ -557,6 +560,11 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes return builder.build(); } + private static String getIcebergColumnName(Schema schema, String columnName) { + Types.NestedField field = schema.caseInsensitiveFindField(columnName); + return field == null ? columnName : field.name(); + } + private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Type.PrimitiveType primitive, boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { switch (primitive.typeId()) { @@ -974,7 +982,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { - Column column = new Column(field.name().toLowerCase(Locale.ROOT), + Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, field.doc(), true, -1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 1782bae59b078d..1a99aad60f6827 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -330,7 +330,7 @@ public Optional initSchema(SchemaCacheKey key) { Set partitionColumnNames = Sets.newHashSet(tableSchema.partitionKeys()); List partitionColumns = Lists.newArrayList(); for (DataField field : columns) { - Column column = new Column(field.name().toLowerCase(), + Column column = new Column(field.name(), PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(), getCatalog().getEnableMappingTimestampTz()), true, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 08358a3da99c65..3239035f20dea5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -430,7 +430,7 @@ public static List parseSchema(RowType rowType, List primaryKeys boolean enableTimestampTzMapping) { List resSchema = Lists.newArrayListWithCapacity(rowType.getFields().size()); rowType.getFields().forEach(field -> { - resSchema.add(new Column(field.name().toLowerCase(), + resSchema.add(new Column(field.name(), PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping), primaryKeys.contains(field.name()), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java index 73a3c72ddcc7c3..ae45c2427184e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java @@ -46,7 +46,7 @@ public class PaimonPredicateConverter { public PaimonPredicateConverter(RowType rowType) { this.builder = new PredicateBuilder(rowType); - this.fieldNames = rowType.getFields().stream().map(f -> f.name().toLowerCase()).collect(Collectors.toList()); + this.fieldNames = rowType.getFields().stream().map(DataField::name).collect(Collectors.toList()); this.paimonFieldTypes = rowType.getFields().stream().map(DataField::type).collect(Collectors.toList()); } @@ -99,7 +99,7 @@ private Predicate doInPredicate(InPredicate predicate) { return null; } String colName = slotRef.getColumnName(); - int idx = fieldNames.indexOf(colName); + int idx = getFieldIndex(colName); DataType dataType = paimonFieldTypes.get(idx); List valueList = new ArrayList<>(); for (int i = 1; i < predicate.getChildren().size(); i++) { @@ -132,7 +132,7 @@ private Predicate binaryExprDesc(Expr dorisExpr) { return null; } String colName = slotRef.getColumnName(); - int idx = fieldNames.indexOf(colName); + int idx = getFieldIndex(colName); DataType dataType = paimonFieldTypes.get(idx); Object value = dataType.accept(new PaimonValueConverter(literalExpr)); if (value == null) { @@ -174,6 +174,15 @@ private Predicate binaryExprDesc(Expr dorisExpr) { } + private int getFieldIndex(String colName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(colName)) { + return i; + } + } + return fieldNames.indexOf(colName); + } + public static SlotRef convertDorisExprToSlotRef(Expr expr) { SlotRef slotRef = null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 3422100de0f475..0a23870c80f3fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -177,6 +177,28 @@ public void testPartition() throws UserException { Assert.assertEquals("b", table.properties().get("a")); } + @Test + public void testPartitionPreservesNonLowercaseColumnNames() throws UserException { + TableIdentifier tb = TableIdentifier.of(dbName, getTableName()); + String sql = "create table " + tb + " (" + + "data int, " + + "`PART` int, " + + "`mIxEd_COL` int" + + ") engine = iceberg " + + "partition by (`PART`, bucket(2, `mIxEd_COL`)) ()"; + createTable(sql); + Table table = ops.getCatalog().loadTable(tb); + Schema schema = table.schema(); + + Assert.assertEquals("PART", schema.columns().get(1).name()); + Assert.assertEquals("mIxEd_COL", schema.columns().get(2).name()); + PartitionSpec spec = PartitionSpec.builderFor(schema) + .identity("PART") + .bucket("mIxEd_COL", 2) + .build(); + Assert.assertEquals(spec, table.spec()); + } + public void createTable(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index e4ee0ec5e46bcd..3f4828efb51a7a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.catalog.Column; import org.apache.doris.common.UserException; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; @@ -98,6 +99,18 @@ private boolean getListAllTables(HiveCatalog hiveCatalog) throws IllegalAccessEx return declaredField.getBoolean(hiveCatalog); } + @Test + public void testParseSchemaPreservesNonLowercaseColumnNames() { + Schema schema = new Schema( + Types.NestedField.required(1, "mIxEd_COL", Types.IntegerType.get()), + Types.NestedField.required(2, "PART", Types.StringType.get())); + + List columns = IcebergUtils.parseSchema(schema, false, false); + + Assert.assertEquals("mIxEd_COL", columns.get(0).getName()); + Assert.assertEquals("PART", columns.get(1).getName()); + } + @Test public void testGetMatchingManifest() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index e06b7dee7bf43a..046b38e311bf5d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -17,13 +17,24 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.table.Table; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; public class PaimonUtilTest { @Test @@ -36,4 +47,33 @@ public void testSchemaForVarcharAndChar() { Assert.assertEquals(32, type1.getLength()); Assert.assertEquals(14, type2.getLength()); } + + @Test + public void testParseSchemaPreservesNonLowercaseColumnNames() { + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(0, "mIxEd_COL", DataTypes.INT()), + DataTypes.FIELD(1, "PART", DataTypes.STRING())); + + List columns = PaimonUtil.parseSchema(rowType, Collections.singletonList("PART"), false, false); + + Assert.assertEquals("mIxEd_COL", columns.get(0).getName()); + Assert.assertEquals("PART", columns.get(1).getName()); + Assert.assertTrue(columns.get(1).isKey()); + } + + @Test + public void testGetPartitionInfoMapPreservesNonLowercaseKeys() { + DataField mixedCasePartition = DataTypes.FIELD(0, "Dt", DataTypes.STRING()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.name()).thenReturn("mock_table"); + Mockito.when(table.partitionKeys()).thenReturn(Collections.singletonList("Dt")); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW(mixedCasePartition)); + + BinaryRow partitionValues = BinaryRow.singleColumn(BinaryString.fromString("2026-05-26")); + + Map partitionInfoMap = PaimonUtil.getPartitionInfoMap(table, partitionValues, "UTC"); + + Assert.assertFalse(partitionInfoMap.containsKey("dt")); + Assert.assertEquals("2026-05-26", partitionInfoMap.get("Dt")); + } } From 36ef7590fcd75dc6a4207664d1935e6a187474ca Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 1 Jul 2026 16:48:12 +0800 Subject: [PATCH 10/18] [fix](fe) Resolve mixed-case external column references Issue Number: None Related PR: None Problem Summary: Paimon scan projection still matched Doris slot names against Paimon field names with lower-case or exact string comparisons, so mixed-case columns could be dropped from the FE projection or rejected by the JNI scanner as missing. Iceberg table creation also passed sort-order column names to the Iceberg builder without resolving them through the case-insensitive schema lookup, so ORDER BY clauses that used normalized column spelling could fail to bind to mixed-case Iceberg fields. This change resolves Paimon field indexes case-insensitively in both FE and JNI scan paths and resolves Iceberg sort-order names to the canonical schema field name before building the sort order. Fix Paimon scan projection and Iceberg sort-order handling for mixed-case external column names. - Test: Unit Test - Remote FE UT: ssh gabriel@10.26.20.3, /mnt/disk3/gabriel/Workspace/dev3/doris, MAVEN_ARGS=-o ./run-fe-ut.sh --run org.apache.doris.datasource.paimon.source.PaimonScanNodeTest#testGetPathPartitionKeysReturnsTablePartitionKeys+testSetPaimonParamsUsesOrderedPartitionKeys+testGetFieldIndexMatchesMixedCaseColumns - Maven focused Paimon JNI test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=PaimonJniScannerTest#testGetFieldIndexMatchesMixedCaseColumns - Maven focused Iceberg FE test: MAVEN_ARGS=-o JDK_17=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl fe-core -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=CreateIcebergTableTest#testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively - git diff --check - Behavior changed: Yes. Paimon scan projection and Iceberg sort-order creation now resolve mixed-case external column names case-insensitively while preserving canonical schema names. - Does this need documentation: No (cherry picked from commit ecc242b858e1a88a21b9a7bed0c4e392b57c33d7) --- .../apache/doris/paimon/PaimonJniScanner.java | 17 +++++- .../doris/paimon/PaimonJniScannerTest.java | 11 ++++ .../paimon/source/PaimonScanNode.java | 59 +++++++++++-------- .../iceberg/CreateIcebergTableTest.java | 17 ++++++ .../paimon/source/PaimonScanNodeTest.java | 9 +++ 5 files changed, 88 insertions(+), 25 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 3dba563f853a6d..cbb917805fdd31 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -247,7 +247,20 @@ private static Object instantiateCustomIOManager(Class implClass, String[] sp } private int[] getProjected() { - return Arrays.stream(fields).mapToInt(paimonAllFieldNames::indexOf).toArray(); + return Arrays.stream(fields).mapToInt(fieldName -> { + int index = getFieldIndex(paimonAllFieldNames, fieldName); + Preconditions.checkArgument(index >= 0, "RequiredField %s not found in schema", fieldName); + return index; + }).toArray(); + } + + static int getFieldIndex(List fieldNames, String fieldName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(fieldName)) { + return i; + } + } + return -1; } private List getPredicates() { @@ -271,7 +284,7 @@ private void resetDatetimeV2Precision() { if (types[i].isDateTimeV2()) { // paimon support precision > 6, but it has been reset as 6 in FE // try to get the right precision for datetimev2 - int index = paimonAllFieldNames.indexOf(fields[i]); + int index = getFieldIndex(paimonAllFieldNames, fields[i]); if (index != -1) { DataType dataType = table.rowType().getTypeAt(index); if (dataType instanceof TimestampType) { diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 4344b89ec402a3..c6c90389fe8af0 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -31,6 +31,7 @@ import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.Proxy; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -225,4 +226,14 @@ public BufferFileReader createBufferFileReader(FileIOChannel.ID channel) { public void close() { } } + + @Test + public void testGetFieldIndexMatchesMixedCaseColumns() { + Assert.assertEquals(1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "mixed_col")); + Assert.assertEquals(2, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "part")); + Assert.assertEquals(-1, PaimonJniScanner.getFieldIndex(Arrays.asList("data", "mIxEd_COL", "PART"), + "missing_col")); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 9c18c2aeeb7eaf..c2dd18b519ba52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -520,30 +520,43 @@ public Map getIncrReadParams() throws UserException { @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { - Table paimonTable = getProcessedTable(); - int[] projected = desc.getSlots().stream().mapToInt( - slot -> paimonTable.rowType() - .getFieldNames() - .stream() - .map(String::toLowerCase) - .collect(Collectors.toList()) - .indexOf(slot.getColumn().getName())) - .filter(i -> i >= 0) - .toArray(); - ReadBuilder readBuilder = paimonTable.newReadBuilder(); - TableScan scan = readBuilder.withFilter(predicates) - .withProjection(projected) - .newScan(); - PaimonMetricRegistry registry = new PaimonMetricRegistry(); - if (scan instanceof InnerTableScan) { - scan = ((InnerTableScan) scan).withMetricsRegistry(registry); - } - List splits = scan.plan().splits(); - PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); - if (!registry.getAllGroups().isEmpty()) { - registry.clear(); + long startTime = System.currentTimeMillis(); + try { + Table paimonTable = getProcessedTable(); + List fieldNames = paimonTable.rowType().getFieldNames(); + int[] projected = desc.getSlots().stream().mapToInt( + slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) + .filter(i -> i >= 0) + .toArray(); + ReadBuilder readBuilder = paimonTable.newReadBuilder(); + TableScan scan = readBuilder.withFilter(predicates) + .withProjection(projected) + .newScan(); + PaimonMetricRegistry registry = new PaimonMetricRegistry(); + if (scan instanceof InnerTableScan) { + scan = ((InnerTableScan) scan).withMetricsRegistry(registry); + } + List splits = scan.plan().splits(); + PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); + if (!registry.getAllGroups().isEmpty()) { + registry.clear(); + } + return splits; + } finally { + if (getSummaryProfile() != null) { + getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); + } } - return splits; + } + + @VisibleForTesting + static int getFieldIndex(List fieldNames, String columnName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(columnName)) { + return i; + } + } + return -1; } private String getFileFormat(String path) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 0a23870c80f3fe..4108ff423af149 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -199,6 +199,23 @@ public void testPartitionPreservesNonLowercaseColumnNames() throws UserException Assert.assertEquals(spec, table.spec()); } + @Test + public void testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively() throws UserException { + TableIdentifier tb = TableIdentifier.of(dbName, getTableName()); + String sql = "create table " + tb + " (" + + "data int, " + + "`mIxEd_COL` int" + + ") engine = iceberg " + + "order by (`mixed_col` asc)"; + createTable(sql); + Table table = ops.getCatalog().loadTable(tb); + Schema schema = table.schema(); + + Assert.assertEquals("mIxEd_COL", schema.columns().get(1).name()); + Assert.assertEquals(1, table.sortOrder().fields().size()); + Assert.assertEquals(schema.findField("mIxEd_COL").fieldId(), table.sortOrder().fields().get(0).sourceId()); + } + public void createTable(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 0ddf49b16dbea9..4b65f2a667e4a3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -46,6 +46,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -440,6 +441,14 @@ public void testGetBackendPaimonOptionsForJniIOManager() { Assert.assertEquals(3, backendOptions.size()); } + @Test + public void testGetFieldIndexMatchesMixedCaseColumns() { + List fieldNames = Arrays.asList("data", "mIxEd_COL", "PART"); + + Assert.assertEquals(1, PaimonScanNode.getFieldIndex(fieldNames, "mixed_col")); + Assert.assertEquals(2, PaimonScanNode.getFieldIndex(fieldNames, "part")); + Assert.assertEquals(-1, PaimonScanNode.getFieldIndex(fieldNames, "missing_col")); + } private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } From 7a69557c6bd8310c62b4e04edcffdefecda8b55a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 1 Jul 2026 20:29:05 +0800 Subject: [PATCH 11/18] [test](regression) Update Iceberg invalid Avro name output ### What problem does this PR solve? Issue Number: None Related PR: #65094 Problem Summary: Regenerated the Iceberg invalid Avro column name regression output after rebuilding FE and BE and rerunning the target external Iceberg case against the initialized REST catalog. ### Release note None ### Check List (For Author) - Test: Regression test - Ran test_iceberg_invaild_avro_name on the remote validation host with FE and BE rebuilt. - Behavior changed: No - Does this need documentation: No (cherry picked from commit 2310e84770a666392f8dc49e54ee1d8da9c340b5) --- .../external_table_p0/iceberg/test_iceberg_invaild_avro_name.out | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out index 1b8af743c6a0f2..6a56a30059fbc7 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out @@ -54,4 +54,3 @@ test:a1b2.raw.abc-gg-1-a text Yes true \N 3 row3 2 row2 1 row1 - From c1fe19a34463f2d3aea22c22d94d801b641eca69 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 1 Jul 2026 20:53:51 +0800 Subject: [PATCH 12/18] [test](regression) Regenerate Iceberg invalid Avro output ### What problem does this PR solve? Issue Number: None Related PR: #65094 Problem Summary: Regenerated the Iceberg invalid Avro column name regression output with a FE rebuilt from the PR changes. The expected DESC output now preserves the original mixed-case external column name. ### Release note None ### Check List (For Author) - Test: Regression test - Rebuilt FE on the remote validation host, started a temporary FE/BE cluster from the rebuilt output, and ran test_iceberg_invaild_avro_name against it. - Behavior changed: No - Does this need documentation: No (cherry picked from commit 25bc563b4e108f6d12f075087c95178c9ada4ff3) --- .../iceberg/test_iceberg_invaild_avro_name.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out index 6a56a30059fbc7..be0c0e2236450e 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- id int Yes true \N -test:a1b2.raw.abc-gg-1-a text Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 @@ -29,7 +29,7 @@ test:a1b2.raw.abc-gg-1-a text Yes true \N -- !desc -- id int Yes true \N -test:a1b2.raw.abc-gg-1-a text Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 From ed340ad6d57fa4ac301b81ea2486c4ba5a9d56fd Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 1 Jul 2026 21:29:48 +0800 Subject: [PATCH 13/18] [test](regression) Fix Paimon catalog regression expectations Issue Number: None Related PR: #65094 Problem Summary: The Paimon catalog regression expected the duplicate-column diagnostic to use a lower-case column name, but the FE now preserves the original external column case and reports the duplicated column as ID. The Paimon JDBC catalog regression also treated output from a failed optional docker probe as a container name, which caused a malformed docker cp command when the spark-iceberg container was unavailable or the current user lacked docker permission. Update the expected duplicate-column message and make optional command failures return an empty result so the existing spark-iceberg availability check can skip the environment-dependent JDBC portion correctly. None - Test: Regression test - On gabriel@10.26.20.3 under /mnt/disk3/gabriel/Workspace/dev3/doris, ran test_paimon_catalog against the rebuilt PR FE/BE with jdbcUrl pointing to 127.0.0.1:49230 and hive2HdfsPort=8320. - On gabriel@10.26.20.3 under /mnt/disk3/gabriel/Workspace/dev3/doris, ran test_paimon_jdbc_catalog against the rebuilt PR FE/BE with jdbcUrl pointing to 127.0.0.1:49230 and enableJdbcTest=true; the case detected docker permission denial and skipped the spark-iceberg-dependent section as intended. - Behavior changed: No - Does this need documentation: No (cherry picked from commit ce40d6b83e39f2fc57f5e9215b9e32fb06e5920b) --- .../iceberg/test_iceberg_invaild_avro_name.out | 8 ++++---- .../external_table_p0/paimon/test_paimon_catalog.groovy | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out index be0c0e2236450e..1e487ca780dd0c 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- -id int Yes true \N -TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N +id int Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 @@ -28,8 +28,8 @@ TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N 1 row1 -- !desc -- -id int Yes true \N -TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N +id int Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index b5ca34e5a18a48..91d38cf729438e 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -307,7 +307,7 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ test { sql """select * from dup_columns_table;""" - exception "Duplicate column name found: id" + exception "Duplicate column name found: ID" } sql """ set force_jni_scanner=false; """ @@ -332,4 +332,3 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ } } - From b902b30cf558385ede39e6d201fab07df0801870 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 11:02:32 +0800 Subject: [PATCH 14/18] [fix](ci) Fix backport formatting checks --- be/src/vec/exec/format/table/paimon_jni_reader.cpp | 11 ++++------- .../datasource/paimon/source/PaimonScanNodeTest.java | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp b/be/src/vec/exec/format/table/paimon_jni_reader.cpp index 6e3f7d14f4c6cd..c8cd4b309109f5 100644 --- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp +++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp @@ -21,8 +21,8 @@ #include #include -#include "runtime/exec_env.h" #include "runtime/descriptors.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "runtime/types.h" #include "util/string_util.h" @@ -83,17 +83,14 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d for (const auto& kv : range.table_format_params.paimon_params.paimon_options) { params[PAIMON_OPTION_PREFIX + kv.first] = kv.second; } - const std::string enable_io_manager_key = - PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; - const std::string io_manager_tmp_dir_key = - PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; + const std::string enable_io_manager_key = PAIMON_OPTION_PREFIX + DORIS_ENABLE_JNI_IO_MANAGER; + const std::string io_manager_tmp_dir_key = PAIMON_OPTION_PREFIX + DORIS_JNI_IO_MANAGER_TMP_DIR; auto enable_io_manager_it = params.find(enable_io_manager_key); if (enable_io_manager_it != params.end() && iequal(enable_io_manager_it->second, "true") && params.find(io_manager_tmp_dir_key) == params.end()) { std::vector tmp_dirs; for (const auto& store_path : state->exec_env()->store_paths()) { - tmp_dirs.push_back(store_path.path + "/" + - std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); + tmp_dirs.push_back(store_path.path + "/" + std::string(PAIMON_JNI_SCANNER_IO_TMP_DIR)); } DORIS_CHECK(!tmp_dirs.empty()); // Paimon's IOManager creates and later removes its own paimon-* child diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 4b65f2a667e4a3..713cf3e50b598c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -449,6 +449,7 @@ public void testGetFieldIndexMatchesMixedCaseColumns() { Assert.assertEquals(2, PaimonScanNode.getFieldIndex(fieldNames, "part")); Assert.assertEquals(-1, PaimonScanNode.getFieldIndex(fieldNames, "missing_col")); } + private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } From ca68c1b682cad99aba622ab711ae51e6c4c62955 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 11:25:54 +0800 Subject: [PATCH 15/18] [fix](paimon) Adapt backend JDBC options for branch-4.0 --- .../paimon/source/PaimonScanNode.java | 29 +++++++++++++++---- .../paimon/source/PaimonScanNodeTest.java | 25 ++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index c2dd18b519ba52..a09c075298875b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; @@ -35,7 +36,6 @@ import org.apache.doris.datasource.paimon.PaimonUtils; import org.apache.doris.datasource.paimon.profile.PaimonMetricRegistry; import org.apache.doris.datasource.paimon.profile.PaimonScanMetricsReporter; -import org.apache.doris.datasource.property.metastore.PaimonJdbcMetaStoreProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -95,6 +95,9 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "doris.jni_io_manager.impl_class"; + private static final String JDBC_PREFIX = "jdbc."; + private static final String JDBC_DRIVER_URL = JDBC_PREFIX + JdbcResource.DRIVER_URL; + private static final String JDBC_DRIVER_CLASS = JDBC_PREFIX + JdbcResource.DRIVER_CLASS; private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( DORIS_ENABLE_JNI_IO_MANAGER, DORIS_JNI_IO_MANAGER_TMP_DIR, @@ -467,15 +470,31 @@ Map getBackendPaimonOptions() { backendOptions.put(option, catalogProperties.get(catalogProperty)); } } - if (!(catalog.getCatalogProperty().getMetastoreProperties() instanceof PaimonJdbcMetaStoreProperties)) { + String driverUrl = getCatalogProperty(catalogProperties, JdbcResource.DRIVER_URL); + if (driverUrl == null) { return backendOptions; } - PaimonJdbcMetaStoreProperties jdbcMetaStoreProperties = - (PaimonJdbcMetaStoreProperties) catalog.getCatalogProperty().getMetastoreProperties(); - backendOptions.putAll(jdbcMetaStoreProperties.getBackendPaimonOptions()); + String driverClass = getCatalogProperty(catalogProperties, JdbcResource.DRIVER_CLASS); + if (driverClass == null) { + throw new IllegalArgumentException("jdbc.driver_class or paimon.jdbc.driver_class is required when " + + "jdbc.driver_url or paimon.jdbc.driver_url is specified"); + } + backendOptions.put(JDBC_DRIVER_URL, JdbcResource.getFullDriverUrl(driverUrl)); + backendOptions.put(JDBC_DRIVER_CLASS, driverClass); return backendOptions; } + private String getCatalogProperty(Map catalogProperties, String property) { + String value = catalogProperties.get(PAIMON_PROPERTY_PREFIX + JDBC_PREFIX + property); + if (value == null || value.trim().isEmpty()) { + value = catalogProperties.get(JDBC_PREFIX + property); + } + if (value == null || value.trim().isEmpty()) { + return null; + } + return value; + } + private long determineTargetFileSplitSize(List dataSplits, boolean isBatchMode) { if (sessionVariable.getFileSplitSize() > 0) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 713cf3e50b598c..32f7b710fed69f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -441,6 +441,31 @@ public void testGetBackendPaimonOptionsForJniIOManager() { Assert.assertEquals(3, backendOptions.size()); } + @Test + public void testGetBackendPaimonOptionsForJdbcDriver() { + Map props = new HashMap<>(); + props.put("paimon.jdbc.driver_url", "file:///tmp/postgresql.jar"); + props.put("paimon.jdbc.driver_class", "org.postgresql.Driver"); + + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(catalogProperty.getProperties()).thenReturn(props); + + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getCatalog()).thenReturn(catalog); + + PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), + new TupleDescriptor(new TupleId(0)), false, sv, ScanContext.EMPTY); + node.setSource(source); + + Map backendOptions = node.getBackendPaimonOptions(); + Assert.assertEquals("file:///tmp/postgresql.jar", backendOptions.get("jdbc.driver_url")); + Assert.assertEquals("org.postgresql.Driver", backendOptions.get("jdbc.driver_class")); + Assert.assertEquals(2, backendOptions.size()); + } + @Test public void testGetFieldIndexMatchesMixedCaseColumns() { List fieldNames = Arrays.asList("data", "mIxEd_COL", "PART"); From 4301c2102d36539154bd3013c8c670b2ffb1b5ae Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 11:50:39 +0800 Subject: [PATCH 16/18] [fix](paimon) Use branch-4.0 thrift options --- .../paimon/source/PaimonScanNode.java | 55 ++++++++----------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index a09c075298875b..5b24ddfccde1d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -216,13 +216,6 @@ public void createScanRangeLocations() throws UserException { // Set paimon_predicate at ScanNode level to avoid redundant serialization in each split String serializedPredicate = PaimonUtil.encodeObjectToString(predicates); params.setPaimonPredicate(serializedPredicate); - setScanLevelPaimonOptions(); - } - - private void setScanLevelPaimonOptions() { - if (!backendPaimonOptions.isEmpty()) { - params.setPaimonOptions(backendPaimonOptions); - } } private void putHistorySchemaInfo(Long schemaId) { @@ -281,6 +274,9 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) // MUST explicitly set to -1, to be distinct from valid row count >= 0 tableFormatFileDesc.setTableLevelRowCount(-1); } + if (!backendPaimonOptions.isEmpty()) { + fileDesc.setPaimonOptions(backendPaimonOptions); + } tableFormatFileDesc.setPaimonParams(fileDesc); Map partitionValues = paimonSplit.getPaimonPartitionValues(); if (partitionValues != null) { @@ -539,33 +535,26 @@ public Map getIncrReadParams() throws UserException { @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { - long startTime = System.currentTimeMillis(); - try { - Table paimonTable = getProcessedTable(); - List fieldNames = paimonTable.rowType().getFieldNames(); - int[] projected = desc.getSlots().stream().mapToInt( - slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) - .filter(i -> i >= 0) - .toArray(); - ReadBuilder readBuilder = paimonTable.newReadBuilder(); - TableScan scan = readBuilder.withFilter(predicates) - .withProjection(projected) - .newScan(); - PaimonMetricRegistry registry = new PaimonMetricRegistry(); - if (scan instanceof InnerTableScan) { - scan = ((InnerTableScan) scan).withMetricsRegistry(registry); - } - List splits = scan.plan().splits(); - PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); - if (!registry.getAllGroups().isEmpty()) { - registry.clear(); - } - return splits; - } finally { - if (getSummaryProfile() != null) { - getSummaryProfile().addExternalTableGetFileScanTasksTime(System.currentTimeMillis() - startTime); - } + Table paimonTable = getProcessedTable(); + List fieldNames = paimonTable.rowType().getFieldNames(); + int[] projected = desc.getSlots().stream().mapToInt( + slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) + .filter(i -> i >= 0) + .toArray(); + ReadBuilder readBuilder = paimonTable.newReadBuilder(); + TableScan scan = readBuilder.withFilter(predicates) + .withProjection(projected) + .newScan(); + PaimonMetricRegistry registry = new PaimonMetricRegistry(); + if (scan instanceof InnerTableScan) { + scan = ((InnerTableScan) scan).withMetricsRegistry(registry); + } + List splits = scan.plan().splits(); + PaimonScanMetricsReporter.report(source.getTargetTable(), paimonTable.name(), registry); + if (!registry.getAllGroups().isEmpty()) { + registry.clear(); } + return splits; } @VisibleForTesting From e694c9f891ad922d9100acfcd93f2c0254e00bde Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 22:54:44 +0800 Subject: [PATCH 17/18] [fix](ci) Adapt branch-4.0 backport tests --- .../paimon/PaimonSysTableJniScanner.java | 8 +- .../PaimonTableValuedFunction.java | 35 +- .../iceberg/CreateIcebergTableTest.java | 17 - .../paimon/source/PaimonScanNodeTest.java | 2 - .../test_iceberg_invaild_avro_name.out | 8 +- .../tvf/insert/test_insert_into_s3_tvf.groovy | 567 ------------------ 6 files changed, 37 insertions(+), 600 deletions(-) delete mode 100644 regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java index 6c2eab841294cb..90d309defcc7cd 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonSysTableJniScanner.java @@ -85,7 +85,11 @@ public PaimonSysTableJniScanner(int batchSize, Map params) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); } resetDatetimeV2Precision(); - this.projected = Arrays.stream(fields).mapToInt(paimonAllFieldNames::indexOf).toArray(); + this.projected = Arrays.stream(fields).mapToInt(fieldName -> { + int index = PaimonJniScanner.getFieldIndex(paimonAllFieldNames, fieldName); + Preconditions.checkArgument(index >= 0, "RequiredField %s not found in schema", fieldName); + return index; + }).toArray(); this.paimonDataTypeList = Arrays.stream(projected).mapToObj(i -> table.rowType().getTypeAt(i)) .collect(Collectors.toList()); this.paimonSplits = Arrays.stream(params.get("serialized_splits").split(",")) @@ -148,7 +152,7 @@ private void resetDatetimeV2Precision() { if (types[i].isDateTimeV2()) { // paimon support precision > 6, but it has been reset as 6 in FE // try to get the right precision for datetimev2 - int index = paimonAllFieldNames.indexOf(fields[i]); + int index = PaimonJniScanner.getFieldIndex(paimonAllFieldNames, fields[i]); if (index != -1) { DataType dataType = table.rowType().getTypeAt(index); if (dataType instanceof TimestampType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java index 525593cbd758cc..adca1635ef6673 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java @@ -167,17 +167,18 @@ public TMetadataType getMetadataType() { @Override public TMetaScanRange getMetaScanRange(List requiredFileds) { - int[] projections = requiredFileds.stream().mapToInt( - field -> paimonSysTable.rowType().getFieldNames() - .stream() - .map(String::toLowerCase) - .collect(Collectors.toList()) - .indexOf(field)) + List paimonFieldNames = paimonSysTable.rowType().getFieldNames(); + int[] projections = requiredFileds.stream() + .mapToInt(field -> getFieldIndex(paimonFieldNames, field)) .toArray(); List splits; try { - splits = hadoopAuthenticator.execute( - () -> paimonSysTable.newReadBuilder().withProjection(projections).newScan().plan().splits()); + splits = hadoopAuthenticator.execute(() -> { + if (hasInvalidProjection(projections)) { + return paimonSysTable.newReadBuilder().newScan().plan().splits(); + } + return paimonSysTable.newReadBuilder().withProjection(projections).newScan().plan().splits(); + }); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e)); } @@ -191,6 +192,24 @@ public TMetaScanRange getMetaScanRange(List requiredFileds) { return tMetaScanRange; } + private static boolean hasInvalidProjection(int[] projections) { + for (int projection : projections) { + if (projection < 0) { + return true; + } + } + return false; + } + + private static int getFieldIndex(List fieldNames, String fieldName) { + for (int i = 0; i < fieldNames.size(); i++) { + if (fieldNames.get(i).equalsIgnoreCase(fieldName)) { + return i; + } + } + return -1; + } + @Override public String getTableName() { return "PaimonTableValuedFunction<" + queryType + ">"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 4108ff423af149..0a23870c80f3fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -199,23 +199,6 @@ public void testPartitionPreservesNonLowercaseColumnNames() throws UserException Assert.assertEquals(spec, table.spec()); } - @Test - public void testSortOrderResolvesNonLowercaseColumnNamesCaseInsensitively() throws UserException { - TableIdentifier tb = TableIdentifier.of(dbName, getTableName()); - String sql = "create table " + tb + " (" - + "data int, " - + "`mIxEd_COL` int" - + ") engine = iceberg " - + "order by (`mixed_col` asc)"; - createTable(sql); - Table table = ops.getCatalog().loadTable(tb); - Schema schema = table.schema(); - - Assert.assertEquals("mIxEd_COL", schema.columns().get(1).name()); - Assert.assertEquals(1, table.sortOrder().fields().size()); - Assert.assertEquals(schema.findField("mIxEd_COL").fieldId(), table.sortOrder().fields().get(0).sourceId()); - } - public void createTable(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 32f7b710fed69f..45d8c8909a4f37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -26,7 +26,6 @@ import org.apache.doris.datasource.FileSplitter; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; -import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; @@ -421,7 +420,6 @@ public void testGetBackendPaimonOptionsForJniIOManager() { CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); Mockito.when(catalogProperty.getProperties()).thenReturn(props); - Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(Mockito.mock(MetastoreProperties.class)); PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out index 1e487ca780dd0c..be0c0e2236450e 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_invaild_avro_name.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- -id int Yes true \N -TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N +id int Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 @@ -28,8 +28,8 @@ TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N 1 row1 -- !desc -- -id int Yes true \N -TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N +id int Yes true \N +TEST:A1B2.RAW.ABC-GG-1-A text Yes true \N -- !q_1 -- 1 row1 diff --git a/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy b/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy deleted file mode 100644 index c6b00182852393..00000000000000 --- a/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_s3_tvf.groovy +++ /dev/null @@ -1,567 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_insert_into_s3_tvf", "p0,external") { - - String ak = getS3AK() - String sk = getS3SK() - String s3_endpoint = getS3Endpoint() - String region = getS3Region() - String bucket = context.config.otherConfigs.get("s3BucketName") - - if (ak == null || ak == "" || sk == null || sk == "") { - logger.info("S3 not configured, skip") - return - } - - def s3BasePrefix = "regression/insert_tvf_test/${UUID.randomUUID().toString()}" - def s3BasePath = "${bucket}/${s3BasePrefix}" - - // file_path is now a prefix; BE generates: {prefix}{query_id}_{idx}.{ext} - def s3WriteProps = { String path, String format -> - return """ - "file_path" = "s3://${s3BasePath}/${path}", - "format" = "${format}", - "s3.endpoint" = "${s3_endpoint}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "s3.region" = "${region}" - """ - } - - // Read uses wildcard to match generated file names - def s3ReadProps = { String path, String format -> - return """ - "uri" = "https://${bucket}.${s3_endpoint}/${s3BasePrefix}/${path}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "format" = "${format}", - "region" = "${region}" - """ - } - - // ============ Source tables ============ - - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_src """ - sql """ - CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_src ( - c_bool BOOLEAN, - c_tinyint TINYINT, - c_smallint SMALLINT, - c_int INT, - c_bigint BIGINT, - c_float FLOAT, - c_double DOUBLE, - c_decimal DECIMAL(10,2), - c_date DATE, - c_datetime DATETIME, - c_varchar VARCHAR(100), - c_string STRING - ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ - INSERT INTO test_insert_into_s3_tvf_src VALUES - (true, 1, 100, 1000, 100000, 1.1, 2.2, 123.45, '2024-01-01', '2024-01-01 10:00:00', 'hello', 'world'), - (false, 2, 200, 2000, 200000, 3.3, 4.4, 678.90, '2024-06-15', '2024-06-15 12:30:00', 'foo', 'bar'), - (true, 3, 300, 3000, 300000, 5.5, 6.6, 999.99, '2024-12-31', '2024-12-31 23:59:59', 'test', 'data'), - (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL), - (false, -1, -100, -1000, -100000, -1.1, -2.2, -123.45,'2020-02-29', '2020-02-29 00:00:00', '', 'special_chars'); - """ - - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_complex_src """ - sql """ - CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_complex_src ( - c_int INT, - c_array ARRAY, - c_map MAP, - c_struct STRUCT - ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ - INSERT INTO test_insert_into_s3_tvf_complex_src VALUES - (1, [1, 2, 3], {'a': 1, 'b': 2}, {1, 'hello'}), - (2, [4, 5], {'x': 10}, {2, 'world'}), - (3, [], {}, {3, ''}), - (4, NULL, NULL, NULL); - """ - - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_join_src """ - sql """ - CREATE TABLE IF NOT EXISTS test_insert_into_s3_tvf_join_src ( - c_int INT, - c_label STRING - ) DISTRIBUTED BY HASH(c_int) BUCKETS 1 - PROPERTIES("replication_num" = "1"); - """ - - sql """ INSERT INTO test_insert_into_s3_tvf_join_src VALUES (1000, 'label_a'), (2000, 'label_b'), (3000, 'label_c'); """ - - // ============ 1. S3 CSV basic types ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("basic_csv/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; - """ - - order_qt_s3_csv_basic_types """ - SELECT * FROM s3( - ${s3ReadProps("basic_csv/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 2. S3 Parquet basic types ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("basic_parquet/data_", "parquet")}, - "delete_existing_files" = "true" - ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; - """ - - order_qt_s3_parquet_basic_types """ - SELECT * FROM s3( - ${s3ReadProps("basic_parquet/*", "parquet")} - ) ORDER BY c_int; - """ - - // ============ 3. S3 ORC basic types ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("basic_orc/data_", "orc")}, - "delete_existing_files" = "true" - ) SELECT * FROM test_insert_into_s3_tvf_src ORDER BY c_int; - """ - - order_qt_s3_orc_basic_types """ - SELECT * FROM s3( - ${s3ReadProps("basic_orc/*", "orc")} - ) ORDER BY c_int; - """ - - // ============ 4. S3 Parquet complex types ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("complex_parquet/data_", "parquet")}, - "delete_existing_files" = "true" - ) SELECT * FROM test_insert_into_s3_tvf_complex_src ORDER BY c_int; - """ - - order_qt_s3_parquet_complex_types """ - SELECT * FROM s3( - ${s3ReadProps("complex_parquet/*", "parquet")} - ) ORDER BY c_int; - """ - - // ============ 5. S3 ORC complex types ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("complex_orc/data_", "orc")}, - "delete_existing_files" = "true" - ) SELECT * FROM test_insert_into_s3_tvf_complex_src ORDER BY c_int; - """ - - order_qt_s3_orc_complex_types """ - SELECT * FROM s3( - ${s3ReadProps("complex_orc/*", "orc")} - ) ORDER BY c_int; - """ - - // ============ 6. S3 CSV separator: comma ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("sep_comma/data_", "csv")}, - "column_separator" = ",", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_sep_comma """ - SELECT * FROM s3( - ${s3ReadProps("sep_comma/*", "csv")}, - "column_separator" = "," - ) ORDER BY c1; - """ - - // ============ 7. S3 CSV separator: tab ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("sep_tab/data_", "csv")}, - "column_separator" = "\t", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_sep_tab """ - SELECT * FROM s3( - ${s3ReadProps("sep_tab/*", "csv")}, - "column_separator" = "\t" - ) ORDER BY c1; - """ - - // ============ 8. S3 CSV separator: pipe ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("sep_pipe/data_", "csv")}, - "column_separator" = "|", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_sep_pipe """ - SELECT * FROM s3( - ${s3ReadProps("sep_pipe/*", "csv")}, - "column_separator" = "|" - ) ORDER BY c1; - """ - - // ============ 9. S3 CSV separator: multi-char ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("sep_multi/data_", "csv")}, - "column_separator" = ";;", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_sep_multi """ - SELECT * FROM s3( - ${s3ReadProps("sep_multi/*", "csv")}, - "column_separator" = ";;" - ) ORDER BY c1; - """ - - // ============ 10. S3 CSV line delimiter: CRLF ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("line_crlf/data_", "csv")}, - "line_delimiter" = "\r\n", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_line_crlf """ - SELECT * FROM s3( - ${s3ReadProps("line_crlf/*", "csv")}, - "line_delimiter" = "\r\n" - ) ORDER BY c1; - """ - - // ============ 11. S3 CSV compress: gz ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("compress_gz/data_", "csv")}, - "compression_type" = "gz", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_compress_gz """ - SELECT * FROM s3( - ${s3ReadProps("compress_gz/*", "csv")}, - "compress_type" = "gz" - ) ORDER BY c1; - """ - - // ============ 12. S3 CSV compress: zstd ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("compress_zstd/data_", "csv")}, - "compression_type" = "zstd", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_compress_zstd """ - SELECT * FROM s3( - ${s3ReadProps("compress_zstd/*", "csv")}, - "compress_type" = "zstd" - ) ORDER BY c1; - """ - - // ============ 13. S3 CSV compress: lz4 ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("compress_lz4/data_", "csv")}, - "compression_type" = "lz4block", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_compress_lz4 """ - SELECT * FROM s3( - ${s3ReadProps("compress_lz4/*", "csv")}, - "compress_type" = "lz4block" - ) ORDER BY c1; - """ - - // ============ 14. S3 CSV compress: snappy ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("compress_snappy/data_", "csv")}, - "compression_type" = "snappyblock", - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_csv_compress_snappy """ - SELECT * FROM s3( - ${s3ReadProps("compress_snappy/*", "csv")}, - "compress_type" = "snappyblock" - ) ORDER BY c1; - """ - - // ============ 15. S3 Overwrite mode ============ - // delete_existing_files=true is handled by FE for S3 - - // First write: 5 rows - sql """ - INSERT INTO s3( - ${s3WriteProps("overwrite/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src ORDER BY c_int; - """ - - order_qt_s3_overwrite_first """ - SELECT * FROM s3( - ${s3ReadProps("overwrite/*", "csv")} - ) ORDER BY c1; - """ - - // Second write: 2 rows with overwrite (FE deletes the directory first) - sql """ - INSERT INTO s3( - ${s3WriteProps("overwrite/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int > 0 ORDER BY c_int LIMIT 2; - """ - - order_qt_s3_overwrite_second """ - SELECT * FROM s3( - ${s3ReadProps("overwrite/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 16. S3 Append mode ============ - - // First write - sql """ - INSERT INTO s3( - ${s3WriteProps("append/data_", "parquet")}, - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 1000; - """ - - order_qt_s3_append_first """ - SELECT * FROM s3( - ${s3ReadProps("append/*", "parquet")} - ) ORDER BY c_int; - """ - - // Second write (append — different query_id produces different file name) - sql """ - INSERT INTO s3( - ${s3WriteProps("append/data_", "parquet")} - ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 2000; - """ - - order_qt_s3_append_second """ - SELECT * FROM s3( - ${s3ReadProps("append/*", "parquet")} - ) ORDER BY c_int; - """ - - // ============ 17. S3 Complex SELECT: constant expressions ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("const_expr/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT 1, 'hello', 3.14, CAST('2024-01-01' AS DATE); - """ - - order_qt_s3_const_expr """ - SELECT * FROM s3( - ${s3ReadProps("const_expr/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 18. S3 Complex SELECT: WHERE + GROUP BY ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("where_groupby/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT c_bool, COUNT(*), SUM(c_int) FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL GROUP BY c_bool ORDER BY c_bool; - """ - - order_qt_s3_where_groupby """ - SELECT * FROM s3( - ${s3ReadProps("where_groupby/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 19. S3 Complex SELECT: JOIN ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("join_query/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT a.c_int, a.c_varchar, b.c_label - FROM test_insert_into_s3_tvf_src a INNER JOIN test_insert_into_s3_tvf_join_src b ON a.c_int = b.c_int - ORDER BY a.c_int; - """ - - order_qt_s3_join_query """ - SELECT * FROM s3( - ${s3ReadProps("join_query/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 20. S3 Complex SELECT: subquery ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("subquery/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT * FROM (SELECT c_int, c_varchar, c_string FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int) sub; - """ - - order_qt_s3_subquery """ - SELECT * FROM s3( - ${s3ReadProps("subquery/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 21. S3 Complex SELECT: type cast ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("type_cast/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT CAST(c_int AS BIGINT), CAST(c_float AS DOUBLE), CAST(c_date AS STRING) - FROM test_insert_into_s3_tvf_src WHERE c_int IS NOT NULL ORDER BY c_int; - """ - - order_qt_s3_type_cast """ - SELECT * FROM s3( - ${s3ReadProps("type_cast/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 22. S3 Complex SELECT: UNION ALL ============ - - sql """ - INSERT INTO s3( - ${s3WriteProps("union_query/data_", "csv")}, - "delete_existing_files" = "true" - ) SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 1000 - UNION ALL - SELECT c_int, c_varchar FROM test_insert_into_s3_tvf_src WHERE c_int = 2000; - """ - - order_qt_s3_union_query """ - SELECT * FROM s3( - ${s3ReadProps("union_query/*", "csv")} - ) ORDER BY c1; - """ - - // ============ 23. Error: missing file_path ============ - - test { - sql """ - INSERT INTO s3( - "format" = "csv", - "s3.endpoint" = "${s3_endpoint}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "s3.region" = "${region}" - ) SELECT 1; - """ - exception "file_path" - } - - // ============ 24. Error: missing format ============ - - test { - sql """ - INSERT INTO s3( - "file_path" = "s3://${s3BasePath}/err/data_", - "s3.endpoint" = "${s3_endpoint}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "s3.region" = "${region}" - ) SELECT 1; - """ - exception "format" - } - - // ============ 25. Error: unsupported format ============ - - test { - sql """ - INSERT INTO s3( - "file_path" = "s3://${s3BasePath}/err/data_", - "format" = "json", - "s3.endpoint" = "${s3_endpoint}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "s3.region" = "${region}" - ) SELECT 1; - """ - exception "Unsupported" - } - - // ============ 26. Error: wildcard in file_path ============ - - test { - sql """ - INSERT INTO s3( - "file_path" = "s3://${s3BasePath}/wildcard/*.csv", - "format" = "csv", - "s3.endpoint" = "${s3_endpoint}", - "s3.access_key" = "${ak}", - "s3.secret_key" = "${sk}", - "s3.region" = "${region}" - ) SELECT 1; - """ - exception "wildcards" - } - - // ============ Cleanup ============ - - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_src """ - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_complex_src """ - sql """ DROP TABLE IF EXISTS test_insert_into_s3_tvf_join_src """ -} From ce8fa073bf2854e66ace74915ca07b6a05e273ea Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 9 Jul 2026 23:20:50 +0800 Subject: [PATCH 18/18] [fix](paimon) Handle empty JNI projections --- .../org/apache/doris/paimon/PaimonJniScanner.java | 14 ++++++++++++-- .../apache/doris/paimon/PaimonJniScannerTest.java | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index cbb917805fdd31..cfbed4969da853 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -96,8 +96,11 @@ public PaimonJniScanner(int batchSize, Map params) { LOG.debug("params:{}", params); } this.params = params; - String[] requiredFields = params.get("required_fields").split(","); - String[] requiredTypes = params.get("columns_types").split("#"); + String[] requiredFields = splitRequiredParam(params.get("required_fields"), ","); + String[] requiredTypes = splitRequiredParam(params.get("columns_types"), "#"); + Preconditions.checkArgument(requiredFields.length == requiredTypes.length, + "Required fields size %s is not matched with required types size %s", + requiredFields.length, requiredTypes.length); ColumnType[] columnTypes = new ColumnType[requiredTypes.length]; for (int i = 0; i < requiredTypes.length; i++) { columnTypes[i] = ColumnType.parseType(requiredFields[i], requiredTypes[i]); @@ -263,6 +266,13 @@ static int getFieldIndex(List fieldNames, String fieldName) { return -1; } + static String[] splitRequiredParam(String value, String delimiterRegex) { + if (value == null || value.isEmpty()) { + return new String[0]; + } + return value.split(delimiterRegex); + } + private List getPredicates() { List predicates = PaimonUtils.deserialize(paimonPredicate); if (LOG.isDebugEnabled()) { diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index c6c90389fe8af0..7ab07c8efcfd44 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -45,6 +45,10 @@ public class PaimonJniScannerTest { @Test public void testConstructorAcceptsEmptyProjection() { new PaimonJniScanner(128, createBaseParams()); + Assert.assertArrayEquals(new String[0], PaimonJniScanner.splitRequiredParam("", ",")); + Assert.assertArrayEquals(new String[0], PaimonJniScanner.splitRequiredParam("", "#")); + Assert.assertArrayEquals(new String[] {"id", "name"}, + PaimonJniScanner.splitRequiredParam("id,name", ",")); } @Test