[GLUTEN-4730][CH] feat: Support date_from_unix_date function: Add ClickHouse backend implementation

cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.cpp

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Functions/SparkFunctionDateFromUnixDate.h>
+
+namespace local_engine
+{
+
+REGISTER_FUNCTION(SparkFunctionDateFromUnixDate)
+{
+    factory.registerFunction<local_engine::SparkFunctionDateFromUnixDate>();
+}
+
+}

cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.h

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeDate32.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/IDataType.h>
+#include <Functions/FunctionFactory.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+}
+
+namespace local_engine
+{
+class SparkFunctionDateFromUnixDate : public DB::IFunction
+{
+public:
+    static constexpr auto name = "sparkDateFromUnixDate";
+    static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared<SparkFunctionDateFromUnixDate>(); }
+
+    SparkFunctionDateFromUnixDate() = default;
+    ~SparkFunctionDateFromUnixDate() override = default;
+
+    String getName() const override { return name; }
+    bool isSuitableForShortCircuitArgumentsExecution(const DB::DataTypesWithConstInfo &) const override { return true; }
+    size_t getNumberOfArguments() const override { return 1; }
+    bool isVariadic() const override { return false; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+    DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &) const override { return std::make_shared<DB::DataTypeDate32>(); }
+
+    DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows) const override
+    {
+        if (arguments.size() != 1)
+            throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly 1 argument", name);
+
+        const DB::ColumnWithTypeAndName & first_arg = arguments[0];
+        DB::WhichDataType which(first_arg.type);
+        if (which.isInt32())
+            return executeInternal<Int32>(first_arg.column, input_rows);
+        else
+            throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} requires Int32 argument", name);
+    }
+
+    template<typename T>
+    DB::ColumnPtr executeInternal(const DB::ColumnPtr & col, size_t input_rows) const
+    {
+        const DB::ColumnVector<T> * col_src = checkAndGetColumn<DB::ColumnVector<T>>(col.get());
+        DB::MutableColumnPtr res = DB::ColumnVector<Int32>::create(col->size());
+        DB::PaddedPODArray<Int32> & data = assert_cast<DB::ColumnVector<Int32> *>(res.get())->getData();
+
+        for (size_t i = 0; i < input_rows; ++i)
+        {
+            // Spark's date_from_unix_date returns a Date that is the given number of days since 1970-01-01.
+            // CH Date32 is also stored as days since 1970-01-01, so the value passes through unchanged.
+            const T unix_date = col_src->getElement(i);
+            data[i] = static_cast<Int32>(unix_date);
+        }
+        return res;
+    }
+};
+} 

cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp

@@ -154,6 +154,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Floor, floor, sparkFloor);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MothsBetween, months_between, sparkMonthsBetween);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixSeconds, unix_seconds, toUnixTimestamp);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixDate, unix_date, toInt32);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateFromUnixDate, date_from_unix_date, sparkDateFromUnixDate);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMillis, unix_millis, toUnixTimestamp64Milli);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMicros, unix_micros, toUnixTimestamp64Micro);
 REGISTER_COMMON_SCALAR_FUNCTION_PARSER(TimestampMillis, timestamp_millis, fromUnixTimestamp64Milli);

gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

@@ -747,7 +747,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("to_timestamp_ntz")
     .exclude("to_timestamp exception mode")
     .exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
-    .exclude("DATE_FROM_UNIX_DATE")
     .exclude("UNIX_SECONDS")
     .exclude("TIMESTAMP_SECONDS") // refer to https://github.com/ClickHouse/ClickHouse/issues/69280
     .exclude("TIMESTAMP_MICROS") // refer to https://github.com/apache/gluten/issues/7127

gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala

@@ -281,4 +281,21 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra
     val df1 = Seq(x1, x2).toDF("x")
     checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29")) :: Row(null) :: Nil)
   }
+  testGluten("date_from_unix_date") {
+    // -100000 and 200000 are outside ClickHouse's native Date32 range
+    // [1900-01-01, 2299-12-31]. They guard against implementations that clamp to that
+    // range (e.g. mapping to CH toDate32), which would silently diverge from Spark.
+    val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000), None).toDF("unix_date")
+    val expected = Seq(
+      Row(Date.valueOf("1970-01-01")),
+      Row(Date.valueOf("1972-09-27")),
+      Row(Date.valueOf("1696-03-17")),
+      Row(Date.valueOf("2517-08-01")),
+      Row(null))
+
+    // date_from_unix_date is only available in the Scala functions API since Spark 3.5,
+    // so go through expr() here.
+    checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+    checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+  }
 }

gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala

@@ -645,7 +645,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("to_timestamp_ntz")
     .exclude("to_timestamp exception mode")
     .exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing")
-    .exclude("DATE_FROM_UNIX_DATE")
     .exclude("UNIX_SECONDS")
     .exclude("TIMESTAMP_SECONDS") // refer to https://github.com/ClickHouse/ClickHouse/issues/69280
     .exclude("TIMESTAMP_MICROS") // refer to https://github.com/apache/gluten/issues/7127