[WIP] add bolt backend in gluten

AGENTS.md

@@ -38,6 +38,43 @@ For incremental builds, cloud-FS flags, the Spark/Scala/JDK matrix, table-format
 and shuffle profiles, see [docs/get-started/Velox.md](docs/get-started/Velox.md)
 and [docs/get-started/ClickHouse.md](docs/get-started/ClickHouse.md).
 
+## Bolt Local Workflow
+
+For the local Bolt Spark 3.5 workflow in this repo, use JDK 11.
+
+```bash
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+```
+
+For this repo's local Bolt workflow, treat `fast-build` as mandatory for
+compile/test iterations. It skips style checks and uses incremental Scala
+compilation to speed up local builds.
+
+For Bolt Spark 3.5 packaging, the profile set aligned with `make test_spark35`
+is:
+
+```bash
+-Pbackends-bolt -Pspark-3.5 -Ppaimon -Phadoop-3.2 -Pceleborn -Piceberg
+```
+
+When using direct Maven commands, pass `-Pfast-build` explicitly.
+
+When using `./dev/run-scala-test.sh`, pass the same profile set in its single
+`-P...` argument, plus `scala-2.12` if needed for target selection. That script
+already appends `fast-build` internally, so the effective invocation still
+includes `fast-build`. For example:
+
+```bash
+./dev/run-scala-test.sh \
+  -Pspark-3.5,scala-2.12,backends-bolt,hadoop-3.2,celeborn,iceberg,paimon \
+  -pl backends-bolt \
+  -s org.apache.spark.sql.catalyst.expressions.BoltCastSuite
+```
+
+Do not pass a separate `-Pfast-build` to `run-scala-test.sh`. That script
+already appends `fast-build` internally; passing another `-P...` overrides the
+earlier profile list and can break reactor module selection.
+
 ## Before Committing
 
 You MUST run the following checks and fix any issues before committing.

Makefile

@@ -0,0 +1,226 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+BUILD_DIR := ${ROOT_DIR}/cpp/build
+CONAN_FILE_DIR := ${ROOT_DIR}/cpp/
+BUILD_TYPE=Debug
+ENABLE_ASAN ?= False
+LDB_BUILD ?= False
+BUILD_BENCHMARKS ?= False
+BUILD_TESTS ?= False
+BUILD_EXAMPLES ?= False
+BUILD_ORC ?= False
+ENABLE_PROTON ?= False
+
+# conan package info
+GLUTEN_BUILD_VERSION ?= main
+BOLT_BUILD_VERSION ?= main
+BUILD_USER ?=
+BUILD_CHANNEL ?=
+
+ENABLE_HDFS ?= True
+ENABLE_S3 ?= False
+RSS_PROFILE ?= ''
+
+ifeq ($(BUILD_BENCHMARKS),True)
+BUILD_ORC = True
+endif
+
+ARCH := $(shell arch)
+ifeq ($(ARCH), x86_64)
+	ARCH := amd64
+endif
+
+SHARED_LIBRARY ?= True
+
+# Gluten package version, read from package/pom.xml so jar paths stay in sync
+# when the project version bumps.
+GLUTEN_PACKAGE_VERSION := $(shell sed -n '0,/<version>\(.*\)<\/version>/s//\1/p' \
+    ${ROOT_DIR}/package/pom.xml | head -1 | xargs)
+
+# Manually specify the number of bolt compilation threads by setting the BOLT_NUM_THREADS environment variable.
+# e.g. export BOLT_NUM_THREADS=50
+ifndef CI_NUM_THREADS
+	ifdef BOLT_NUM_THREADS
+		NUM_THREADS ?= $(BOLT_NUM_THREADS)
+	else
+		NUM_THREADS ?= $$((  $(shell grep -c ^processor /proc/cpuinfo) / 2 ))
+	endif
+else
+    NUM_THREADS ?= $(CI_NUM_THREADS)
+endif
+
+ALLOWED_VERSIONS := 11 17
+ifeq ($(JAVA_HOME),)
+    $(error ERROR: JAVA_HOME is not set)
+endif
+ifneq ($(wildcard $(JAVA_HOME)/bin/java),)
+    ifneq ($(wildcard $(JAVA_HOME)/bin/javac),)
+        JDK_VERSION := $(shell $(JAVA_HOME)/bin/java -version 2>&1 | sed -n 's/.*version "\(1\.\)\{0,1\}\([0-9]\+\).*/\2/p')
+        ifneq ($(filter $(JDK_VERSION),$(ALLOWED_VERSIONS)),$(JDK_VERSION))
+            $(error ERROR: JDK version $(JDK_VERSION) is not supported, only 11 and 17 are allowed now)
+        endif
+    endif
+endif
+
+.PHONY: build clean clean_cpp debug release RelWithDebInfo \
+	install_debug install_release release-with-tests debug-with-tests \
+	release-with-benchmarks debug-with-benchmarks \
+	release-with-tests-and-benchmarks debug-with-tests-and-benchmarks \
+	java jar jar-skip-check jar_spark33 jar_spark34 jar_spark35 \
+	fast-jar zip fast-zip test test_spark35 cpp-test-release cpp-test-debug
+
+bolt-recipe:
+	@echo "Install Bolt recipe into local cache"
+	rm -rf ep/bolt
+	git clone --depth=1 --branch ${BOLT_BUILD_VERSION} https://github.com/bytedance/bolt.git ep/bolt &&\
+    bash ep/bolt/scripts/install-bolt-deps.sh && \
+    conan export ep/bolt/conanfile.py --name=bolt --version=${BOLT_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL}
+	@echo "Bolt recipe has been installed"
+
+build:
+	mkdir -p ${BUILD_DIR} && mkdir -p ${BUILD_DIR}/releases &&\
+	cd  ${CONAN_FILE_DIR} && export BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} &&\
+	ALL_CONAN_OPTIONS=" -o gluten/*:shared=${SHARED_LIBRARY} \
+			-o gluten/*:enable_hdfs=${ENABLE_HDFS} \
+			-o gluten/*:enable_s3=${ENABLE_S3} \
+			-o gluten/*:enable_asan=${ENABLE_ASAN} \
+			-o gluten/*:build_benchmarks=${BUILD_BENCHMARKS} \
+			-o gluten/*:build_tests=${BUILD_TESTS} \
+			-o gluten/*:build_examples=${BUILD_EXAMPLES} " && \
+	conan graph info . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL} -c "arrow/*:tools.build:download_source=True" $${ALL_CONAN_OPTIONS} --format=html > gluten.conan.graph.html  && \
+	NUM_THREADS=$(NUM_THREADS) conan install . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL}  \
+			-s llvm-core/*:build_type=Release -s build_type=${BUILD_TYPE} --build=missing $${ALL_CONAN_OPTIONS} && \
+	cmake --preset `echo conan-${BUILD_TYPE} | tr A-Z a-z` && \
+	cmake --build build/${BUILD_TYPE} -j $(NUM_THREADS) && \
+	if [ "${SHARED_LIBRARY}" = "True" ]; then  cmake --build ${BUILD_DIR}/${BUILD_TYPE} --target install ; fi && \
+	if [ "${SHARED_LIBRARY}" = "False" ]; then \
+	    conan export-pkg . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL} -s build_type=${BUILD_TYPE} \
+		$${ALL_CONAN_OPTIONS} ; \
+	fi && cd -
+
+release :
+	$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
+
+debug:
+	$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
+
+RelWithDebInfo:
+	$(MAKE) build BUILD_TYPE=RelWithDebInfo GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
+
+clean_cpp:
+	rm -rf ${ROOT_DIR}/cpp/build &&\
+	rm -f cpp/conan.lock cpp/conaninfo.txt cpp/graph_info.json CMakeCache.txt
+
+install_debug:
+	$(MAKE) clean_cpp
+	$(MAKE) debug SHARED_LIBRARY=False
+
+install_release:
+	$(MAKE) clean_cpp
+	$(MAKE) release SHARED_LIBRARY=False
+
+release-with-tests :
+	$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_TESTS=True
+
+debug-with-tests :
+	$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_TESTS=True
+
+release-with-benchmarks :
+	$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} B	UILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True
+
+debug-with-benchmarks :
+	$(MAKE) build BUILD_TYPE=Debug  GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True
+
+release-with-tests-and-benchmarks :
+	$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True BUILD_TESTS=True
+
+debug-with-tests-and-benchmarks :
+	$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True BUILD_TESTS=True
+
+arrow:
+	bash dev/build_bolt_arrow.sh
+
+# build gluten jar
+jar:
+	java -version && mvn package -Pbackends-bolt -Pspark-3.3 -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon &&\
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+jar-skip-check:
+	java -version && mvn package -Pbackends-bolt -Pspark-3.2 -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon -Dcheckstyle.skip=true -Dspotless.check.skip=true &&\
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+spark32-las:
+	java -version && mvn package -Pbackends-bolt -Pspark-3.2-las -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon &&\
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+fast-jar:
+	if [ ! -f "output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar" ] ; then \
+		$(MAKE) jar; \
+	else \
+		jar uf output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar -C cpp/build/releases/ libbolt_backend.so; \
+	fi
+
+zip:
+	$(MAKE) jar
+	rm -rf output/gluten-spark*.zip
+	zip -j output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.zip output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+fast-zip:
+	$(MAKE) fast-jar
+	rm -rf output/gluten-spark*.zip
+	zip -j output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.zip output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+jar_spark33:
+	java -version && mvn -T32 clean package -Pbackends-bolt -Pspark-3.3 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true -Ppaimon && \
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.3_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+jar_spark34:
+	java -version && mvn clean package -Pbackends-bolt -Pspark-3.4 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true -Ppaimon && \
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.4_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+jar_spark35:
+	java -version && mvn -T32 package -Pfast-build -Pbackends-bolt -Pspark-3.5 -Phadoop-3.2 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true && \
+	mkdir -p output && \
+	rm -rf output/gluten-spark*.jar
+	mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.5_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+
+test:
+	SPARK_TESTING=true mvn -Pbackends-bolt -Pspark-3.2 -Pceleborn -Ppaimon package -Denforcer.skip=true
+
+test_spark35:
+	SPARK_TESTING=true mvn -Pbackends-bolt -Pspark-3.5 -Ppaimon -Phadoop-3.2 -Pceleborn -Piceberg package -Denforcer.skip=true -DfailIfNoTests=false -Dexec.skip -Dmaven.test.failure.ignore=true
+
+cpp-test-release: release-with-tests
+	cd $(BUILD_DIR)/Release && ctest --timeout 7200 -j $(NUM_THREADS) --output-on-failure -V
+
+cpp-test-debug: debug-with-tests
+	cd $(BUILD_DIR)/Debug && ctest --timeout 7200 -j $(NUM_THREADS) --output-on-failure -V
+
+clean :
+	$(MAKE) clean_cpp
+	mvn clean -Pbackends-bolt -Pspark-3.2 -Pceleborn -Ppaimon -DskipTests -Denforcer.skip=true && \
+	rm -rf ${ROOT_DIR}/output/gluten-*.jar

README.md

@@ -154,6 +154,60 @@ ClickHouse backend demonstrated an average speedup of 2.12x, with up to 3.48x sp
 
 <sub>Test environment: a 8-nodes AWS cluster with 1TB data, using Spark 3.1.1 as the baseline and with Gluten integrated into the same Spark version.</sub>
 
+### Bolt Backend
+#### Prerequisites
+* Linux operating system
+* GCC 10/11/12 or Clang 16
+* python 3 (virtualenv or Conda) for conan
+
+Linux with kernel version(>5.4) is preferred, since Bolt will enable io-uring when the kernel supports.
+
+if the system gcc version is too older, it is recommended to install GCC from source code:
+```shell
+# run with root privilege
+bash ./dev/install-gcc.sh 12.5.0
+```
+
+Bolt adopts Conan as its package manager. Conan is an open-source, cross-platform package management tool.
+We provide dedicated scripts to assist developers in setting up and installing Bolt's dependencies.
+```shell
+bash ./dev/install-conan.sh
+```
+
+We also provide a Dockerfile to build a Docker image for the **Bolt** backend, it includes all the prerequisites required to build Gluten with Bolt backend.
+```shell
+docker buildx build -t bolt -f dev/docker/Dockerfile.centos8-bolt .
+```
+
+#### Build Bolt Backend
+To install bolt recipe for Gluten:
+```shell
+# Install the recipes of Bolt and its third-party dependencies
+make bolt-recipe
+
+# specific a version of Bolt (release or branch)
+# `main` branch is the default
+make bolt-recipe  BOLT_BUILD_VERSION=main
+```
+
+To build bolt backend:
+```shell
+make release
+
+# or specific the version for Bolt, and the version for Gluten
+make release BOLT_BUILD_VERSION=main GLUTEN_BUILD_VERSION=main
+```
+Note that, the missing third-parties binaries will be built from source for the first time.
+
+To build gluten:
+
+```shell
+# install arrow dependency for gluten
+make arrow
+
+make jar_spark35
+```
+
 ## 8. Qualification Tool
 
 The [Qualification Tool](./tools/qualification-tool/README.md) is a utility to analyze Spark event log files and assess the compatibility and performance of SQL workloads with Gluten. This tool helps users understand how their workloads can benefit from Gluten.

backends-bolt/benchmark/ColumnarTableCacheBenchmark-results.txt

@@ -0,0 +1,23 @@
+OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
+Apple M1 Pro
+table cache count:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+disable columnar table cache                      16773          17024         401          1.2         838.7       1.0X
+enable columnar table cache                        9985          10051          65          2.0         499.3       1.0X
+
+
+OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
+Apple M1 Pro
+table cache column pruning:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+disable columnar table cache                      16429          16873         688          1.2         821.5       1.0X
+enable columnar table cache                       15118          15495         456          1.3         755.9       1.0X
+
+
+OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
+Apple M1 Pro
+table cache filter:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+disable columnar table cache                      22895          23527         722          0.9        1144.7       1.0X
+enable columnar table cache                       16673          17462         765          1.2         833.7       1.0X
+