Skip to content

Commit f7e5751

Browse files
zjw1111zjw1111
authored
feat(fs): introduce Jindo file system support (#148)
1 parent 5c81256 commit f7e5751

27 files changed

Lines changed: 3940 additions & 0 deletions

‎CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
5252
option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
5353
option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
5454
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
55+
option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
5556
option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
5657
if(PAIMON_ENABLE_LUCENE)
5758
message(FATAL_ERROR "PAIMON_ENABLE_LUCENE is temporarily disabled. Please configure with -DPAIMON_ENABLE_LUCENE=OFF."
@@ -64,6 +65,9 @@ endif()
6465
if(PAIMON_ENABLE_AVRO)
6566
add_definitions(-DPAIMON_ENABLE_AVRO)
6667
endif()
68+
if(PAIMON_ENABLE_JINDO)
69+
add_definitions(-DPAIMON_ENABLE_JINDO)
70+
endif()
6771
if(PAIMON_USE_CXX11_ABI)
6872
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
6973
else()
@@ -405,6 +409,11 @@ if(PAIMON_BUILD_TESTS)
405409
list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
406410
list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
407411
endif()
412+
if(PAIMON_ENABLE_JINDO)
413+
list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
414+
list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
415+
list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
416+
endif()
408417
if(PAIMON_ENABLE_LUMINA)
409418
list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
410419
list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
@@ -437,6 +446,9 @@ config_summary_message()
437446
438447
add_subdirectory(src/paimon)
439448
add_subdirectory(src/paimon/fs/local)
449+
if(PAIMON_ENABLE_JINDO)
450+
add_subdirectory(src/paimon/fs/jindo)
451+
endif()
440452
add_subdirectory(src/paimon/format/blob)
441453
add_subdirectory(src/paimon/format/orc)
442454
add_subdirectory(src/paimon/format/parquet)

‎ci/scripts/build_paimon.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ CMAKE_ARGS=(
3939
"-G Ninja"
4040
"-DCMAKE_BUILD_TYPE=${build_type}"
4141
"-DPAIMON_BUILD_TESTS=ON"
42+
"-DPAIMON_ENABLE_JINDO=ON"
4243
"-DPAIMON_ENABLE_LUCENE=OFF"
4344
)
4445

‎cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,56 @@ else()
281281
endif()
282282
endif()
283283

284+
if(APPLE)
285+
set(JINDOSDK_C_DYNAMIC_LIB_NAME "jindosdk_c.${PAIMON_JINDOSDK_C_BUILD_VERSION}")
286+
set(JINDOSDK_C_DYNAMIC_LIB_FILE "lib${JINDOSDK_C_DYNAMIC_LIB_NAME}.dylib")
287+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
288+
set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
289+
"${PAIMON_JINDOSDK_C_MACOS_X86_64_BUILD_SHA256_CHECKSUM}")
290+
if(DEFINED ENV{PAIMON_JINDOSDK_C_MACOS_X86_64_URL})
291+
set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_MACOS_X86_64_URL}")
292+
else()
293+
set_urls(JINDOSDK_C_SOURCE_URL
294+
"https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-macos-11_0-x86_64.tar.gz"
295+
)
296+
endif()
297+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
298+
set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
299+
"${PAIMON_JINDOSDK_C_MACOS_AARCH64_BUILD_SHA256_CHECKSUM}")
300+
if(DEFINED ENV{PAIMON_JINDOSDK_C_MACOS_AARCH64_URL})
301+
set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_MACOS_AARCH64_URL}")
302+
else()
303+
set_urls(JINDOSDK_C_SOURCE_URL
304+
"https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-macos-11_0-aarch64.tar.gz"
305+
)
306+
endif()
307+
endif()
308+
else()
309+
set(JINDOSDK_C_DYNAMIC_LIB_NAME "jindosdk_c")
310+
set(JINDOSDK_C_DYNAMIC_LIB_FILE "lib${JINDOSDK_C_DYNAMIC_LIB_NAME}.so")
311+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
312+
set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
313+
"${PAIMON_JINDOSDK_C_LINUX_X86_64_BUILD_SHA256_CHECKSUM}")
314+
if(DEFINED ENV{PAIMON_JINDOSDK_C_LINUX_X86_64_URL})
315+
set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_LINUX_X86_64_URL}")
316+
else()
317+
set_urls(JINDOSDK_C_SOURCE_URL
318+
"https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux.tar.gz"
319+
)
320+
endif()
321+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
322+
set(JINDOSDK_C_BUILD_SHA256_CHECKSUM
323+
"${PAIMON_JINDOSDK_C_LINUX_AARCH64_BUILD_SHA256_CHECKSUM}")
324+
if(DEFINED ENV{PAIMON_JINDOSDK_C_LINUX_AARCH64_URL})
325+
set(JINDOSDK_C_SOURCE_URL "$ENV{PAIMON_JINDOSDK_C_LINUX_AARCH64_URL}")
326+
else()
327+
set_urls(JINDOSDK_C_SOURCE_URL
328+
"https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux-el7-aarch64.tar.gz"
329+
)
330+
endif()
331+
endif()
332+
endif()
333+
284334
set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
285335
set(EP_C_FLAGS "${CMAKE_C_FLAGS}")
286336
string(REPLACE "-Wglobal-constructors" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
@@ -1105,6 +1155,97 @@ macro(build_lz4)
11051155
add_dependencies(lz4 lz4_ep)
11061156
endmacro()
11071157

1158+
macro(build_jindosdk_c)
1159+
message(STATUS "Building jindosdk-c from precompiled package")
1160+
1161+
set(JINDOSDK_C_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jindosdk_ep-install")
1162+
set(JINDOSDK_C_HOME "${JINDOSDK_C_PREFIX}")
1163+
set(JINDOSDK_C_INCLUDE_DIR "${JINDOSDK_C_PREFIX}/include")
1164+
set(JINDOSDK_C_LIB_DIR "${JINDOSDK_C_PREFIX}/lib/native")
1165+
set(JINDOSDK_C_DYNAMIC_LIB "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_DYNAMIC_LIB_FILE}")
1166+
1167+
# Extract and install jindosdk from precompiled package
1168+
externalproject_add(jindosdk_ep
1169+
URL ${JINDOSDK_C_SOURCE_URL}
1170+
URL_HASH "SHA256=${JINDOSDK_C_BUILD_SHA256_CHECKSUM}"
1171+
${THIRDPARTY_LOG_OPTIONS}
1172+
CONFIGURE_COMMAND ""
1173+
BUILD_COMMAND ""
1174+
INSTALL_COMMAND bash -c
1175+
"cp -r <SOURCE_DIR>/include/* ${JINDOSDK_C_INCLUDE_DIR}"
1176+
COMMAND bash -c
1177+
"cp -r <SOURCE_DIR>/lib/native/${JINDOSDK_C_DYNAMIC_LIB_FILE}* ${JINDOSDK_C_LIB_DIR}"
1178+
BUILD_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}")
1179+
1180+
# The include directory must exist before it is referenced by a target.
1181+
file(MAKE_DIRECTORY "${JINDOSDK_C_INCLUDE_DIR}")
1182+
file(MAKE_DIRECTORY "${JINDOSDK_C_LIB_DIR}")
1183+
1184+
add_library(jindosdk::c_sdk SHARED IMPORTED)
1185+
set_target_properties(jindosdk::c_sdk
1186+
PROPERTIES IMPORTED_LOCATION "${JINDOSDK_C_DYNAMIC_LIB}"
1187+
INTERFACE_INCLUDE_DIRECTORIES
1188+
"${JINDOSDK_C_INCLUDE_DIR}")
1189+
list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_C_INCLUDE_DIR})
1190+
1191+
add_dependencies(jindosdk::c_sdk jindosdk_ep)
1192+
install(DIRECTORY "${JINDOSDK_C_LIB_DIR}/"
1193+
DESTINATION ${CMAKE_INSTALL_LIBDIR}
1194+
FILES_MATCHING
1195+
PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*")
1196+
1197+
endmacro()
1198+
1199+
macro(build_jindosdk_nextarch)
1200+
message(STATUS "Building jindosdk-nextarch from local source")
1201+
1202+
set(JINDOSDK_NEXTARCH_PREFIX
1203+
"${CMAKE_CURRENT_BINARY_DIR}/jindosdk-nextarch_ep-install")
1204+
set(JINDOSDK_NEXTARCH_HOME "${JINDOSDK_NEXTARCH_PREFIX}")
1205+
set(JINDOSDK_NEXTARCH_INCLUDE_DIR "${JINDOSDK_NEXTARCH_PREFIX}/include")
1206+
set(JINDOSDK_NEXTARCH_LIB_DIR "${JINDOSDK_NEXTARCH_PREFIX}/lib")
1207+
set(JINDOSDK_NEXTARCH_SOURCE_DIR "${CMAKE_SOURCE_DIR}/third_party/jindosdk-nextarch")
1208+
set(JINDOSDK_NEXTARCH_STATIC_LIB
1209+
"${JINDOSDK_NEXTARCH_LIB_DIR}/libjindosdk-nextarch.a")
1210+
1211+
# Get jindosdk dependencies (headers and dynamic library)
1212+
get_target_property(JINDOSDK_C_INCLUDE_DIR jindosdk::c_sdk
1213+
INTERFACE_INCLUDE_DIRECTORIES)
1214+
get_target_property(JINDOSDK_C_LIBRARY_LOCATION jindosdk::c_sdk IMPORTED_LOCATION)
1215+
get_filename_component(JINDOSDK_C_DIR_ROOT "${JINDOSDK_C_INCLUDE_DIR}" DIRECTORY)
1216+
1217+
# Compile flags for jindosdk-nextarch
1218+
set(JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
1219+
set(JINDOSDK_NEXTARCH_CMAKE_C_FLAGS "${EP_C_FLAGS}")
1220+
set(JINDOSDK_NEXTARCH_CMAKE_ARGS
1221+
${EP_COMMON_CMAKE_ARGS}
1222+
"-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}"
1223+
"-DCMAKE_CXX_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS}"
1224+
"-DCMAKE_C_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_C_FLAGS}"
1225+
-DJINDOSDK_ROOT=${JINDOSDK_C_DIR_ROOT}
1226+
-DJINDOSDK_LIBRARY_NAME=${JINDOSDK_C_DYNAMIC_LIB_NAME})
1227+
1228+
externalproject_add(jindosdk-nextarch_ep
1229+
SOURCE_DIR ${JINDOSDK_NEXTARCH_SOURCE_DIR}
1230+
CMAKE_ARGS ${JINDOSDK_NEXTARCH_CMAKE_ARGS}
1231+
BUILD_BYPRODUCTS "${JINDOSDK_NEXTARCH_STATIC_LIB}"
1232+
DEPENDS jindosdk::c_sdk ${THIRDPARTY_LOG_OPTIONS})
1233+
1234+
# The include directory must exist before it is referenced by a target.
1235+
file(MAKE_DIRECTORY "${JINDOSDK_NEXTARCH_INCLUDE_DIR}")
1236+
file(MAKE_DIRECTORY "${JINDOSDK_NEXTARCH_LIB_DIR}")
1237+
1238+
add_library(jindosdk::nextarch STATIC IMPORTED)
1239+
set_target_properties(jindosdk::nextarch
1240+
PROPERTIES IMPORTED_LOCATION "${JINDOSDK_NEXTARCH_STATIC_LIB}"
1241+
INTERFACE_INCLUDE_DIRECTORIES
1242+
"${JINDOSDK_NEXTARCH_INCLUDE_DIR}")
1243+
target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread dl)
1244+
list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_NEXTARCH_INCLUDE_DIR})
1245+
1246+
add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep)
1247+
endmacro()
1248+
11081249
macro(build_protobuf)
11091250
message(STATUS "Building protobuf from source")
11101251
set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
@@ -1672,6 +1813,10 @@ if(PAIMON_ENABLE_ORC)
16721813
resolve_dependency(Protobuf)
16731814
resolve_dependency(ORC)
16741815
endif()
1816+
if(PAIMON_ENABLE_JINDO)
1817+
build_jindosdk_c()
1818+
build_jindosdk_nextarch()
1819+
endif()
16751820
if(PAIMON_ENABLE_LUCENE)
16761821
build_boost()
16771822
build_lucene()

‎docs/source/build_system.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
8686
- File system plugins:
8787

8888
- ``paimon_local_file_system_shared`` / ``paimon_local_file_system_static``
89+
- ``paimon_jindo_file_system_shared`` / ``paimon_jindo_file_system_static``
8990

9091
- Index plugins:
9192

‎docs/source/building.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ boolean flags to ``cmake``.
123123

124124
* ``-DPAIMON_ENABLE_ORC=ON``: Paimon integration with Apache ORC
125125
* ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration
126+
* ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
126127

127128
Third-party dependency source
128129
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

‎src/paimon/fs/jindo/CMakeLists.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
if(PAIMON_ENABLE_JINDO)
18+
set(PAIMON_JINDO_FILE_SYSTEM jindo_file_system.cpp jindo_file_system_factory.cpp)
19+
20+
add_paimon_lib(paimon_jindo_file_system
21+
SOURCES
22+
${PAIMON_JINDO_FILE_SYSTEM}
23+
EXTRA_INCLUDES
24+
${JINDOSDK_INCLUDE_DIR}
25+
DEPENDENCIES
26+
paimon_shared
27+
jindosdk::nextarch
28+
STATIC_LINK_LIBS
29+
jindosdk::nextarch
30+
fmt
31+
SHARED_LINK_LIBS
32+
paimon_shared
33+
SHARED_LINK_FLAGS
34+
${PAIMON_VERSION_SCRIPT_FLAGS})
35+
endif()

‎src/paimon/fs/jindo/jindo_file_status.h‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#pragma once
20+
21+
#include <string>
22+
#include <utility>
23+
24+
#include "JdoFileInfo.hpp" // NOLINT(build/include_subdir)
25+
#include "paimon/fs/file_system.h"
26+
27+
namespace paimon::jindo {
28+
class JindoBasicFileStatus : public BasicFileStatus {
29+
public:
30+
explicit JindoBasicFileStatus(JdoFileInfo&& file_info) : file_info_(std::move(file_info)) {}
31+
32+
std::string GetPath() const override {
33+
return file_info_.getPath();
34+
}
35+
36+
bool IsDir() const override {
37+
return file_info_.isDir();
38+
}
39+
40+
private:
41+
JdoFileInfo file_info_;
42+
};
43+
44+
class JindoFileStatus : public FileStatus {
45+
public:
46+
explicit JindoFileStatus(JdoFileInfo&& file_info) : file_info_(std::move(file_info)) {}
47+
48+
std::string GetPath() const override {
49+
return file_info_.getPath();
50+
}
51+
52+
uint64_t GetLen() const override {
53+
return file_info_.getLength();
54+
}
55+
56+
int64_t GetModificationTime() const override {
57+
return file_info_.getMtime();
58+
}
59+
60+
bool IsDir() const override {
61+
return file_info_.isDir();
62+
}
63+
64+
private:
65+
JdoFileInfo file_info_;
66+
};
67+
} // namespace paimon::jindo

0 commit comments

Comments
 (0)