Storages: Deprecated libclara for normal build (#10325)

close #10324

Storages: Deprecated libclara for normal build
  * Add `ENABLE_CLARA` and disable it by default in normal build
  * Exclude the files under `dbms/src/Storages/DeltaMerge/Index/FullTextIndex` when `ENABLE_CLARA` is disabled

Signed-off-by: JaySon-Huang <[email protected]>

Author: JaySon-Huang

CMakeLists.txt

@@ -44,6 +44,8 @@ include (cmake/utils.cmake)
 
 option(ENABLE_PCH "Enable `Precompiled header`" OFF)
 
+option(ENABLE_CLARA "Enable `Clara` library" OFF)
+
 include (cmake/find_ccache.cmake)
 
 # Write compile_commands.json
@@ -312,7 +314,9 @@ include (cmake/find_tipb.cmake)
 include (cmake/find_curl.cmake)
 include (cmake/find_prometheus.cmake)
 include (cmake/find_tiflash_proxy.cmake)
-include (cmake/find_libclara.cmake)
+if (ENABLE_CLARA)
+    include (cmake/find_libclara.cmake)
+endif ()
 include (cmake/find_xxhash.cmake)
 
 if (OS_LINUX AND ARCH_AMD64 AND TIFLASH_ENABLE_AVX512_SUPPORT)

dbms/CMakeLists.txt

@@ -225,7 +225,6 @@ target_link_libraries (dbms
     absl::synchronization
     tiflash_contrib::aws_s3
     tiflash_vector_search
-    clara_shared
 
     etcdpb
     tiflash_parsers
@@ -241,6 +240,10 @@ target_link_libraries (dbms
     kvstore
 )
 
+if (ENABLE_CLARA)
+    target_link_libraries(dbms clara_shared)
+endif ()
+
 # always add GmSSL include dir to the include path for static analysis
 target_include_directories(dbms PRIVATE ${TiFlash_SOURCE_DIR}/contrib/GmSSL/include)
 if (USE_GM_SSL)
@@ -375,7 +378,7 @@ if (ENABLE_TESTS)
                 DESTINATION ".")
     endif ()
 
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-gtest
                 DESTINATION ".")
@@ -384,7 +387,9 @@ if (ENABLE_TESTS)
     set_target_properties(gtests_dbms PROPERTIES BUILD_RPATH "$ORIGIN/")
     set_target_properties(gtests_dbms PROPERTIES INSTALL_RPATH "$ORIGIN/")
 
-    install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest)
+    if (ENABLE_CLARA)
+        install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest)
+    endif ()
 
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)

dbms/src/Common/TiFlashBuildInfo.cpp

@@ -146,6 +146,11 @@ String getEnabledFeatures()
 #if ENABLE_NEXT_GEN
             "next-gen", 
 #endif
+
+// Clara
+#if ENABLE_CLARA
+            "clara",
+#endif
     };
     {
         auto f = DB::DM::VectorIndexHNSWSIMDFeatures::get();

dbms/src/Common/config.h.in

@@ -7,3 +7,4 @@
 #cmakedefine01 USE_GM_SSL
 #cmakedefine01 USE_QPL
 #cmakedefine01 ENABLE_NEXT_GEN
+#cmakedefine01 ENABLE_CLARA

dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp

@@ -18,6 +18,7 @@
 #include <Common/Stopwatch.h>
 #include <Common/TiFlashException.h>
 #include <Common/TiFlashMetrics.h>
+#include <Common/config.h> // For ENABLE_CLARA
 #include <DataStreams/ExpressionBlockInputStream.h>
 #include <DataStreams/FilterBlockInputStream.h>
 #include <DataStreams/GeneratedColumnPlaceholderBlockInputStream.h>
@@ -41,7 +42,6 @@
 #include <Operators/NullSourceOp.h>
 #include <Operators/UnorderedSourceOp.h>
 #include <Parsers/makeDummyQuery.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx.h>
 #include <Storages/DeltaMerge/Remote/DisaggSnapshot.h>
 #include <Storages/DeltaMerge/Remote/WNDisaggSnapshotManager.h>
@@ -62,6 +62,10 @@
 #include <kvproto/coprocessor.pb.h>
 #include <tipb/select.pb.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
+#endif
+
 namespace DB
 {
 namespace FailPoints
@@ -1610,8 +1614,10 @@ std::pair<Names, std::vector<UInt8>> DAGStorageInterpreter::getColumnsForTableSc
             name = MutSup::extra_table_id_column_name;
         else if (cid == DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.id)
             name = DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.name;
+#if ENABLE_CLARA
         else if (cid == DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.id)
             name = DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.name;
+#endif
         else
             name = storage_for_logical_table->getTableInfo().getColumnName(cid);
         required_columns_tmp.emplace_back(std::move(name));

dbms/src/Server/CMakeLists.txt

@@ -124,7 +124,7 @@ if (OS_LINUX)
                 DESTINATION ".")
     endif()
 
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-release
                 DESTINATION ".")
@@ -134,12 +134,14 @@ if (OS_LINUX)
         install (SCRIPT ${TiFlash_SOURCE_DIR}/cmake/tiflash_linux_post_install.cmake COMPONENT tiflash-release)
     endif()
     install (SCRIPT ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy-cmake/tiflash_proxy_linux_post_install.cmake COMPONENT tiflash-release)
-    install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release)
+    if (ENABLE_CLARA)
+        install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release)
+    endif()
 elseif(APPLE)
     # set build rpaths, so that executables can be directly called in build tree (easy to debug)
     set_target_properties(tiflash PROPERTIES BUILD_RPATH "@executable_path/;@executable_path/${TIFLASH_PROXY_LIB_RPATH}/")
     set_target_properties(tiflash PROPERTIES INSTALL_RPATH "@executable_path/")
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-release
                 DESTINATION ".")

dbms/src/Storages/DeltaMerge/CMakeLists.txt

@@ -32,8 +32,10 @@ add_headers_and_sources(delta_merge ./Index/InvertedIndex)
 add_headers_and_sources(delta_merge ./Index/InvertedIndex/Reader)
 add_headers_and_sources(delta_merge ./Index/VectorIndex)
 add_headers_and_sources(delta_merge ./Index/VectorIndex/Stream)
-add_headers_and_sources(delta_merge ./Index/FullTextIndex)
-add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream)
+if (ENABLE_CLARA)
+    add_headers_and_sources(delta_merge ./Index/FullTextIndex)
+    add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream)
+endif ()
 add_headers_and_sources(delta_merge ./ReadThread)
 add_headers_and_sources(delta_merge ./Remote)
 add_headers_and_sources(delta_merge ./Remote/DataStore)

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp

@@ -12,13 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Interpreters/Context.h>
 #include <Storages/DeltaMerge/File/DMFileBlockInputStream.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Perf.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Reader.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Perf.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Reader.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx.h>
@@ -27,6 +23,13 @@
 #include <Storages/DeltaMerge/ReadThread/SegmentReader.h>
 #include <Storages/DeltaMerge/ScanContext.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Perf.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Reader.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h>
+#endif
 namespace DB::DM
 {
 
@@ -145,10 +148,12 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(
     const RowKeyRanges & rowkey_ranges,
     const ScanContextPtr & scan_context)
 {
+    // Note: this file may not have index built
     {
-        // Note: this file may not have index built
+#if ENABLE_CLARA
         if (fts_index_ctx)
             return buildForFullTextIndex(dmfile, read_columns, rowkey_ranges, scan_context);
+#endif
         if (vec_index_ctx)
             return buildForVectorIndex(dmfile, read_columns, rowkey_ranges, scan_context);
     }
@@ -241,6 +246,7 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForVectorIndex(
         std::move(rest_columns_reader));
 }
 
+#if ENABLE_CLARA
 SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextIndex(
     const DMFilePtr & dmfile,
     const ColumnDefines & read_columns,
@@ -327,5 +333,6 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextInde
         dmfile,
         std::move(rest_columns_reader));
 }
+#endif
 
 } // namespace DB::DM

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h

@@ -15,18 +15,21 @@
 #pragma once
 
 #include <Common/Exception.h>
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/Settings.h>
 #include <Storages/DeltaMerge/DeltaMergeDefines.h>
 #include <Storages/DeltaMerge/File/ColumnCache.h>
 #include <Storages/DeltaMerge/File/ColumnCacheLongTerm_fwd.h>
 #include <Storages/DeltaMerge/File/DMFileReader.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx_fwd.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx_fwd.h>
 #include <Storages/DeltaMerge/RowKeyRange.h>
 #include <Storages/DeltaMerge/ScanContext_fwd.h>
 #include <Storages/DeltaMerge/SkippableBlockInputStream.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx_fwd.h>
+#endif
 
 namespace DB::DM
 {
@@ -122,11 +125,13 @@ class DMFileBlockInputStreamBuilder
         return *this;
     }
 
+#if ENABLE_CLARA
     DMFileBlockInputStreamBuilder & setFtsIndexQuery(const FullTextIndexStreamCtxPtr & ctx)
     {
         fts_index_ctx = ctx;
         return *this;
     }
+#endif
 
     DMFileBlockInputStreamBuilder & setReadPacks(const IdSetPtr & read_packs_)
     {
@@ -201,12 +206,14 @@ class DMFileBlockInputStreamBuilder
         const RowKeyRanges & rowkey_ranges,
         const ScanContextPtr & scan_context);
 
+#if ENABLE_CLARA
     /// The returned stream should be plugged into a FullTextIndexInputStream. Plug to somewhere else will not work.
     SkippableBlockInputStreamPtr buildForFullTextIndex(
         const DMFilePtr & dmfile,
         const ColumnDefines & read_columns,
         const RowKeyRanges & rowkey_ranges,
         const ScanContextPtr & scan_context);
+#endif
 
 private:
     // These methods are called by the ctor
@@ -253,7 +260,9 @@ class DMFileBlockInputStreamBuilder
     /// If set, will *try* to build a VectorIndexDMFileInputStream
     /// instead of a normal DMFileBlockInputStream.
     VectorIndexStreamCtxPtr vec_index_ctx = nullptr;
+#if ENABLE_CLARA
     FullTextIndexStreamCtxPtr fts_index_ctx = nullptr;
+#endif
 
     // Note: column_cache_long_term is currently only filled when performing Vector Search.
     ColumnCacheLongTermPtr column_cache_long_term = nullptr;

dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Storages/DeltaMerge/File/ReadBlockInfo.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/IProvideVectorIndex.h>
 
 #include <span>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h>
+#endif
+
 namespace DB::DM
 {
 
@@ -140,11 +144,13 @@ template ReadBlockInfos ReadBlockInfo::createWithRowIDs<IProvideVectorIndex::Sea
     const DMFileMeta::PackStats & pack_stats,
     size_t rows_threshold_per_read);
 
+#if ENABLE_CLARA
 template ReadBlockInfos ReadBlockInfo::createWithRowIDs<IProvideFullTextIndex::SearchResult>(
     std::span<IProvideFullTextIndex::SearchResult> row_ids,
     const std::vector<size_t> & pack_offset,
     const RSResults & pack_res,
     const DMFileMeta::PackStats & pack_stats,
     size_t rows_threshold_per_read);
+#endif
 
 } // namespace DB::DM