From f3faf06802692c7f69bfba1c5eb3f8c7f618a27c Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Thu, 31 Oct 2024 16:19:55 +0800 Subject: [PATCH 1/3] build: use llvm@17 in darwin Signed-off-by: Lloyd-Pottiger --- README.md | 14 +++++--------- release-darwin/build/build-release.sh | 5 +++++ .../prepare-environments/prepare-llvm.sh | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 release-darwin/prepare-environments/prepare-llvm.sh diff --git a/README.md b/README.md index 158330114d9..6a232a3cf49 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ And the following operating systems: The following packages are required: - CMake 3.23.0+ -- Clang 17.0.0+ under Linux or AppleClang 14.0.0+ under MacOS +- Clang 17.0.0+ - Rust - Python 3.0+ - Ninja-Build or GNU Make @@ -123,11 +123,7 @@ xcode-select --install # Install other dependencies brew install ninja cmake openssl@1.1 ccache -``` - -If your MacOS is higher or equal to 13.0 (Ventura), it should work out of the box because by default Xcode 14.3 provides Apple clang 14.0.0. But if your MacOS is lower than 13.0, you should install llvm clang manually. -```shell brew install llvm@17 # check llvm version @@ -163,9 +159,9 @@ In MacOS, if you install llvm clang, you need to explicitly specify to use llvm Add the following lines to your shell environment, e.g. `~/.bash_profile`. ```shell -export PATH="/opt/homebrew/opt/llvm/bin:$PATH" -export CC="/opt/homebrew/opt/llvm/bin/clang" -export CXX="/opt/homebrew/opt/llvm/bin/clang++" +export PATH="$(brew --prefix)/opt/llvm/bin:$PATH" +export CC="$(brew --prefix)/opt/llvm/bin/clang" +export CXX="$(brew --prefix)/opt/llvm/bin/clang++" ``` Or use `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` to specify the compiler, like this: @@ -173,7 +169,7 @@ Or use `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` to specify the compiler, like mkdir cmake-build-debug cd cmake-build-debug -cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm/bin/clang++ +cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_C_COMPILER="$(brew --prefix)/opt/llvm/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix)/opt/llvm/bin/clang++" ninja tiflash ``` diff --git a/release-darwin/build/build-release.sh b/release-darwin/build/build-release.sh index 624520bedb9..5ee1481928f 100755 --- a/release-darwin/build/build-release.sh +++ b/release-darwin/build/build-release.sh @@ -32,6 +32,11 @@ if [ -d "$install_dir" ]; then rm -rf "${install_dir:?}"/*; else mkdir -p "$inst build_dir="$SRCPATH/release-darwin/build-release" rm -rf $build_dir && mkdir -p $build_dir && cd $build_dir +# use llvm@17 +export PATH="$(brew --prefix)/opt/llvm@17/bin:$PATH" +export CC="$(brew --prefix)/opt/llvm@17/bin/clang" +export CXX="$(brew --prefix)/opt/llvm@17/bin/clang++" + cmake "$SRCPATH" \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DUSE_INTERNAL_SSL_LIBRARY=ON \ diff --git a/release-darwin/prepare-environments/prepare-llvm.sh b/release-darwin/prepare-environments/prepare-llvm.sh new file mode 100644 index 00000000000..6bfe25956fb --- /dev/null +++ b/release-darwin/prepare-environments/prepare-llvm.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2023 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ueox pipefail + +brew install llvm@17 From e46bf2ab0b6cb3906853682f89eb7b11d76169f5 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Fri, 1 Nov 2024 15:08:38 +0800 Subject: [PATCH 2/3] use ninja Signed-off-by: Lloyd-Pottiger --- .../File/DMFileWithVectorIndexBlockInputStream.cpp | 8 ++------ .../Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp | 2 +- release-darwin/build/build-release.sh | 1 + release-darwin/prepare-environments/prepare-llvm.sh | 1 + 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp index ba86b01cab8..a6e76f8270e 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp @@ -170,9 +170,7 @@ std::tuple DMFileWithVectorIndexBlockInputStream::readByIndexRead auto begin = std::lower_bound(sorted_results.cbegin(), sorted_results.cend(), block_start_row_id); auto end = std::lower_bound(begin, sorted_results.cend(), index_reader_next_row_id); - const std::span block_selected_rows{ - &*begin, - static_cast(std::distance(begin, end))}; + const std::span block_selected_rows{begin, end}; vec_index_reader->read(vec_column, block_selected_rows, block_start_row_id, read_rows); block.insert(ColumnWithTypeAndName{std::move(vec_column), vec_cd.type, vec_cd.name, vec_cd.id}); @@ -200,9 +198,7 @@ std::tuple DMFileWithVectorIndexBlockInputStream::readByFollowing // Then read from vector index for the same pack. auto begin = std::lower_bound(sorted_results.cbegin(), sorted_results.cend(), block_others.startOffset()); auto end = std::lower_bound(begin, sorted_results.cend(), block_others.startOffset() + read_rows); - const std::span block_selected_rows{ - &*begin, - static_cast(std::distance(begin, end))}; + const std::span block_selected_rows{begin, end}; vec_index_reader->read(vec_column, block_selected_rows, block_others.startOffset(), read_rows); // Re-assemble block using the same layout as header. diff --git a/dbms/src/Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp b/dbms/src/Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp index c7f5e6da16a..063df3feacb 100644 --- a/dbms/src/Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp @@ -343,7 +343,7 @@ std::vector VectorIndexHNSWViewer::searchWithDi { const auto rowid = result[i].member.key; if (valid_rows[rowid]) - search_results.push_back({rowid, result[i].distance}); + search_results.emplace_back(rowid, result[i].distance); } return search_results; } diff --git a/release-darwin/build/build-release.sh b/release-darwin/build/build-release.sh index 5ee1481928f..24abecce132 100755 --- a/release-darwin/build/build-release.sh +++ b/release-darwin/build/build-release.sh @@ -38,6 +38,7 @@ export CC="$(brew --prefix)/opt/llvm@17/bin/clang" export CXX="$(brew --prefix)/opt/llvm@17/bin/clang++" cmake "$SRCPATH" \ + -GNinja \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DUSE_INTERNAL_SSL_LIBRARY=ON \ -Wno-dev \ diff --git a/release-darwin/prepare-environments/prepare-llvm.sh b/release-darwin/prepare-environments/prepare-llvm.sh index 6bfe25956fb..38c7604c6ba 100644 --- a/release-darwin/prepare-environments/prepare-llvm.sh +++ b/release-darwin/prepare-environments/prepare-llvm.sh @@ -15,4 +15,5 @@ set -ueox pipefail +brew install ninja brew install llvm@17 From cc78453a8ffab0470cd0fb64503255aa8fab24dd Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Fri, 1 Nov 2024 17:44:04 +0800 Subject: [PATCH 3/3] fix Signed-off-by: Lloyd-Pottiger --- .../ColumnFile/ColumnFileSetWithVectorIndexInputStream.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetWithVectorIndexInputStream.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetWithVectorIndexInputStream.cpp index 0143439168e..957942471e7 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetWithVectorIndexInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetWithVectorIndexInputStream.cpp @@ -161,9 +161,7 @@ Block ColumnFileSetWithVectorIndexInputStream::readImpl(FilterPtr & res_filter) // read vector type column by vector index auto tiny_reader = tiny_readers[current_file_index]; auto vec_column = vec_cd.type->createColumn(); - const std::span file_selected_rows{ - &*selected_row_begin, - static_cast(std::distance(selected_row_begin, selected_row_end))}; + const std::span file_selected_rows{selected_row_begin, selected_row_end}; tiny_reader->read(vec_column, file_selected_rows, /* rowid_start_offset= */ read_rows, file_rows); assert(vec_column->size() == file_rows);