Skip to content

Commit 1697c72

Browse files
Lloyd-PottigerLloyd-Pottiger
authored
Storages: refine VectorIndexCache to LocalIndexCache (#9840)
ref #9032 In order to accept more index kinds like FullTextIndex, this PR refine VectorIndexCache to LocalIndexCache. Signed-off-by: Lloyd-Pottiger <[email protected]>
1 parent fe563a1 commit 1697c72

25 files changed

+126
-82
lines changed

‎dbms/src/Interpreters/Context.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@
5353
#include <Storages/DeltaMerge/ColumnFile/ColumnFileSchema.h>
5454
#include <Storages/DeltaMerge/DeltaIndexManager.h>
5555
#include <Storages/DeltaMerge/File/ColumnCacheLongTerm.h>
56+
#include <Storages/DeltaMerge/Index/LocalIndexCache.h>
5657
#include <Storages/DeltaMerge/Index/MinMaxIndex.h>
57-
#include <Storages/DeltaMerge/Index/VectorIndexCache.h>
5858
#include <Storages/DeltaMerge/LocalIndexerScheduler.h>
5959
#include <Storages/DeltaMerge/StoragePool/GlobalPageIdAllocator.h>
6060
#include <Storages/DeltaMerge/StoragePool/GlobalStoragePool.h>
@@ -148,7 +148,7 @@ struct ContextShared
148148
mutable DBGInvoker dbg_invoker; /// Execute inner functions, debug only.
149149
mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files.
150150
mutable DM::MinMaxIndexCachePtr minmax_index_cache; /// Cache of minmax index in compressed files.
151-
mutable DM::VectorIndexCachePtr vector_index_cache;
151+
mutable DM::LocalIndexCachePtr local_index_cache;
152152
mutable DM::ColumnCacheLongTermPtr column_cache_long_term;
153153
mutable DM::DeltaIndexManagerPtr delta_index_manager; /// Manage the Delta Indies of Segments.
154154
ProcessList process_list; /// Executing queries at the moment.
@@ -1370,26 +1370,26 @@ void Context::dropMinMaxIndexCache() const
13701370
shared->minmax_index_cache->reset();
13711371
}
13721372

1373-
void Context::setVectorIndexCache(size_t cache_entities)
1373+
void Context::setLocalIndexCache(size_t cache_entities)
13741374
{
13751375
auto lock = getLock();
13761376

1377-
RUNTIME_CHECK(!shared->vector_index_cache);
1377+
RUNTIME_CHECK(!shared->local_index_cache);
13781378

1379-
shared->vector_index_cache = std::make_shared<DM::VectorIndexCache>(cache_entities);
1379+
shared->local_index_cache = std::make_shared<DM::LocalIndexCache>(cache_entities);
13801380
}
13811381

1382-
DM::VectorIndexCachePtr Context::getVectorIndexCache() const
1382+
DM::LocalIndexCachePtr Context::getLocalIndexCache() const
13831383
{
13841384
auto lock = getLock();
1385-
return shared->vector_index_cache;
1385+
return shared->local_index_cache;
13861386
}
13871387

1388-
void Context::dropVectorIndexCache() const
1388+
void Context::dropLocalIndexCache() const
13891389
{
13901390
auto lock = getLock();
1391-
if (shared->vector_index_cache)
1392-
shared->vector_index_cache.reset();
1391+
if (shared->local_index_cache)
1392+
shared->local_index_cache.reset();
13931393
}
13941394

13951395
void Context::setColumnCacheLongTerm(size_t cache_size_in_bytes)

‎dbms/src/Interpreters/Context.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ enum class PageStorageRunMode : UInt8;
109109
namespace DM
110110
{
111111
class MinMaxIndexCache;
112-
class VectorIndexCache;
112+
class LocalIndexCache;
113113
class ColumnCacheLongTerm;
114114
class DeltaIndexManager;
115115
class GlobalStoragePool;
@@ -399,9 +399,9 @@ class Context
399399
std::shared_ptr<DM::MinMaxIndexCache> getMinMaxIndexCache() const;
400400
void dropMinMaxIndexCache() const;
401401

402-
void setVectorIndexCache(size_t cache_entities);
403-
std::shared_ptr<DM::VectorIndexCache> getVectorIndexCache() const;
404-
void dropVectorIndexCache() const;
402+
void setLocalIndexCache(size_t cache_entities);
403+
std::shared_ptr<DM::LocalIndexCache> getLocalIndexCache() const;
404+
void dropLocalIndexCache() const;
405405

406406
void setColumnCacheLongTerm(size_t cache_size_in_bytes);
407407
std::shared_ptr<DM::ColumnCacheLongTerm> getColumnCacheLongTerm() const;

‎dbms/src/Server/Server.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1083,7 +1083,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
10831083
/// The vector index cache by number instead of bytes. Because it use `mmap` and let the operator system decide the memory usage.
10841084
size_t vec_index_cache_entities = config().getUInt64("vec_index_cache_entities", 1000);
10851085
if (vec_index_cache_entities)
1086-
global_context->setVectorIndexCache(vec_index_cache_entities);
1086+
global_context->setLocalIndexCache(vec_index_cache_entities);
10871087

10881088
size_t column_cache_long_term_size
10891089
= config().getUInt64("column_cache_long_term_size", 512 * 1024 * 1024 /* 512MB */);

‎dbms/src/Storages/DeltaMerge/ColumnDefine_fwd.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,12 @@
1818
#include <TiDB/Schema/TiDB_fwd.h>
1919

2020
#include <memory>
21-
#include <unordered_map>
2221
#include <vector>
2322

2423
namespace DB::DM
2524
{
2625
struct ColumnDefine;
2726
using ColumnDefines = std::vector<ColumnDefine>;
2827
using ColumnDefinesPtr = std::shared_ptr<ColumnDefines>;
29-
using ColumnDefineMap = std::unordered_map<DB::ColumnID, ColumnDefine>;
3028

3129
} // namespace DB::DM

‎dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileSetWithVectorIndexInputStream.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ class ColumnFileSetWithVectorIndexInputStream : public VectorIndexBlockInputStre
3838
const ANNQueryInfoPtr ann_query_info;
3939
const BitmapFilterView valid_rows;
4040
// Global vector index cache
41-
const VectorIndexCachePtr vec_index_cache;
41+
const LocalIndexCachePtr vec_index_cache;
4242
const ColumnDefine vec_cd;
4343
const ColumnDefinesPtr rest_col_defs;
4444

@@ -69,7 +69,7 @@ class ColumnFileSetWithVectorIndexInputStream : public VectorIndexBlockInputStre
6969
, data_provider(data_provider_)
7070
, ann_query_info(ann_query_info_)
7171
, valid_rows(std::move(valid_rows_))
72-
, vec_index_cache(context_.global_context.getVectorIndexCache())
72+
, vec_index_cache(context_.global_context.getLocalIndexCache())
7373
, vec_cd(std::move(vec_cd_))
7474
, rest_col_defs(rest_col_defs_)
7575
, column_files(reader.snapshot->getColumnFiles())

‎dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h>
1919
#include <Storages/DeltaMerge/ColumnFile/ColumnFileTiny.h>
2020
#include <Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.h>
21-
#include <Storages/DeltaMerge/Index/VectorIndexCache.h>
21+
#include <Storages/DeltaMerge/Index/LocalIndexCache.h>
2222
#include <Storages/DeltaMerge/Index/VectorSearchPerf.h>
2323

2424
namespace DB::DM
@@ -84,10 +84,11 @@ void ColumnFileTinyVectorIndexReader::loadVectorIndex()
8484
CompressedReadBuffer compressed(read_buf);
8585
return VectorIndexViewer::load(index_info_iter->index_props().vector_index(), compressed);
8686
};
87-
if (vec_index_cache)
87+
if (local_index_cache)
8888
{
89-
const auto key = fmt::format("{}{}", VectorIndexCache::COLUMNFILETINY_INDEX_NAME_PREFIX, index_page_id);
90-
vec_index = vec_index_cache->getOrSet(key, load_from_page_storage);
89+
const auto key = fmt::format("{}{}", LocalIndexCache::COLUMNFILETINY_INDEX_NAME_PREFIX, index_page_id);
90+
auto local_index = local_index_cache->getOrSet(key, load_from_page_storage);
91+
vec_index = std::dynamic_pointer_cast<VectorIndexViewer>(local_index);
9192
}
9293
else
9394
vec_index = load_from_page_storage();

‎dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616

1717
#include <Storages/DeltaMerge/BitmapFilter/BitmapFilterView.h>
1818
#include <Storages/DeltaMerge/ColumnFile/ColumnFile.h>
19+
#include <Storages/DeltaMerge/Index/LocalIndex_fwd.h>
1920
#include <Storages/DeltaMerge/Index/VectorIndex.h>
20-
#include <Storages/DeltaMerge/Index/VectorIndexCache_fwd.h>
2121

2222

2323
namespace DB::DM
@@ -35,8 +35,8 @@ class ColumnFileTinyVectorIndexReader
3535
const BitmapFilterView valid_rows;
3636
// Note: ColumnDefine comes from read path does not have vector_index fields.
3737
const ColumnDefine vec_cd;
38-
// Global vector index cache
39-
const VectorIndexCachePtr vec_index_cache;
38+
// Global local index cache
39+
const LocalIndexCachePtr local_index_cache;
4040
LoggerPtr log;
4141

4242
// Performance statistics
@@ -62,13 +62,13 @@ class ColumnFileTinyVectorIndexReader
6262
const ANNQueryInfoPtr & ann_query_info_,
6363
const BitmapFilterView && valid_rows_,
6464
const ColumnDefine & vec_cd_,
65-
const VectorIndexCachePtr & vec_index_cache_)
65+
const LocalIndexCachePtr & local_index_cache_)
6666
: tiny_file(tiny_file_)
6767
, data_provider(data_provider_)
6868
, ann_query_info(ann_query_info_)
6969
, valid_rows(std::move(valid_rows_))
7070
, vec_cd(vec_cd_)
71-
, vec_index_cache(vec_index_cache_)
71+
, local_index_cache(local_index_cache_)
7272
, log(Logger::get())
7373
{}
7474

‎dbms/src/Storages/DeltaMerge/ConcatSkippableBlockInputStream.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
#pragma once
1616

1717
#include <Flash/ResourceControl/LocalAdmissionController.h>
18-
#include <Storages/DeltaMerge/Index/VectorIndex_fwd.h>
1918
#include <Storages/DeltaMerge/ScanContext_fwd.h>
2019
#include <Storages/DeltaMerge/SkippableBlockInputStream.h>
2120
#include <Storages/DeltaMerge/VectorIndexBlockInputStream.h>

‎dbms/src/Storages/DeltaMerge/File/DMFile.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ class DMFile : private boost::noncopyable
191191
*/
192192
ColumnDefines getColumnDefines(bool sort_by_id = true) const
193193
{
194-
ColumnDefines results{};
194+
ColumnDefines results;
195195
results.reserve(this->meta->column_stats.size());
196196
for (const auto & cs : this->meta->column_stats)
197197
{

‎dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ DMFileBlockInputStreamBuilder::DMFileBlockInputStreamBuilder(const Context & con
3030
setCaches(
3131
global_context.getMarkCache(),
3232
global_context.getMinMaxIndexCache(),
33-
global_context.getVectorIndexCache(),
33+
global_context.getLocalIndexCache(),
3434
global_context.getColumnCacheLongTerm());
3535
// init from settings
3636
setFromSettings(context.getSettingsRef());
@@ -229,7 +229,7 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(
229229
std::move(rest_columns_reader),
230230
std::move(vec_column.value()),
231231
scan_context,
232-
vector_index_cache,
232+
local_index_cache,
233233
bitmap_filter.value(),
234234
tracing_id);
235235

0 commit comments

Comments
 (0)