PageCtl: Support getting blob data from PageStorage instance (release-7.5) (#9717)

close #9716

PageCtl: Support getting blob data from PageStorage instance
And output the "PageEntryV3.offset" in decimal instead of hexadecimal

Signed-off-by: JaySon-Huang <[email protected]>

Author: JaySon-Huang

dbms/src/Storages/Page/V3/PageEntry.h

@@ -130,7 +130,7 @@ struct fmt::formatter<DB::PS::V3::PageEntryV3>
 
         return format_to(
             ctx.out(),
-            "PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
+            "PageEntry{{file: {}, offset: {}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
             "checkpoint_info: {}}}",
             entry.file_id,
             entry.offset,

dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp

@@ -17,10 +17,13 @@
 #include <Poco/ConsoleChannel.h>
 #include <Poco/PatternFormatter.h>
 #include <Server/CLIService.h>
+#include <Storages/Page/PageDefinesBase.h>
+#include <Storages/Page/V3/PageDefines.h>
 #include <Storages/Page/V3/PageDirectory.h>
 #include <Storages/Page/V3/PageDirectoryFactory.h>
 #include <Storages/Page/V3/PageStorageImpl.h>
 #include <Storages/Page/V3/Universal/RaftDataReader.h>
+#include <Storages/Page/V3/Universal/UniversalPageId.h>
 #include <Storages/Page/V3/Universal/UniversalPageIdFormatImpl.h>
 #include <Storages/Page/V3/Universal/UniversalPageStorage.h>
 #include <Storages/PathPool.h>
@@ -29,6 +32,7 @@
 #include <common/types.h>
 
 #include <boost/program_options.hpp>
+#include <cstdint>
 #include <magic_enum.hpp>
 #include <unordered_set>
 
@@ -47,12 +51,15 @@ struct ControlOptions
         CHECK_ALL_DATA_CRC = 4,
         DISPLAY_WAL_ENTRIES = 5,
         DISPLAY_REGION_INFO = 6,
+        DISPLAY_BLOB_DATA = 7,
     };
 
     std::vector<std::string> paths;
     DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO;
     UInt64 page_id = UINT64_MAX;
-    UInt32 blob_id = UINT32_MAX;
+    BlobFileId blob_id = INVALID_BLOBFILE_ID;
+    BlobFileOffset blob_offset = INVALID_BLOBFILE_OFFSET;
+    size_t blob_size = UINT64_MAX;
     UInt64 namespace_id = DB::TEST_NAMESPACE_ID;
     StorageType storage_type = StorageType::Unknown; // only useful for universal page storage
     UInt32 keyspace_id = NullspaceID; // only useful for universal page storage
@@ -85,6 +92,7 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
  4 is check every data is valid
  5 is dump entries in WAL log files
  6 is display all region info
+ 7 is display blob data (in hex)
 )") //
         ("show_entries",
          value<bool>()->default_value(true),
@@ -106,8 +114,14 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
          value<UInt64>()->default_value(UINT64_MAX),
          "Query a single Page id, and print its version chain.") //
         ("blob_id,B",
-         value<UInt32>()->default_value(UINT32_MAX),
-         "Query a single Blob id, and print its data distribution.") //
+         value<BlobFileId>()->default_value(INVALID_BLOBFILE_ID),
+         "Specify the blob_id") //
+        ("blob_offset",
+         value<BlobFileOffset>()->default_value(INVALID_BLOBFILE_OFFSET),
+         "Specify the offset.") //
+        ("blob_size",
+         value<size_t>()->default_value(0),
+         "Specify the size.") //
         //
         ("imitative,I",
          value<bool>()->default_value(true),
@@ -140,7 +154,9 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
     opt.paths = options["paths"].as<std::vector<std::string>>();
     auto mode_int = options["mode"].as<int>();
     opt.page_id = options["page_id"].as<UInt64>();
-    opt.blob_id = options["blob_id"].as<UInt32>();
+    opt.blob_id = options["blob_id"].as<BlobFileId>();
+    opt.blob_offset = options["blob_offset"].as<BlobFileOffset>();
+    opt.blob_size = options["blob_size"].as<size_t>();
     opt.show_entries = options["show_entries"].as<bool>();
     opt.check_fields = options["check_fields"].as<bool>();
     auto storage_type_int = options["storage_type"].as<int>();
@@ -346,6 +362,12 @@ class PageStorageControlV3
                 }
                 break;
             }
+            case ControlOptions::DisplayType::DISPLAY_BLOB_DATA:
+            {
+                String hex_data = getBlobData(blob_store, opts.blob_id, opts.blob_offset, opts.blob_size);
+                fmt::print("hex:{}\n", hex_data);
+                break;
+            }
             default:
                 std::cout << "Invalid display mode." << std::endl;
                 break;
@@ -372,7 +394,7 @@ class PageStorageControlV3
         return 0;
     }
 
-    static String getBlobsInfo(typename Trait::BlobStore & blob_store, UInt32 blob_id)
+    static String getBlobsInfo(typename Trait::BlobStore & blob_store, BlobFileId blob_id)
     {
         auto stat_info = [](const BlobStats::BlobStatPtr & stat, const String & path) {
             FmtBuffer stat_str;
@@ -402,7 +424,7 @@ class PageStorageControlV3
         {
             for (const auto & stat : stats)
             {
-                if (blob_id != UINT32_MAX)
+                if (blob_id != INVALID_BLOBFILE_ID)
                 {
                     if (stat->id == blob_id)
                     {
@@ -416,7 +438,7 @@ class PageStorageControlV3
             }
         }
 
-        if (blob_id != UINT32_MAX)
+        if (blob_id != INVALID_BLOBFILE_ID)
         {
             stats_info.fmtAppend("    no found blob {}", blob_id);
         }
@@ -447,34 +469,41 @@ class PageStorageControlV3
             size_t count = 0;
             for (const auto & [version, entry_or_del] : versioned_entries->entries)
             {
-                const auto & entry = entry_or_del.entry;
-                page_str.fmtAppend(
-                    "      entry {}\n"
-                    "       sequence: {}\n"
-                    "       epoch: {}\n"
-                    "       is del: {}\n"
-                    "       blob id: {}\n"
-                    "       offset: {}\n"
-                    "       size: {}\n"
-                    "       crc: 0x{:X}\n", //
-                    count++, //
-                    version.sequence, //
-                    version.epoch, //
-                    entry_or_del.isDelete(), //
-                    entry.file_id, //
-                    entry.offset, //
-                    entry.size, //
-                    entry.checksum, //
-                    entry.field_offsets.size() //
-                );
-                if (!entry.field_offsets.empty())
+                if (entry_or_del.isEntry())
                 {
-                    page_str.append("          field offset:\n");
-                    for (const auto & [offset, crc] : entry.field_offsets)
+                    const auto & entry = entry_or_del.entry;
+                    page_str.fmtAppend(
+                        "      entry {}\n"
+                        "       sequence: {}\n"
+                        "       epoch: {}\n"
+                        "       is del: false\n"
+                        "       blob id: {}\n"
+                        "       offset: {}\n"
+                        "       size: {}\n"
+                        "       crc: 0x{:X}\n", //
+                        count++, //
+                        version.sequence, //
+                        version.epoch, //
+                        entry.file_id, //
+                        entry.offset, //
+                        entry.size, //
+                        entry.checksum, //
+                        entry.field_offsets.size() //
+                    );
+                    if (!entry.field_offsets.empty())
                     {
-                        page_str.fmtAppend("            offset: {} crc: 0x{:X}\n", offset, crc);
+                        page_str.append("          field offset:\n");
+                        for (const auto & [offset, crc] : entry.field_offsets)
+                        {
+                            page_str.fmtAppend("            offset: {} crc: 0x{:X}\n", offset, crc);
+                        }
+                        page_str.append("\n");
                     }
-                    page_str.append("\n");
+                }
+                else
+                {
+                    page_str.append("      entry is null\n"
+                                    "       is del: true\n");
                 }
             }
             return page_str.toString();
@@ -814,6 +843,32 @@ class PageStorageControlV3
         return error_msg.toString();
     }
 
+    static String getBlobData(
+        typename Trait::BlobStore & blob_store,
+        BlobFileId blob_id,
+        BlobFileOffset offset,
+        size_t size)
+    {
+        auto page_id = []() {
+            if constexpr (std::is_same_v<Trait, u128::PageStorageControlV3Trait>)
+                return PageIdV3Internal(0, 0);
+            else
+                return UniversalPageId("");
+        }();
+        char * buffer = new char[size];
+        blob_store.read(page_id, blob_id, offset, buffer, size, nullptr, false);
+
+        using ChecksumClass = Digest::CRC64;
+        ChecksumClass digest;
+        digest.update(buffer, size);
+        auto checksum = digest.checksum();
+        fmt::print("checksum: 0x{:X}\n", checksum);
+
+        auto hex_str = Redact::keyToHexString(buffer, size);
+        delete[] buffer;
+        return hex_str;
+    }
+
 private:
     ControlOptions options;
 };

release-centos7-llvm/env/.gitignore

@@ -1,3 +1,6 @@
 tiflash-env
 tiflash-env-*.tar.xz
+llvm-project
+sysroot
 *.log
+