WIP

```
ERROR 2024-12-16 12:15:55,115 [shard 0:main] crash_tracker - service.cc:107 - Crash loop detected. Too many consecutive crashes 6, exceeded crash_loop_limit configured value 5. To recover Redpanda from this state, manually remove file at path "/var/lib/redpanda/data/startup_log". Crash loop automatically resets 1h after last crash or with node configuration changes. The following crashes have been recorded:
Crash #1 at 2024-12-16 12:14:03 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
Crash #2 at 2024-12-16 12:14:09 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
Crash #3 at 2024-12-16 12:14:14 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
Crash #4 at 2024-12-16 12:14:20 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
Crash redpanda-data#5 at 2024-12-16 12:14:26 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
Crash redpanda-data#6 at 2024-12-16 12:14:32 - Segmentation fault on shard 0. Backtrace: 0xab575a3 0x5400cdb 0x5409de2 0x376b120 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libc.so.6+0x11e88c 0xa95055d 0xa93d12e 0xa8ab47d 0xa8a8316 0xa78cf74 0xa78ae70 0x35a0393 0xadce1b9 /lib/x86_64-linux-gnu/libc.so.6+0x29d8f /lib/x86_64-linux-gnu/libc.so.6+0x29e3f 0x3598be4
```

Author: pgellert

src/v/CMakeLists.txt

@@ -136,6 +136,7 @@ add_subdirectory(datalake)
 add_subdirectory(cloud_io)
 add_subdirectory(cloud_topics)
 add_subdirectory(schema)
+add_subdirectory(crash_tracker)
 
 option(ENABLE_GIT_VERSION "Build with Git metadata" OFF)
 

src/v/config/node_config.h

@@ -121,6 +121,10 @@ struct node_config final : public config_store {
         return data_directory().path / "startup_log";
     }
 
+    std::filesystem::path crash_report_dir_path() const {
+        return data_directory().path / "crash_reports";
+    }
+
     /**
      * Return the configured cache path if set, otherwise a default
      * path within the data directory.

src/v/crash_tracker/BUILD

@@ -0,0 +1,38 @@
+load("//bazel:build.bzl", "redpanda_cc_library")
+
+redpanda_cc_library(
+    name = "crash_tracker",
+    srcs = [
+        "api.cc",
+        "logger.cc",
+        "service.cc",
+    ],
+    hdrs = [
+        "api.h",
+        "logger.h",
+        "service.h",
+        "types.h",
+    ],
+    implementation_deps = [
+        "//src/v/bytes",
+        "//src/v/bytes:iobuf",
+        "//src/v/config",
+        "//src/v/hashing:xx",
+        "//src/v/model",
+        "//src/v/serde",
+        "//src/v/serde:bytes",
+        "//src/v/serde:iobuf",
+        "//src/v/serde:sstring",
+        "//src/v/serde:vector",
+        "//src/v/utils:file_io",
+        "@fmt",
+    ],
+    include_prefix = "crash_tracker",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//src/v/base",
+        "//src/v/model",
+        "//src/v/serde",
+        "@seastar",
+    ],
+)

src/v/crash_tracker/CMakeLists.txt

@@ -0,0 +1,14 @@
+v_cc_library(
+  NAME crash_tracker
+  SRCS
+    api.cc
+    service.cc
+    logger.cc
+  DEPS
+    Seastar::seastar
+    v::base
+    v::bytes
+    v::model
+    v::serde
+    v::hashing
+)

src/v/crash_tracker/api.cc

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.md
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0
+ */
+
+#include "crash_tracker/api.h"
+
+#include "base/vassert.h"
+#include "config/node_config.h"
+#include "crash_tracker/logger.h"
+#include "crash_tracker/service.h"
+#include "hashing/xx.h"
+#include "serde/rw/envelope.h"
+#include "utils/file_io.h"
+
+#include <seastar/core/memory.hh>
+#include <seastar/util/print_safe.hh>
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <iterator>
+#include <stdio.h>
+
+using namespace std::chrono_literals;
+
+namespace crash_tracker {
+
+namespace {
+
+service& instance() {
+    static service inst;
+    return inst;
+}
+
+} // namespace
+
+ss::future<> initialize() { return instance().start(); }
+
+ss::future<> record_clean_shutdown() { return instance().stop(); }
+
+namespace {
+std::string_view failure_msg_prefix(int signo) {
+    if (signo == SIGSEGV) {
+        return "Segmentation fault";
+    } else if (signo == SIGILL) {
+        return "Illegal instruction";
+    } else {
+        return "Aborting";
+    }
+}
+} // namespace
+
+void record_signal_crash(crash_description& cd, int signo) {
+    auto& format_buf = cd._crash_reason;
+    fmt::format_to_n(
+      format_buf.begin(),
+      format_buf.size(),
+      "{} on shard {}. Backtrace: {}",
+      failure_msg_prefix(signo),
+      ss::this_shard_id(),
+      ss::current_backtrace());
+
+    ss::backtrace([&cd](ss::frame f) { cd._stacktrace.push_back(f.addr); });
+}
+
+void record_sigsegv_crash() {
+    instance().record_crash(
+      [](crash_description& cd) { record_signal_crash(cd, SIGSEGV); });
+}
+
+void record_sigabrt_crash() {
+    instance().record_crash(
+      [](crash_description& cd) { record_signal_crash(cd, SIGABRT); });
+}
+
+void record_sigill_crash() {
+    instance().record_crash(
+      [](crash_description& cd) { record_signal_crash(cd, SIGILL); });
+}
+
+} // namespace crash_tracker

src/v/crash_tracker/api.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.md
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0
+ */
+
+#pragma once
+
+#include "base/seastarx.h"
+
+#include <seastar/core/future.hh>
+
+namespace crash_tracker {
+
+/// Should be called on startup before any call to record_crash, otherwise
+/// record_crash is a noop.
+/// Here we check for too many consecutive unclean
+/// shutdowns/crashes and abort the startup sequence if the limit exceeds
+/// crash_loop_limit until the operator intervenes. Crash tracking
+/// is reset if the node configuration changes or its been 1h since
+/// the broker last failed to start. This metadata is tracked in the
+/// tracker file. This is to prevent on disk state from piling up in
+/// each unclean run and creating more state to recover for the next run.
+ss::future<> initialize();
+
+/// On a clean shutdown,
+/// the tracker file should be deleted thus reseting the crash count on the
+/// next run. In case of an unclean shutdown, we already bumped
+/// the crash count and that should be taken into account in the
+/// next run.
+ss::future<> record_clean_shutdown();
+
+/// Async-signal safe
+void record_sigsegv_crash();
+
+/// Async-signal safe
+void record_sigabrt_crash();
+
+/// Async-signal safe
+void record_sigill_crash();
+
+} // namespace crash_tracker

src/v/crash_tracker/logger.cc

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.md
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0
+ */
+#include "logger.h"
+
+namespace crash_tracker {
+
+seastar::logger ctlog("crash_tracker");
+
+} // namespace crash_tracker

src/v/crash_tracker/logger.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.md
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0
+ */
+
+#pragma once
+
+#include "base/seastarx.h"
+
+#include <seastar/util/log.hh>
+
+namespace crash_tracker {
+extern ss::logger ctlog;
+} // namespace crash_tracker