chore: symbolize stack traces in tests upon crash

We disable address space randomization when building the binary
and use addr2line to symbolize the stacktrace if it exists.

Signed-off-by: Roman Gershman <[email protected]>

Author: romange

.github/workflows/ci.yml

@@ -100,13 +100,14 @@ jobs:
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
         run: |
+          # -no-pie to disable address randomization so we could symbolize stacktraces
           cmake -B ${GITHUB_WORKSPACE}/build \
             -DCMAKE_BUILD_TYPE=${{matrix.build-type}} \
             -GNinja \
             -DCMAKE_C_COMPILER="${{matrix.compiler.c}}" \
             -DCMAKE_CXX_COMPILER="${{matrix.compiler.cxx}}" \
             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache \
-            -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}}" -DWITH_AWS:BOOL=OFF \
+            -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}} -no-pie" -DWITH_AWS:BOOL=OFF \
             -L
           cd ${GITHUB_WORKSPACE}/build && pwd
           du -hcs _deps/

.github/workflows/regression-tests.yml

@@ -29,8 +29,10 @@ jobs:
 
       - name: Configure & Build
         run: |
+          # -no-pie to disable address randomization so we could symbolize stacktraces
           cmake -B ${GITHUB_WORKSPACE}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -GNinja \
-                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DPRINT_STACKTRACES_ON_SIGNAL=ON
+                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DPRINT_STACKTRACES_ON_SIGNAL=ON \
+                -DCMAKE_CXX_FLAGS=-no-pie
 
           cd ${GITHUB_WORKSPACE}/build  && ninja dragonfly
           pwd

tests/dragonfly/instance.py

@@ -47,6 +47,22 @@ class DflyStartException(Exception):
     pass
 
 
+def symbolize_stack_trace(binary_path, lines):
+    pattern = rb"@\s*(0x[0-9a-fA-F]+)"
+    matcher = re.compile(pattern)
+    addr2line_proc = subprocess.Popen(
+        ["/usr/bin/addr2line", "-fCa", "-e", binary_path], stdin=subprocess.PIPE
+    )
+    for line in lines:
+        res = matcher.search(line)
+        if res:
+            g = res.group(1) + b"\n"
+            addr2line_proc.stdin.write(g)
+
+    addr2line_proc.stdin.close()
+    addr2line_proc.wait()
+
+
 class DflyInstance:
     """
     Represents a runnable and stoppable Dragonfly instance
@@ -150,7 +166,7 @@ def _wait_for_server(self):
         sed_cmd = ["sed", "-u", "-e", sed_format]
         if self.params.buffered_out:
             sed_cmd.remove("-u")
-        subprocess.Popen(sed_cmd, stdin=self.proc.stdout)
+        self.sed_proc = subprocess.Popen(sed_cmd, stdin=self.proc.stdout, stdout=subprocess.PIPE)
 
     def set_proc_to_none(self):
         self.proc = None
@@ -188,6 +204,17 @@ def stop(self, kill=False):
             proc.kill()
             proc.communicate()
             raise Exception("Unable to terminate DragonflyDB gracefully, it was killed")
+        finally:
+            sed_out = self.sed_proc.stdout.readlines()
+
+            # Deduplicate output - we somewhere duplicate the output, probably due
+            # to tty redirections.
+            seen = set()
+            sed_out = [x for x in sed_out if not (x in seen or seen.add(x))]
+
+            for str in sed_out:
+                print(str.decode())
+            symbolize_stack_trace(proc.args[0], sed_out)
 
     def _start(self):
         if self.params.existing_port: