chore: symbolize stack traces in tests upon crash

We disable address space randomization when building the binary
and use addr2line to symbolize the stacktrace if it exists.

Signed-off-by: Roman Gershman <[email protected]>

Author: romange

.github/workflows/ci.yml

@@ -100,13 +100,14 @@ jobs:
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
         run: |
+          # -no-pie to disable address randomization so we could symbolize stacktraces
           cmake -B ${GITHUB_WORKSPACE}/build \
             -DCMAKE_BUILD_TYPE=${{matrix.build-type}} \
             -GNinja \
             -DCMAKE_C_COMPILER="${{matrix.compiler.c}}" \
             -DCMAKE_CXX_COMPILER="${{matrix.compiler.cxx}}" \
             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache \
-            -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}}" -DWITH_AWS:BOOL=OFF \
+            -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}} -no-pie" -DWITH_AWS:BOOL=OFF \
             -L
           cd ${GITHUB_WORKSPACE}/build && pwd
           du -hcs _deps/

.github/workflows/regression-tests.yml

@@ -29,8 +29,10 @@ jobs:
 
       - name: Configure & Build
         run: |
+          # -no-pie to disable address randomization so we could symbolize stacktraces
           cmake -B ${GITHUB_WORKSPACE}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -GNinja \
-                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DPRINT_STACKTRACES_ON_SIGNAL=ON
+                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DPRINT_STACKTRACES_ON_SIGNAL=ON \
+                -DCMAKE_CXX_FLAGS=-no-pie
 
           cd ${GITHUB_WORKSPACE}/build  && ninja dragonfly
           pwd

tests/dragonfly/connection_test.py

@@ -707,6 +707,8 @@ async def test_unix_domain_socket(df_factory, tmp_dir):
 
 async def test_unix_socket_only(df_factory, tmp_dir):
     server = df_factory.create(proactor_threads=1, port=0, unixsocket="./df.sock")
+    # we call _start because we start() wait for the port to become available and
+    # we run here a process without a port.
     server._start()
 
     await asyncio.sleep(1)

tests/dragonfly/instance.py

@@ -47,6 +47,22 @@ class DflyStartException(Exception):
     pass
 
 
+def symbolize_stack_trace(binary_path, lines):
+    pattern = rb"@\s*(0x[0-9a-fA-F]+)"
+    matcher = re.compile(pattern)
+    addr2line_proc = subprocess.Popen(
+        ["/usr/bin/addr2line", "-fCa", "-e", binary_path], stdin=subprocess.PIPE
+    )
+    for line in lines:
+        res = matcher.search(line)
+        if res:
+            g = res.group(1) + b"\n"
+            addr2line_proc.stdin.write(g)
+
+    addr2line_proc.stdin.close()
+    addr2line_proc.wait()
+
+
 class DflyInstance:
     """
     Represents a runnable and stoppable Dragonfly instance
@@ -60,8 +76,9 @@ def __init__(self, params: DflyParams, args):
         self.proc: Optional[subprocess.Popen] = None
         self._client: Optional[RedisClient] = None
         self.log_files: List[str] = []
-
         self.dynamic_port = False
+        self.sed_proc = None
+
         if self.params.existing_port:
             self._port = self.params.existing_port
         elif "port" in self.args:
@@ -150,7 +167,7 @@ def _wait_for_server(self):
         sed_cmd = ["sed", "-u", "-e", sed_format]
         if self.params.buffered_out:
             sed_cmd.remove("-u")
-        subprocess.Popen(sed_cmd, stdin=self.proc.stdout)
+        self.sed_proc = subprocess.Popen(sed_cmd, stdin=self.proc.stdout, stdout=subprocess.PIPE)
 
     def set_proc_to_none(self):
         self.proc = None
@@ -188,6 +205,18 @@ def stop(self, kill=False):
             proc.kill()
             proc.communicate()
             raise Exception("Unable to terminate DragonflyDB gracefully, it was killed")
+        finally:
+            if self.sed_proc:
+                sed_out = self.sed_proc.stdout.readlines()
+
+                # Deduplicate output - we somewhere duplicate the output, probably due
+                # to tty redirections.
+                seen = set()
+                sed_out = [x for x in sed_out if not (x in seen or seen.add(x))]
+
+                for str in sed_out:
+                    print(str.decode())
+                symbolize_stack_trace(proc.args[0], sed_out)
 
     def _start(self):
         if self.params.existing_port: