#!/usr/bin/env python3 """ Script to run a matrix of tests via gdb via a running OpenOCD capturing UART output and checking for "PASSED" - execution stops with a timeout or when GDB halts (e.g. breakpoint) It requires 'grabserial'. Iterates over path_segments and elf_filenames. For every combination where /// exists: - Flashes it via OpenOCD (program + verify + reset + exit) - Captures UART output using grabserial - Saves to /..out At the end: - Checks that the last line of every generated .out file is exactly "PASSED" (after strip) - Prints success count / attempted count - Exits with 0 only if ALL attempted tests succeeded, else 1 Example usage: $ tools/run_test_matrix.py \ --path-segments \ gcc_build \ clang_build \ --elf-filenames \ custom_float_funcs_test_compiler.elf \ custom_float_funcs_test_pico_dcp.elf \ custom_float_funcs_test_pico.elf \ custom_float_funcs_test_pico_vfp.elf \ --path-postfix \ test/pico_float_test \ -- \ out_dir Which will run each of the following if they exist: gcc_build/test/pico_float_test/custom_float_funcs_test_compiler.elf gcc_build/test/pico_float_test/custom_float_funcs_test_pico_dcp.elf gcc_build/test/pico_float_test/custom_float_funcs_test_pico.elf gcc_build/test/pico_float_test/custom_float_funcs_test_pico_vfp.elf clang_build/test/pico_float_test/custom_float_funcs_test_compiler.elf clang_build/test/pico_float_test/custom_float_funcs_test_pico_dcp.elf clang_build/test/pico_float_test/custom_float_funcs_test_pico.elf clang_build/test/pico_float_test/custom_float_funcs_test_pico_vfp.elf storing their UART output in: out_dir/gcc_build.custom_float_funcs_test_compiler.elf.out out_dir/gcc_build.custom_float_funcs_test_pico_dcp.elf.out out_dir/gcc_build.custom_float_funcs_test_pico.elf.out out_dir/gcc_build.custom_float_funcs_test_pico_vfp.elf.out out_dir/clang_build.custom_float_funcs_test_compiler.elf.out out_dir/clang_build.custom_float_funcs_test_pico_dcp.elf.out out_dir/clang_build.custom_float_funcs_test_pico.elf.out out_dir/clang_build.custom_float_funcs_test_pico_vfp.elf.out This would run the same tests: (but store the output in e.g out_dir/gcc_build.pico_float_test.custom_float_funcs_test_compiler.elf.out ) $ tools/run_test_matrix.py \ --path-segments \ gcc_build \ clang_build \ --elf-filenames \ pico_float_test/custom_float_funcs_test_compiler.elf \ pico_float_test/custom_float_funcs_test_pico_dcp.elf \ pico_float_test/custom_float_funcs_test_pico.elf \ pico_float_test/custom_float_funcs_test_pico_vfp.elf \ --path-postfix \ test \ -- \ out_dir """ import argparse import os import subprocess import sys import time import signal def main(): parser = argparse.ArgumentParser(description="RP2xxx ELF test runner with OpenOCD + grabserial UART capture") parser.add_argument("output_dir", help="Directory where *.out files will be written (will be created if missing)") parser.add_argument("--path-segments", nargs="+", required=True, help="Space-separated list of path segments; e.g. foo, bar, foo/bar/humbug") parser.add_argument("--path-prefix", default=".", help="Path prefix") parser.add_argument("--path-postfix", default="", help="Path postfix") parser.add_argument("--gdb", default="arm-none-eabi-gdb", help="gdb executable") parser.add_argument("--gdb-timeout", type=int, default=60, help="gdb timeout for executable if it doesn't finish") parser.add_argument("--openocd-server", default="localhost:3333", help="openocd server address:port"), parser.add_argument("--elf-filenames", nargs="+", required=True, help="Space-separated list of ELF filenames (including .elf), e.g. test1.elf test2.elf") parser.add_argument("--serial-port", default="/dev/ttyACM0", help="UART device (default: /dev/ttyACM0)") parser.add_argument("--baud", type=int, default=115200, help="UART baud rate (default: 115200)") args = parser.parse_args() os.makedirs(args.output_dir, exist_ok=True) attempted = 0 succeeded = 0 for seg in args.path_segments: for elf in args.elf_filenames: full_path = os.path.join(args.path_prefix, seg, args.path_postfix, elf) if not os.path.isfile(full_path): print(f"\n=== Skipping missing ELF {full_path} ===") continue attempted += 1 sanitized_out_file = f"{seg}.{elf}.out".replace("/",".") outfile = os.path.join(args.output_dir, sanitized_out_file) print(f"\n=== Processing {seg}/{elf} ===") print(f" ELF: {full_path}") print(f" Out: {outfile}") # ====================== START GRABSERIAL EARLY FOR FULL UART CAPTURE ====================== print(f" Starting grabserial capture early on {args.serial_port} @ {args.baud} baud...") logger_proc = None try: logger_proc = subprocess.Popen( [ "grabserial", "-d", args.serial_port, "-b", str(args.baud), "-o", outfile, # output file "--quiet", # suppress extra messages ], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, preexec_fn=os.setsid, ) print(f" grabserial started (PID {logger_proc.pid}) — capturing from now on") # Give grabserial a moment to open the port and settle time.sleep(0.5) # ====================== FLASH + START VIA GDB ====================== print(" Flashing and running via GDB (breakpoint signals test end)...") gdb_cmd = [ args.gdb, "--batch", "--nx", full_path, "-ex", f"target extended-remote {args.openocd_server}", "-ex", "monitor reset init", "-ex", "load", "-ex", "monitor verify", # optional "-ex", "continue", ] gdb_result = subprocess.run( gdb_cmd, check=False, timeout=args.gdb_timeout, capture_output=True, text=True, encoding="utf-8", errors="replace" ) # Show GDB output always (super useful for diagnosing) combined_output = gdb_result.stdout + "\n" + gdb_result.stderr if combined_output.strip(): print(" GDB output:") print(" ────────────────────────────────────────────────") print(combined_output.strip()) print(" ────────────────────────────────────────────────") if "breakpoint" in combined_output: print(" ✓ Breakpoint hit → test finished normally") else: print(" Note: No breakpoint detected in GDB output") # Optional: tiny extra wait for any trailing UART bytes after breakpoint time.sleep(0.5) except FileNotFoundError: print(" ERROR: grabserial or gdb not found in PATH") sys.exit(1) except subprocess.TimeoutExpired: print(" GDB timed out") # fall through to kill grabserial finally: # ====================== CLEANLY STOP GRABSERIAL ====================== if logger_proc is not None: try: print(" Stopping grabserial...") os.killpg(os.getpgid(logger_proc.pid), signal.SIGTERM) logger_proc.wait(timeout=5) print(" grabserial stopped cleanly") except ProcessLookupError: print(" grabserial already exited") except Exception as e: print(f" Error stopping grabserial: {e}") # forceful kill if needed try: os.killpg(os.getpgid(logger_proc.pid), signal.SIGKILL) except: pass # ====================== CHECK WE CAPTURED SOME OUTPUT ====================== if not os.path.isfile(outfile) or os.path.getsize(outfile) == 0: print(" No output captured") continue # ====================== PRINT THE CAPTURED OUTPUT TO CONSOLE ====================== print("\n" + "="*60) print(f"Full captured UART output from {seg}/{elf}:") print("="*60) try: with open(outfile, "r", encoding="utf-8", errors="replace") as f: captured_content = f.read() if captured_content.strip(): print(captured_content.rstrip()) # rstrip to avoid trailing newlines else: print("(No output captured)") except FileNotFoundError: print("Output file not found — capture may have failed") except Exception as e: print(f"Error reading output file: {e}") print("="*60 + "\n") # ====================== CHECK LAST LINE ====================== try: with open(outfile, "r", encoding="utf-8", errors="ignore") as f: lines = f.readlines() last_line = lines[-1].strip() if lines else "" if last_line == "PASSED": succeeded += 1 print(" ✓ PASSED") else: print(f" ✗ FAILED (last line: '{last_line}')") except Exception as e: print(f" Failed to read output file: {e}") # ====================== FINAL SUMMARY ====================== print("\n" + "="*60) print(f"TEST SUMMARY: {succeeded}/{attempted} succeeded") print("="*60) if attempted == 0: print("No ELFs were found to test.") elif succeeded == attempted: print("All tests passed!") else: print("Some tests failed.") sys.exit(1) if __name__ == "__main__": main()