pico-sdk/tools/run_test_matrix.py

#!/usr/bin/env python3
"""
Script to run a matrix of tests via gdb via a running OpenOCD capturing UART output and checking for "PASSED" -
execution stops with a timeout or when GDB halts (e.g. breakpoint)

It requires 'grabserial'.

Iterates over path_segments and elf_filenames.

For every combination where <path_prefix>/<path_segment>/<path_postfix>/<elf_filename> exists:
  - Flashes it via OpenOCD (program + verify + reset + exit)
  - Captures UART output using grabserial
  - Saves to <output_dir>/<sanitized_path_segment>.<elf_filename>.out

At the end:
  - Checks that the last line of every generated .out file is exactly "PASSED" (after strip)
  - Prints success count / attempted count
  - Exits with 0 only if ALL attempted tests succeeded, else 1

Example usage:

$ tools/run_test_matrix.py \
    --path-segments \
        gcc_build \
        clang_build \
    --elf-filenames \
        custom_float_funcs_test_compiler.elf \
        custom_float_funcs_test_pico_dcp.elf \
        custom_float_funcs_test_pico.elf \
        custom_float_funcs_test_pico_vfp.elf \
    --path-postfix \
       test/pico_float_test \
    -- \
    out_dir

Which will run each of the following if they exist:
   gcc_build/test/pico_float_test/custom_float_funcs_test_compiler.elf
   gcc_build/test/pico_float_test/custom_float_funcs_test_pico_dcp.elf
   gcc_build/test/pico_float_test/custom_float_funcs_test_pico.elf
   gcc_build/test/pico_float_test/custom_float_funcs_test_pico_vfp.elf
   clang_build/test/pico_float_test/custom_float_funcs_test_compiler.elf
   clang_build/test/pico_float_test/custom_float_funcs_test_pico_dcp.elf
   clang_build/test/pico_float_test/custom_float_funcs_test_pico.elf
   clang_build/test/pico_float_test/custom_float_funcs_test_pico_vfp.elf
storing their UART output in:
   out_dir/gcc_build.custom_float_funcs_test_compiler.elf.out
   out_dir/gcc_build.custom_float_funcs_test_pico_dcp.elf.out
   out_dir/gcc_build.custom_float_funcs_test_pico.elf.out
   out_dir/gcc_build.custom_float_funcs_test_pico_vfp.elf.out
   out_dir/clang_build.custom_float_funcs_test_compiler.elf.out
   out_dir/clang_build.custom_float_funcs_test_pico_dcp.elf.out
   out_dir/clang_build.custom_float_funcs_test_pico.elf.out
   out_dir/clang_build.custom_float_funcs_test_pico_vfp.elf.out

This would run the same tests:
(but store the output in e.g out_dir/gcc_build.pico_float_test.custom_float_funcs_test_compiler.elf.out )

$ tools/run_test_matrix.py \
    --path-segments \
        gcc_build \
        clang_build \
    --elf-filenames \
        pico_float_test/custom_float_funcs_test_compiler.elf \
        pico_float_test/custom_float_funcs_test_pico_dcp.elf \
        pico_float_test/custom_float_funcs_test_pico.elf \
        pico_float_test/custom_float_funcs_test_pico_vfp.elf \
    --path-postfix \
       test \
    -- \
    out_dir
"""

import argparse
import os
import subprocess
import sys
import time
import signal

def main():
    parser = argparse.ArgumentParser(description="RP2xxx ELF test runner with OpenOCD + grabserial UART capture")
    parser.add_argument("output_dir", help="Directory where *.out files will be written (will be created if missing)")
    parser.add_argument("--path-segments", nargs="+", required=True,
                        help="Space-separated list of path segments; e.g. foo, bar, foo/bar/humbug")
    parser.add_argument("--path-prefix", default=".",
                        help="Path prefix")
    parser.add_argument("--path-postfix", default="",
                        help="Path postfix")
    parser.add_argument("--gdb", default="arm-none-eabi-gdb",
                        help="gdb executable")
    parser.add_argument("--gdb-timeout", type=int, default=60,
                        help="gdb timeout for executable if it doesn't finish")
    parser.add_argument("--openocd-server", default="localhost:3333",
                        help="openocd server address:port"),
    parser.add_argument("--elf-filenames", nargs="+", required=True,
                        help="Space-separated list of ELF filenames (including .elf), e.g. test1.elf test2.elf")
    parser.add_argument("--serial-port", default="/dev/ttyACM0", help="UART device (default: /dev/ttyACM0)")
    parser.add_argument("--baud", type=int, default=115200, help="UART baud rate (default: 115200)")

    args = parser.parse_args()

    os.makedirs(args.output_dir, exist_ok=True)

    attempted = 0
    succeeded = 0

    for seg in args.path_segments:
        for elf in args.elf_filenames:
            full_path = os.path.join(args.path_prefix, seg, args.path_postfix, elf)
            if not os.path.isfile(full_path):
                print(f"\n=== Skipping missing ELF {full_path} ===")
                continue

            attempted += 1
            sanitized_out_file = f"{seg}.{elf}.out".replace("/",".")
            outfile = os.path.join(args.output_dir, sanitized_out_file)

            print(f"\n=== Processing {seg}/{elf} ===")
            print(f"   ELF: {full_path}")
            print(f"   Out: {outfile}")

            # ====================== START GRABSERIAL EARLY FOR FULL UART CAPTURE ======================
            print(f"   Starting grabserial capture early on {args.serial_port} @ {args.baud} baud...")

            logger_proc = None
            try:
                logger_proc = subprocess.Popen(
                    [
                        "grabserial",
                        "-d", args.serial_port,
                        "-b", str(args.baud),
                        "-o", outfile,          # output file
                        "--quiet",             # suppress extra messages
                    ],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.PIPE,
                    preexec_fn=os.setsid,
                )
                print(f"   grabserial started (PID {logger_proc.pid}) — capturing from now on")

                # Give grabserial a moment to open the port and settle
                time.sleep(0.5)

                # ====================== FLASH + START VIA GDB ======================
                print("   Flashing and running via GDB (breakpoint signals test end)...")

                gdb_cmd = [
                    args.gdb, "--batch", "--nx", full_path,
                    "-ex", f"target extended-remote {args.openocd_server}",
                    "-ex", "monitor reset init",
                    "-ex", "load",
                    "-ex", "monitor verify",   # optional
                    "-ex", "continue",
                ]

                gdb_result = subprocess.run(
                    gdb_cmd,
                    check=False,
                    timeout=args.gdb_timeout,
                    capture_output=True,
                    text=True, encoding="utf-8", errors="replace"
                )

                # Show GDB output always (super useful for diagnosing)
                combined_output = gdb_result.stdout + "\n" + gdb_result.stderr
                if combined_output.strip():
                    print("   GDB output:")
                    print("   ────────────────────────────────────────────────")
                    print(combined_output.strip())
                    print("   ────────────────────────────────────────────────")

                if "breakpoint" in combined_output:
                    print("   ✓ Breakpoint hit → test finished normally")
                else:
                    print("   Note: No breakpoint detected in GDB output")

                # Optional: tiny extra wait for any trailing UART bytes after breakpoint
                time.sleep(0.5)

            except FileNotFoundError:
                print("   ERROR: grabserial or gdb not found in PATH")
                sys.exit(1)
            except subprocess.TimeoutExpired:
                print("   GDB timed out")
                # fall through to kill grabserial
            finally:
                # ====================== CLEANLY STOP GRABSERIAL ======================
                if logger_proc is not None:
                    try:
                        print("   Stopping grabserial...")
                        os.killpg(os.getpgid(logger_proc.pid), signal.SIGTERM)
                        logger_proc.wait(timeout=5)
                        print("   grabserial stopped cleanly")
                    except ProcessLookupError:
                        print("   grabserial already exited")
                    except Exception as e:
                        print(f"   Error stopping grabserial: {e}")
                        # forceful kill if needed
                        try:
                            os.killpg(os.getpgid(logger_proc.pid), signal.SIGKILL)
                        except:
                            pass

            # ====================== CHECK WE CAPTURED SOME OUTPUT ======================
            if not os.path.isfile(outfile) or os.path.getsize(outfile) == 0:
                print("   No output captured")
                continue

            # ====================== PRINT THE CAPTURED OUTPUT TO CONSOLE ======================
            print("\n" + "="*60)
            print(f"Full captured UART output from {seg}/{elf}:")
            print("="*60)

            try:
                with open(outfile, "r", encoding="utf-8", errors="replace") as f:
                    captured_content = f.read()

                if captured_content.strip():
                    print(captured_content.rstrip())   # rstrip to avoid trailing newlines
                else:
                    print("(No output captured)")

            except FileNotFoundError:
                print("Output file not found — capture may have failed")
            except Exception as e:
                print(f"Error reading output file: {e}")

            print("="*60 + "\n")
            # ====================== CHECK LAST LINE ======================
            try:
                with open(outfile, "r", encoding="utf-8", errors="ignore") as f:
                    lines = f.readlines()
                    last_line = lines[-1].strip() if lines else ""
                if last_line == "PASSED":
                    succeeded += 1
                    print("   ✓ PASSED")
                else:
                    print(f"   ✗ FAILED (last line: '{last_line}')")
            except Exception as e:
                print(f"   Failed to read output file: {e}")

    # ====================== FINAL SUMMARY ======================
    print("\n" + "="*60)
    print(f"TEST SUMMARY: {succeeded}/{attempted} succeeded")
    print("="*60)

    if attempted == 0:
        print("No ELFs were found to test.")
    elif succeeded == attempted:
        print("All tests passed!")
    else:
        print("Some tests failed.")
        sys.exit(1)


if __name__ == "__main__":
    main()