From ead320990436a8b40fe3e4e5698d879c0ee7eb4d Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Thu, 3 Jul 2025 19:51:58 +0200 Subject: [PATCH 1/2] Add whence2yaml.py and helper library Add a script and helper library that convert the WHENCE file to YAML format and write the data to WHENCE.yaml. This can we be used as a starting point for making the WHENCE data better machine-parseable and also serves as a foundation for adding more relavant data for firmware consumers like distros. Signed-off-by: Juerg Haefliger --- contrib/pylib/whence.py | 179 ++++++++++++++++++++++++++++++++++++++++ contrib/whence2yaml.py | 12 +++ 2 files changed, 191 insertions(+) create mode 100644 contrib/pylib/whence.py create mode 100755 contrib/whence2yaml.py diff --git a/contrib/pylib/whence.py b/contrib/pylib/whence.py new file mode 100644 index 00000000..5d794278 --- /dev/null +++ b/contrib/pylib/whence.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# +# WHENCE data classes +# + +import json +import re +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from ruamel.yaml import YAML + + +def strip(obj: list) -> list: + """Remove leading and trailing empty strings from a list of strings""" + if obj: + while obj[0] == "": + obj = obj[1:] + while obj[-1] == "": + obj = obj[:-1] + return obj + + +@dataclass +class BaseWhence: + """Base WHENCE data class""" + + def __str__(self): + return json.dumps(asdict(self), indent=2) + + +@dataclass +class Entry(BaseWhence): + """WHENCE file/link entry class""" + + type: str = "" # File or RawFile or Link + path: str = "" # Firmware file or link + target: str = "" # Link target + sources: list[str] = field(default_factory=list) # List of source files + version: str = "" + info: str = "" + origin: str = "" + + +@dataclass +class Section(BaseWhence): + """WHENCE driver section class""" + + driver: str = "" + module: str = "" + description: str = "" + licence: list[str] = field(default_factory=list) + licence_file: str = "" + notes: list[str] = field(default_factory=list) + entries: list[Entry] = field(default_factory=list) + + +@dataclass +class Whence(BaseWhence): + """Main WHENCE data class""" + + header: list[str] = field(default_factory=list) + sections: list[Section] = field(default_factory=list) + + def load_whence(self, whence_file: Path) -> None: + """Load data from a WHENCE file""" + self.sections = [] + + HEADER = 0 + SECTION = 1 + NOTES = 2 + LICENCE = 3 + + with open(whence_file, encoding="utf-8") as fh: + whence_type = HEADER + for line in fh: + line = line.rstrip() + + if ":" in line: + key, val = line.split(":", 1) + val = val.strip() + else: + key = None + val = None + + if line.startswith("----------"): + whence_type = SECTION + section = Section() + self.sections.append(section) + continue + + if key == "Driver": + section.driver = val + if " " in val: + module, desc = val.split(" ", 1) + desc = desc.strip(" -") + else: + module = val + desc = "" + section.module = module + section.description = desc + continue + + if key in ("File", "RawFile"): + entry = Entry(type=key, path=val) + section.entries.append(entry) + continue + + if key == "Version": + entry.version = val + continue + + if key == "Link": + path, _, target = val.partition(" -> ") + entry = Entry(type=key, path=path, target=target) + section.entries.append(entry) + continue + + if key == "Source": + entry.sources.append(val) + continue + + if key == "Info": + entry.info = val + continue + + if key == "Origin": + entry.origin = val + continue + + if key in ("Licence", "License"): + whence_type = LICENCE + section.licence.append(val or "--") + m = re.search(r"(LICEN[CS]E\.[^ ]+) for details", line) + if m: + section.licence_file = m.group(1) + continue + + if whence_type == HEADER: + self.header.append(line) + continue + + if whence_type == NOTES or (whence_type == SECTION and line): + whence_type = NOTES + section.notes.append(line) + continue + + if whence_type == LICENCE: + section.licence.append(line) + continue + + if line: + raise Exception(f"Failed to parse line: {line}") + + # Strip leading and trailing empty lines + self.header = strip(self.header) + for s in self.sections: + s.licence = strip(s.licence) + s.notes = strip(s.notes) + + def save_yaml(self, whence_yaml: Path, remove_empty: bool = False) -> None: + """Save the WHENCE data to a YAML file""" + d = asdict(self) + + if remove_empty: + # Hack: Remove empty items to reduce clutter + for s in d["sections"]: + for key in list(s): + if not s[key]: + del s[key] + for e in s["entries"]: + for key in list(e): + if not e[key]: + del e[key] + + with open(whence_yaml, mode="w", encoding="utf-8") as fh: + yaml = YAML() + yaml.dump(d, fh) diff --git a/contrib/whence2yaml.py b/contrib/whence2yaml.py new file mode 100755 index 00000000..9745d4cf --- /dev/null +++ b/contrib/whence2yaml.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# +# Convert WHENCE to YAML format +# + +from pylib.whence import Whence + +whence = Whence() +whence.load_whence("WHENCE") +whence.save_yaml("WHENCE.yaml", remove_empty=True) +print("WHENCE data converted to YAML and saved to WHENCE.yaml") From 6eec0ebeac3a78368adcd97c2a756a3057a8e8b0 Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Tue, 8 Jul 2025 12:52:47 +0200 Subject: [PATCH 2/2] check_whence.py: Add new files to exceptions lists Signed-off-by: Juerg Haefliger --- check_whence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/check_whence.py b/check_whence.py index 357c1311..21a70b77 100755 --- a/check_whence.py +++ b/check_whence.py @@ -89,10 +89,12 @@ def main(): "build_packages.py", "check_whence.py", "contrib/process_linux_firmware.py", + "contrib/pylib/whence.py", "contrib/templates/debian.changelog", "contrib/templates/debian.control", "contrib/templates/debian.copyright", "contrib/templates/rpm.spec", + "contrib/whence2yaml.py", "copy-firmware.sh", "dedup-firmware.sh", ] @@ -106,6 +108,7 @@ def main(): "carl9170fw/autogen.sh", "check_whence.py", "contrib/process_linux_firmware.py", + "contrib/whence2yaml.py", "copy-firmware.sh", "dedup-firmware.sh", ]