mirror/pico-sdk
1
0
Fork 0
mirror of https://github.com/raspberrypi/pico-sdk.git synced 2025-12-10 07:14:36 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add config check action (#2525)

Browse source 
* Add action to check configs & defines

* Check for all errors before exiting scripts

* Add extract_cmake_functions to the checks

* Fix invalid escape sequence warning
This commit is contained in:
will-v-pi 2025-06-18 16:18:25 +01:00 • committed by GitHub
parent a7ec8c3f79
commit 287196517b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 153 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
.github/workflows/check_configs.yml vendored Normal file
View file

@ -0,0 +1,33 @@
name: Check Configs
on:
push:
pull_request:
jobs:
check-configs:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Check Build Defines
if: always()
run: |
tools/extract_build_defines.py .
- name: Check CMake Configs
if: always()
run: |
tools/extract_cmake_configs.py .
- name: Check CMake Functions
if: always()
run: |
tools/extract_cmake_functions.py .
- name: Check Configs
if: always()
run: |
tools/extract_configs.py .

50
tools/extract_build_defines.py
View file

@ -24,6 +24,13 @@ import logging
from collections import defaultdict from collections import defaultdict
if sys.version_info < (3, 11):
# Python <3.11 doesn't have ExceptionGroup, so define a simple one
class ExceptionGroup(Exception):
def __init__(self, message, errors):
message += "\n" + "\n".join(e.__str__() for e in errors)
super().__init__(message)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -51,11 +58,12 @@ chips_all_descriptions = defaultdict(dict)
def ValidateAttrs(config_name, config_attrs, file_path, linenum): def ValidateAttrs(config_name, config_attrs, file_path, linenum):
_type = config_attrs.get('type') _type = config_attrs.get('type')
errors = []
# Validate attrs # Validate attrs
for key in config_attrs.keys(): for key in config_attrs.keys():
if key not in ALLOWED_CONFIG_PROPERTIES: if key not in ALLOWED_CONFIG_PROPERTIES:
raise Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)) errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)))
if _type == 'int': if _type == 'int':
_min = _max = _default = None _min = _max = _default = None
@ -86,13 +94,13 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
pass pass
if _min is not None and _max is not None: if _min is not None and _max is not None:
if _min > _max: if _min > _max:
raise Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])))
if _min is not None and _default is not None: if _min is not None and _default is not None:
if _min > _default: if _min > _default:
raise Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])) errors.append(Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])))
if _default is not None and _max is not None: if _default is not None and _max is not None:
if _default > _max: if _default > _max:
raise Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])))
elif _type == 'bool': elif _type == 'bool':
assert 'min' not in config_attrs assert 'min' not in config_attrs
assert 'max' not in config_attrs assert 'max' not in config_attrs
@ -111,10 +119,12 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
assert 'max' not in config_attrs assert 'max' not in config_attrs
_default = config_attrs.get('default', None) _default = config_attrs.get('default', None)
else: else:
raise Exception("Found unknown {} type {} at {}:{}".format(BASE_BUILD_DEFINE_NAME, _type, file_path, linenum)) errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_BUILD_DEFINE_NAME, _type, file_path, linenum)))
return errors
errors = []
# Scan all CMakeLists.txt and .cmake files in the specific path, recursively. # Scan all CMakeLists.txt and .cmake files in the specific path, recursively.
@ -135,14 +145,14 @@ for dirpath, dirnames, filenames in os.walk(scandir):
linenum += 1 linenum += 1
line = line.strip() line = line.strip()
if BASE_CONFIG_RE.search(line): if BASE_CONFIG_RE.search(line):
raise Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CONFIG_NAME, file_path, linenum, line, filename if filename == 'CMakeLists.txt' else file_ext)) errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CONFIG_NAME, file_path, linenum, line, filename if filename == 'CMakeLists.txt' else file_ext)))
elif BASE_BUILD_DEFINE_RE.search(line): elif BASE_BUILD_DEFINE_RE.search(line):
m = BUILD_DEFINE_RE.match(line) m = BUILD_DEFINE_RE.match(line)
if not m: if not m:
if re.match(r"^\s*#\s*# ", line): if re.match(r"^\s*#\s*# ", line):
logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line)) logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line))
else: else:
raise Exception("Found misformatted {} at {}:{} ({})".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line)) errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line)))
else: else:
config_name = m.group(1) config_name = m.group(1)
config_description = m.group(2) config_description = m.group(2)
@ -151,10 +161,10 @@ for dirpath, dirnames, filenames in os.walk(scandir):
_attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs) _attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs)
if '=' in config_description: if '=' in config_description:
raise Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)) errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)))
all_descriptions = chips_all_descriptions[applicable] all_descriptions = chips_all_descriptions[applicable]
if config_description in all_descriptions: if config_description in all_descriptions:
raise Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])) errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])))
else: else:
all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum} all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum}
@ -168,19 +178,19 @@ for dirpath, dirnames, filenames in os.walk(scandir):
try: try:
k, v = (i.strip() for i in item.split('=')) k, v = (i.strip() for i in item.split('='))
except ValueError: except ValueError:
raise Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)) errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)))
config_attrs[k] = v.replace('\0', ',') config_attrs[k] = v.replace('\0', ',')
all_attrs.add(k) all_attrs.add(k)
prev = item prev = item
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
if 'group' not in config_attrs: if 'group' not in config_attrs:
raise Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)) errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)))
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
all_configs = chips_all_configs[applicable] all_configs = chips_all_configs[applicable]
if config_name in all_configs: if config_name in all_configs:
raise Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])) errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])))
else: else:
all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description} all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description}
@ -194,14 +204,14 @@ for applicable, all_configs in chips_all_configs.items():
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum) errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum))
# All settings in "host" should also be in "all" # All settings in "host" should also be in "all"
for config_name, config_obj in chips_all_configs["host"].items(): for config_name, config_obj in chips_all_configs["host"].items():
if config_name not in chips_all_configs["all"]: if config_name not in chips_all_configs["all"]:
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
raise Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)) errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)))
# Any chip-specific settings should not be in "all" # Any chip-specific settings should not be in "all"
for chip in CHIP_NAMES: for chip in CHIP_NAMES:
@ -212,7 +222,7 @@ for chip in CHIP_NAMES:
chip_linenum = chip_config_obj['line_number'] chip_linenum = chip_config_obj['line_number']
all_file_path = os.path.join(scandir, all_config_obj['filename']) all_file_path = os.path.join(scandir, all_config_obj['filename'])
all_linenum = all_config_obj['line_number'] all_linenum = all_config_obj['line_number']
raise Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)) errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)))
def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2): def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2):
obj1_filepath = os.path.join(scandir, config_obj1['filename']) obj1_filepath = os.path.join(scandir, config_obj1['filename'])
@ -232,14 +242,18 @@ for applicable in chips_all_configs:
applicable_value = applicable_config_obj[field] applicable_value = applicable_config_obj[field]
other_value = other_config_obj[field] other_value = other_config_obj[field]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Check that attributes match # Check that attributes match
for attr in applicable_config_obj['attrs']: for attr in applicable_config_obj['attrs']:
if attr != 'default': # totally fine for defaults to vary per-platform if attr != 'default': # totally fine for defaults to vary per-platform
applicable_value = applicable_config_obj['attrs'][attr] applicable_value = applicable_config_obj['attrs'][attr]
other_value = other_config_obj['attrs'][attr] other_value = other_config_obj['attrs'][attr]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Raise errors if any were found
if errors:
raise ExceptionGroup("Errors in {}".format(outfile), errors)
# Sort the output alphabetically by name and then by chip # Sort the output alphabetically by name and then by chip
output_rows = set() output_rows = set()

51
tools/extract_cmake_configs.py
View file

@ -24,6 +24,13 @@ import logging
from collections import defaultdict from collections import defaultdict
if sys.version_info < (3, 11):
# Python <3.11 doesn't have ExceptionGroup, so define a simple one
class ExceptionGroup(Exception):
def __init__(self, message, errors):
message += "\n" + "\n".join(e.__str__() for e in errors)
super().__init__(message)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -51,11 +58,12 @@ chips_all_descriptions = defaultdict(dict)
def ValidateAttrs(config_name, config_attrs, file_path, linenum): def ValidateAttrs(config_name, config_attrs, file_path, linenum):
_type = config_attrs.get('type') _type = config_attrs.get('type')
errors = []
# Validate attrs # Validate attrs
for key in config_attrs.keys(): for key in config_attrs.keys():
if key not in ALLOWED_CONFIG_PROPERTIES: if key not in ALLOWED_CONFIG_PROPERTIES:
raise Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)) errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)))
if _type == 'int': if _type == 'int':
_min = _max = _default = None _min = _max = _default = None
@ -86,13 +94,13 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
pass pass
if _min is not None and _max is not None: if _min is not None and _max is not None:
if _min > _max: if _min > _max:
raise Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])))
if _min is not None and _default is not None: if _min is not None and _default is not None:
if _min > _default: if _min > _default:
raise Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])) errors.append(Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])))
if _default is not None and _max is not None: if _default is not None and _max is not None:
if _default > _max: if _default > _max:
raise Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])))
elif _type == 'bool': elif _type == 'bool':
assert 'min' not in config_attrs assert 'min' not in config_attrs
assert 'max' not in config_attrs assert 'max' not in config_attrs
@ -111,10 +119,11 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
assert 'max' not in config_attrs assert 'max' not in config_attrs
_default = config_attrs.get('default', None) _default = config_attrs.get('default', None)
else: else:
raise Exception("Found unknown {} type {} at {}:{}".format(BASE_CMAKE_CONFIG_NAME, _type, file_path, linenum)) errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_CMAKE_CONFIG_NAME, _type, file_path, linenum)))
return errors
errors = []
# Scan all CMakeLists.txt and .cmake files in the specific path, recursively. # Scan all CMakeLists.txt and .cmake files in the specific path, recursively.
@ -135,14 +144,14 @@ for dirpath, dirnames, filenames in os.walk(scandir):
linenum += 1 linenum += 1
line = line.strip() line = line.strip()
if BASE_CONFIG_RE.search(line): if BASE_CONFIG_RE.search(line):
raise Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CONFIG_NAME, file_path, linenum, line, filename if filename == 'CMakeLists.txt' else file_ext)) errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CONFIG_NAME, file_path, linenum, line, filename if filename == 'CMakeLists.txt' else file_ext)))
elif BASE_CMAKE_CONFIG_RE.search(line): elif BASE_CMAKE_CONFIG_RE.search(line):
m = CMAKE_CONFIG_RE.match(line) m = CMAKE_CONFIG_RE.match(line)
if not m: if not m:
if re.match("^\s*#\s*# ", line): if re.match(r"^\s*#\s*# ", line):
logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line)) logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line))
else: else:
raise Exception("Found misformatted {} at {}:{} ({})".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line)) errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line)))
else: else:
config_name = m.group(1) config_name = m.group(1)
config_description = m.group(2) config_description = m.group(2)
@ -151,10 +160,10 @@ for dirpath, dirnames, filenames in os.walk(scandir):
_attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs) _attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs)
if '=' in config_description: if '=' in config_description:
raise Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)) errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)))
all_descriptions = chips_all_descriptions[applicable] all_descriptions = chips_all_descriptions[applicable]
if config_description in all_descriptions: if config_description in all_descriptions:
raise Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])) errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])))
else: else:
all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum} all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum}
@ -168,19 +177,19 @@ for dirpath, dirnames, filenames in os.walk(scandir):
try: try:
k, v = (i.strip() for i in item.split('=')) k, v = (i.strip() for i in item.split('='))
except ValueError: except ValueError:
raise Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)) errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)))
config_attrs[k] = v.replace('\0', ',') config_attrs[k] = v.replace('\0', ',')
all_attrs.add(k) all_attrs.add(k)
prev = item prev = item
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
if 'group' not in config_attrs: if 'group' not in config_attrs:
raise Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)) errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)))
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
all_configs = chips_all_configs[applicable] all_configs = chips_all_configs[applicable]
if config_name in all_configs: if config_name in all_configs:
raise Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])) errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])))
else: else:
all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description} all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description}
@ -194,14 +203,14 @@ for applicable, all_configs in chips_all_configs.items():
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum) errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum))
# All settings in "host" should also be in "all" # All settings in "host" should also be in "all"
for config_name, config_obj in chips_all_configs["host"].items(): for config_name, config_obj in chips_all_configs["host"].items():
if config_name not in chips_all_configs["all"]: if config_name not in chips_all_configs["all"]:
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
raise Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)) errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)))
# Any chip-specific settings should not be in "all" # Any chip-specific settings should not be in "all"
for chip in CHIP_NAMES: for chip in CHIP_NAMES:
@ -212,7 +221,7 @@ for chip in CHIP_NAMES:
chip_linenum = chip_config_obj['line_number'] chip_linenum = chip_config_obj['line_number']
all_file_path = os.path.join(scandir, all_config_obj['filename']) all_file_path = os.path.join(scandir, all_config_obj['filename'])
all_linenum = all_config_obj['line_number'] all_linenum = all_config_obj['line_number']
raise Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)) errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)))
def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2): def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2):
obj1_filepath = os.path.join(scandir, config_obj1['filename']) obj1_filepath = os.path.join(scandir, config_obj1['filename'])
@ -232,14 +241,18 @@ for applicable in chips_all_configs:
applicable_value = applicable_config_obj[field] applicable_value = applicable_config_obj[field]
other_value = other_config_obj[field] other_value = other_config_obj[field]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Check that attributes match # Check that attributes match
for attr in applicable_config_obj['attrs']: for attr in applicable_config_obj['attrs']:
if attr != 'default': # totally fine for defaults to vary per-platform if attr != 'default': # totally fine for defaults to vary per-platform
applicable_value = applicable_config_obj['attrs'][attr] applicable_value = applicable_config_obj['attrs'][attr]
other_value = other_config_obj['attrs'][attr] other_value = other_config_obj['attrs'][attr]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Raise errors if any were found
if errors:
raise ExceptionGroup("Errors in {}".format(outfile), errors)
# Sort the output alphabetically by name and then by chip # Sort the output alphabetically by name and then by chip
output_rows = set() output_rows = set()

29
tools/extract_cmake_functions.py
View file

@ -22,6 +22,13 @@ import re
import csv import csv
import logging import logging
if sys.version_info < (3, 11):
# Python <3.11 doesn't have ExceptionGroup, so define a simple one
class ExceptionGroup(Exception):
def __init__(self, message, errors):
message += "\n" + "\n".join(e.__str__() for e in errors)
super().__init__(message)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -95,6 +102,7 @@ def process_commands(description, name, group, signature):
brief = '' brief = ''
params = [] params = []
desc = '' desc = ''
errors = []
for line in description.split('\n'): for line in description.split('\n'):
line = line.strip() line = line.strip()
if line.startswith('\\'): if line.startswith('\\'):
@ -114,29 +122,29 @@ def process_commands(description, name, group, signature):
# Group name override # Group name override
group = remainder group = remainder
else: else:
logger.error("{}:{} has unknown command: {}".format(group, name, command)) errors.append(Exception("{}:{} has unknown command: {}".format(group, name, command)))
elif '\\' in line: elif '\\' in line:
logger.error("{}:{} has a line containing '\\': {}".format(group, name, line)) errors.append(Exception("{}:{} has a line containing '\\': {}".format(group, name, line)))
else: else:
desc += line + '\\n' desc += line + '\\n'
# Check that there are no semicolons in the parameter descriptions, as that's the delimiter for the parameter list # Check that there are no semicolons in the parameter descriptions, as that's the delimiter for the parameter list
if any([';' in x for x in params]): if any([';' in x for x in params]):
logger.error("{}:{} has a parameter description containing ';'".format(group, name)) errors.append(Exception("{}:{} has a parameter description containing ';'".format(group, name)))
# Check that all parameters are in the signature # Check that all parameters are in the signature
signature_words = set(re.split('\W+', signature)) signature_words = set(re.split(r'\W+', signature))
for param in params: for param in params:
param_name = param.split(' ', maxsplit=1)[0] param_name = param.split(' ', maxsplit=1)[0]
if param_name not in signature_words: if param_name not in signature_words:
logger.error("{}:{} has a parameter {} which is not in the signature {}".format(group, name, param_name, signature)) errors.append(Exception("{}:{} has a parameter {} which is not in the signature {}".format(group, name, param_name, signature)))
# Check that the brief description is not empty # Check that the brief description is not empty
if not brief: if not brief:
logger.warning("{}:{} has no brief description".format(group, name)) logger.warning("{}:{} has no brief description".format(group, name))
# Check that the group has a description # Check that the group has a description
if group not in group_names_descriptions: if group not in group_names_descriptions:
logger.error("{} has no group description (referenced from {})".format(group, name)) errors.append(Exception("{} has no group description (referenced from {})".format(group, name)))
desc = re.sub(r'^(\\n)*(.*?)(\\n)*$', r'\2', desc) desc = re.sub(r'^(\\n)*(.*?)(\\n)*$', r'\2', desc)
return desc.strip(), brief, ';'.join(params), group return desc.strip(), brief, ';'.join(params), group, errors
def sort_functions(item): def sort_functions(item):
@ -155,6 +163,7 @@ def sort_functions(item):
precedence = 3 precedence = 3
return group + str(precedence) + name return group + str(precedence) + name
all_errors = []
# Scan all CMakeLists.txt and .cmake files in the specific path, recursively. # Scan all CMakeLists.txt and .cmake files in the specific path, recursively.
@ -178,7 +187,8 @@ for dirpath, dirnames, filenames in os.walk(scandir):
name = match.group(4) name = match.group(4)
signature = match.group(1).strip() signature = match.group(1).strip()
if signature.startswith(name): if signature.startswith(name):
description, brief, params, processed_group = process_commands(match.group(2).replace('#', ''), name, group, signature) description, brief, params, processed_group, errors = process_commands(match.group(2).replace('#', ''), name, group, signature)
all_errors.extend(errors)
new_dict = { new_dict = {
'name': name, 'name': name,
'group': processed_group, 'group': processed_group,
@ -197,6 +207,9 @@ for dirpath, dirnames, filenames in os.walk(scandir):
if name not in all_functions and name not in allowed_missing_functions: if name not in all_functions and name not in allowed_missing_functions:
logger.warning("{} function has no description in {}".format(name, file_path)) logger.warning("{} function has no description in {}".format(name, file_path))
if all_errors:
raise ExceptionGroup("Errors in {}".format(outfile), all_errors)
with open(outfile, 'w', newline='') as csvfile: with open(outfile, 'w', newline='') as csvfile:
fieldnames = ('name', 'group', 'signature', 'brief', 'description', 'params') fieldnames = ('name', 'group', 'signature', 'brief', 'description', 'params')

57
tools/extract_configs.py
View file

@ -24,6 +24,13 @@ import logging
from collections import defaultdict from collections import defaultdict
if sys.version_info < (3, 11):
# Python <3.11 doesn't have ExceptionGroup, so define a simple one
class ExceptionGroup(Exception):
def __init__(self, message, errors):
message += "\n" + "\n".join(e.__str__() for e in errors)
super().__init__(message)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -53,11 +60,12 @@ chips_all_defines = defaultdict(dict)
def ValidateAttrs(config_name, config_attrs, file_path, linenum): def ValidateAttrs(config_name, config_attrs, file_path, linenum):
_type = config_attrs.get('type', 'int') _type = config_attrs.get('type', 'int')
errors = []
# Validate attrs # Validate attrs
for key in config_attrs.keys(): for key in config_attrs.keys():
if key not in ALLOWED_CONFIG_PROPERTIES: if key not in ALLOWED_CONFIG_PROPERTIES:
raise Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)) errors.append(Exception('{} at {}:{} has unexpected property "{}"'.format(config_name, file_path, linenum, key)))
if _type == 'int': if _type == 'int':
assert 'enumvalues' not in config_attrs assert 'enumvalues' not in config_attrs
@ -95,13 +103,13 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
logger.info('{} at {}:{} has non-integer default value "{}"'.format(config_name, file_path, linenum, config_attrs['default'])) logger.info('{} at {}:{} has non-integer default value "{}"'.format(config_name, file_path, linenum, config_attrs['default']))
if _min is not None and _max is not None: if _min is not None and _max is not None:
if _min > _max: if _min > _max:
raise Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has min {} > max {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['max'])))
if _min is not None and _default is not None: if _min is not None and _default is not None:
if _min > _default: if _min > _default:
raise Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])) errors.append(Exception('{} at {}:{} has min {} > default {}'.format(config_name, file_path, linenum, config_attrs['min'], config_attrs['default'])))
if _default is not None and _max is not None: if _default is not None and _max is not None:
if _default > _max: if _default > _max:
raise Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])) errors.append(Exception('{} at {}:{} has default {} > max {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['max'])))
elif _type == 'bool': elif _type == 'bool':
assert 'min' not in config_attrs assert 'min' not in config_attrs
@ -126,12 +134,13 @@ def ValidateAttrs(config_name, config_attrs, file_path, linenum):
_default = config_attrs['default'] _default = config_attrs['default']
if _default is not None: if _default is not None:
if _default not in _enumvalues: if _default not in _enumvalues:
raise Exception('{} at {}:{} has default value {} which isn\'t in list of enumvalues {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['enumvalues'])) errors.append(Exception('{} at {}:{} has default value {} which isn\'t in list of enumvalues {}'.format(config_name, file_path, linenum, config_attrs['default'], config_attrs['enumvalues'])))
else: else:
raise Exception("Found unknown {} type {} at {}:{}".format(BASE_CONFIG_NAME, _type, file_path, linenum)) errors.append(Exception("Found unknown {} type {} at {}:{}".format(BASE_CONFIG_NAME, _type, file_path, linenum)))
return errors
errors = []
# Scan all .c and .h and .S files in the specific path, recursively. # Scan all .c and .h and .S files in the specific path, recursively.
@ -152,16 +161,16 @@ for dirpath, dirnames, filenames in os.walk(scandir):
linenum += 1 linenum += 1
line = line.strip() line = line.strip()
if BASE_CMAKE_CONFIG_RE.search(line): if BASE_CMAKE_CONFIG_RE.search(line):
raise Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line, file_ext)) errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_CMAKE_CONFIG_NAME, file_path, linenum, line, file_ext)))
elif BASE_BUILD_DEFINE_RE.search(line): elif BASE_BUILD_DEFINE_RE.search(line):
raise Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line, file_ext)) errors.append(Exception("Found {} at {}:{} ({}) which isn't expected in {} files".format(BASE_BUILD_DEFINE_NAME, file_path, linenum, line, file_ext)))
elif BASE_CONFIG_RE.search(line): elif BASE_CONFIG_RE.search(line):
m = CONFIG_RE.match(line) m = CONFIG_RE.match(line)
if not m: if not m:
if re.match(r"^\s*//\s*// ", line): if re.match(r"^\s*//\s*// ", line):
logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)) logger.info("Possible misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line))
else: else:
raise Exception("Found misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)) errors.append(Exception("Found misformatted {} at {}:{} ({})".format(BASE_CONFIG_NAME, file_path, linenum, line)))
else: else:
config_name = m.group(1) config_name = m.group(1)
config_description = m.group(2) config_description = m.group(2)
@ -170,10 +179,10 @@ for dirpath, dirnames, filenames in os.walk(scandir):
_attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs) _attrs = re.sub(r'(\(.+\))', lambda m: m.group(1).replace(',', '\0'), _attrs)
if '=' in config_description: if '=' in config_description:
raise Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)) errors.append(Exception("For {} at {}:{} the description was set to '{}' - has the description field been omitted?".format(config_name, file_path, linenum, config_description)))
all_descriptions = chips_all_descriptions[applicable] all_descriptions = chips_all_descriptions[applicable]
if config_description in all_descriptions: if config_description in all_descriptions:
raise Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])) errors.append(Exception("Found description {} at {}:{} but it was already used at {}:{}".format(config_description, file_path, linenum, os.path.join(scandir, all_descriptions[config_description]['filename']), all_descriptions[config_description]['line_number'])))
else: else:
all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum} all_descriptions[config_description] = {'config_name': config_name, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum}
@ -187,19 +196,19 @@ for dirpath, dirnames, filenames in os.walk(scandir):
try: try:
k, v = (i.strip() for i in item.split('=')) k, v = (i.strip() for i in item.split('='))
except ValueError: except ValueError:
raise Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)) errors.append(Exception('{} at {}:{} has malformed value {}'.format(config_name, file_path, linenum, item)))
config_attrs[k] = v.replace('\0', ',') config_attrs[k] = v.replace('\0', ',')
all_attrs.add(k) all_attrs.add(k)
prev = item prev = item
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
if 'group' not in config_attrs: if 'group' not in config_attrs:
raise Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)) errors.append(Exception('{} at {}:{} has no group attribute'.format(config_name, file_path, linenum)))
#print(file_path, config_name, config_attrs) #print(file_path, config_name, config_attrs)
all_configs = chips_all_configs[applicable] all_configs = chips_all_configs[applicable]
if config_name in all_configs: if config_name in all_configs:
raise Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])) errors.append(Exception("Found {} at {}:{} but it was already declared at {}:{}".format(config_name, file_path, linenum, os.path.join(scandir, all_configs[config_name]['filename']), all_configs[config_name]['line_number'])))
else: else:
all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description} all_configs[config_name] = {'attrs': config_attrs, 'filename': os.path.relpath(file_path, scandir), 'line_number': linenum, 'description': config_description}
else: else:
@ -246,7 +255,7 @@ for applicable, all_configs in chips_all_configs.items():
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum) errors.extend(ValidateAttrs(config_name, config_obj['attrs'], file_path, linenum))
# Check that default values match up # Check that default values match up
if 'default' in config_obj['attrs']: if 'default' in config_obj['attrs']:
@ -259,16 +268,16 @@ for applicable, all_configs in chips_all_configs.items():
# There _may_ be multiple matching defines, but arbitrarily display just one in the error message # There _may_ be multiple matching defines, but arbitrarily display just one in the error message
first_define_value = list(defines_obj.keys())[0] first_define_value = list(defines_obj.keys())[0]
first_define_file_path, first_define_linenum = defines_obj[first_define_value] first_define_file_path, first_define_linenum = defines_obj[first_define_value]
raise Exception('Found {} at {}:{} with a default of {}, but #define says {} (at {}:{})'.format(config_name, file_path, linenum, config_default, first_define_value, first_define_file_path, first_define_linenum)) errors.append(Exception('Found {} at {}:{} with a default of {}, but #define says {} (at {}:{})'.format(config_name, file_path, linenum, config_default, first_define_value, first_define_file_path, first_define_linenum)))
else: else:
raise Exception('Found {} at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default)) errors.append(Exception('Found {} at {}:{} with a default of {}, but no matching #define found'.format(config_name, file_path, linenum, config_default)))
# All settings in "host" should also be in "all" # All settings in "host" should also be in "all"
for config_name, config_obj in chips_all_configs["host"].items(): for config_name, config_obj in chips_all_configs["host"].items():
if config_name not in chips_all_configs["all"]: if config_name not in chips_all_configs["all"]:
file_path = os.path.join(scandir, config_obj['filename']) file_path = os.path.join(scandir, config_obj['filename'])
linenum = config_obj['line_number'] linenum = config_obj['line_number']
raise Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)) errors.append(Exception("Found 'host' config {} at {}:{}, but no matching non-host config found".format(config_name, file_path, linenum)))
# Any chip-specific settings should not be in "all" # Any chip-specific settings should not be in "all"
for chip in CHIP_NAMES: for chip in CHIP_NAMES:
@ -279,7 +288,7 @@ for chip in CHIP_NAMES:
chip_linenum = chip_config_obj['line_number'] chip_linenum = chip_config_obj['line_number']
all_file_path = os.path.join(scandir, all_config_obj['filename']) all_file_path = os.path.join(scandir, all_config_obj['filename'])
all_linenum = all_config_obj['line_number'] all_linenum = all_config_obj['line_number']
raise Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)) errors.append(Exception("'{}' config {} at {}:{} also found at {}:{}".format(chip, config_name, chip_file_path, chip_linenum, all_file_path, all_linenum)))
def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2): def build_mismatch_exception_message(name, thing, config_obj1, value1, config_obj2, value2):
obj1_filepath = os.path.join(scandir, config_obj1['filename']) obj1_filepath = os.path.join(scandir, config_obj1['filename'])
@ -299,14 +308,18 @@ for applicable in chips_all_configs:
applicable_value = applicable_config_obj[field] applicable_value = applicable_config_obj[field]
other_value = other_config_obj[field] other_value = other_config_obj[field]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, field, applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Check that attributes match # Check that attributes match
for attr in applicable_config_obj['attrs']: for attr in applicable_config_obj['attrs']:
if attr != 'default': # totally fine for defaults to vary per-platform if attr != 'default': # totally fine for defaults to vary per-platform
applicable_value = applicable_config_obj['attrs'][attr] applicable_value = applicable_config_obj['attrs'][attr]
other_value = other_config_obj['attrs'][attr] other_value = other_config_obj['attrs'][attr]
if applicable_value != other_value: if applicable_value != other_value:
raise Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)) errors.append(Exception(build_mismatch_exception_message(config_name, "attribute '{}'".format(attr), applicable_config_obj, applicable_value, other_config_obj, other_value)))
# Raise errors if any were found
if errors:
raise ExceptionGroup("Errors in {}".format(outfile), errors)
# Sort the output alphabetically by name and then by chip # Sort the output alphabetically by name and then by chip
output_rows = set() output_rows = set()