diff --git a/overlaycheck/overlaycheck2 b/overlaycheck/overlaycheck2 new file mode 100755 index 0000000..8d3daa6 --- /dev/null +++ b/overlaycheck/overlaycheck2 @@ -0,0 +1,1166 @@ +#!/usr/bin/env python3 + +# Successor to the Perl 'overlaycheck'. Validates the overlay .dts files, +# README, and Makefile in a kernel tree, the same way the Perl tool did, +# but structural DT checks (dormant fragments, gpio/pinctrl wiring, +# container-overlay parameter targets, override parameter extraction) work +# by walking a parsed tree in-process (ovmerge_engine) instead of shelling +# out to dtc/ovmerge and regexing their serialized text output. Real +# compilation (cpp+dtc to .dtb/.dtbo) and the final base+overlay merge +# validity check (dtmerge) still run as external tools - those exercise +# the real toolchain, which isn't something to reimplement. +# +# Deliberately kept as a single file (plus the shared ovmerge_engine.py, +# reused from the ovmerge tool) rather than split further, so installing +# this script is just "copy one file" - see _search_dirs()/_find_file() +# below for where it expects to find ovmerge_engine.py. + +import glob +import os +import re +import shutil +import subprocess +import sys +import tempfile + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _search_dirs(): + """Places ovmerge_engine.py / the ovmerge CLI might live, in priority + order: next to this script (the two tools installed side by side into + one bin directory), the development-tree sibling layout (this script + under overlaycheck/, ovmerge under ../ovmerge/), then anywhere on + PATH (installed separately from this script but still reachable - + including a future 'ovmerge' with no '.py' suffix).""" + dirs = [_HERE, os.path.normpath(os.path.join(_HERE, '..', 'ovmerge'))] + dirs += [d for d in os.environ.get('PATH', '').split(os.pathsep) if d] + seen = set() + for d in dirs: + d = os.path.abspath(d) + if d not in seen: + seen.add(d) + yield d + + +def _find_file(names): + """Full path to the first of names found across _search_dirs(), or None.""" + for d in _search_dirs(): + for name in names: + path = os.path.join(d, name) + if os.path.isfile(path): + return path + return None + + +_engine_file = _find_file(['ovmerge_engine.py']) +if _engine_file: + sys.path.insert(0, os.path.dirname(_engine_file)) + +try: + from ovmerge_engine import ( # noqa: E402 + NAME, OvMergeError, dtparam, dtparse, find_wakeable_fragments, + get_child, get_children, get_label_ref, get_labels, + get_node, get_prop, get_props, get_prop_string, node_path, ovapply1, + ovapply2, + ) +except ImportError: + fatal_msg = ( + "Can't find ovmerge_engine.py. Install it next to this script, in a " + "sibling '../ovmerge' directory, or anywhere on PATH." + ) + print(f"* {fatal_msg}") + sys.exit(1) + +# The 'ovmerge' CLI (currently ovmerge.py, eventually just 'ovmerge') is only +# needed for the '// redo:' regeneration check - not for the import above. +OVMERGE_PY = _find_file(['ovmerge', 'ovmerge.py']) +DTMERGE = 'dtmerge' + + +# Diagnostics +# --------------------------------------------------------------------------- +# Accumulator for overlaycheck2, replacing the Perl tool's global $fail flag +# plus ad hoc error()/print() calls. + +class Reporter: + def __init__(self): + self.fail = False + self.context = None + + def _emit(self, msg): + if self.context: + print(f"* {self.context}: {msg}") + else: + print(f"* {msg}") + + def error(self, msg): + self._emit(msg) + self.fail = True + + def warn(self, msg): + self._emit(msg) + + +# Text-format parsers +# --------------------------------------------------------------------------- +# Parsers for the plain-text formats overlaycheck cross-references against +# the Device Tree source: the overlays README, the overlays Makefile, and +# the local exclusions file. These aren't tree-shaped, so they stay simple +# line-oriented parsers (near-verbatim ports of the Perl tool's +# parse_readme/parse_makefile/parse_exclusions), just taking a Reporter +# instead of a global $fail flag. + +WORD_PATTERN = r'[0-9a-zA-Z0-9][-_a-zA-Z0-9]*' +README_PARAM_PATTERN = r'([-_a-zA-Z0-9]|<[a-z]>|<[a-z]-[a-z]>)*' + +_DESCR_COLUMN = 32 + + +def parse_exclusions(path, reporter): + """Returns (ignore_missing, ignore_vestigial): overlay name -> list of + parameter names to exclude from the 'undocumented'/'vestigial' checks.""" + ignore_missing = {} + ignore_vestigial = {} + + try: + fh = open(path) + except OSError: + reporter.error(f"Failed to open '{path}'") + return ignore_missing, ignore_vestigial + + overlay = None + with fh: + for line in fh: + line = line.rstrip('\n') + + m = re.match(r'^=\s*(.+)$', line) + if m: + overlay = m.group(1) + ignore_missing[overlay] = [] + ignore_vestigial[overlay] = [] + continue + + m = re.match(r'^-\s*(.+)$', line) + if m: + ignore_missing[overlay].append(m.group(1)) + continue + + m = re.match(r'^\+\s*(.+)$', line) + if m: + ignore_vestigial[overlay].append(m.group(1)) + continue + + return ignore_missing, ignore_vestigial + + +def parse_readme(path, ignore_missing, ignore_vestigial, documentation, reporter): + """Parses the overlays README into {overlay_name: [param, ...]}. + + ignore_missing/ignore_vestigial/documentation are dicts (as returned by + parse_exclusions for the first two) that get further entries added for + '' and '' pseudo-overlays, matching the + original tool's behaviour of sharing these across both parsers. + """ + overlays = {} + + try: + fh = open(path) + except OSError: + reporter.error(f"Failed to open '{path}'") + return overlays + + overlay = None + last_overlay = '' + params = None + has_params = False + in_params = False + blank_count = 0 + linenum = 0 + + with fh: + for line in fh: + linenum += 1 + line = line.rstrip('\n') + + if '\t' in line: + reporter.error(f"TABs in README ({linenum})") + if re.search(r'\s$', line): + reporter.error(f"Trailing whitespace in README ({linenum})") + if len(line) > 80 and not re.match(r'^\s*[^\s]+$', line): + reporter.error(f"Line too long in README ({linenum})") + + if overlay and not line: + blank_count += 1 + if blank_count == 2: + if params is None: + reporter.error(f"Missing params for overlay {overlay} ({linenum})") + if isinstance(params, list): + overlays[overlay] = sorted(params) + elif params: + overlays[overlay] = params + else: + overlays[overlay] = [] + + overlay = None + params = None + in_params = False + continue + + blank_count = 0 + + if re.match(r'^\w+:\s', line) and not re.match(r'^(Name|Info|Load|Params):', line): + reporter.error(f"Bad label ({linenum})") + + m = re.match(r'^Name:(\s*)(.*)\s*$', line) + if m: + if m.group(1) != ' ': + reporter.error(f"Bad formatting in README ({linenum})") + if params is not None: + reporter.error(f"Missing blank lines after overlay? ({linenum})") + overlay = m.group(2) + params = None + in_params = False + if not re.match(rf'^{WORD_PATTERN}$', overlay) and overlay != '': + reporter.error(f"Illegal overlay name '{overlay}' in README ({linenum})") + + if not (overlay > last_overlay): + reporter.error(f"Overlay '{overlay}' - order violation in README ({linenum})") + last_overlay = overlay if overlay != '' else ' ' + continue + + m = re.match(r'^Info:(\s*)(.*)\s*$', line) + if m: + if m.group(1) != ' ': + reporter.error(f"Bad formatting in README ({linenum})") + if not overlay: + reporter.error(f"Info label with no Name? ({linenum})") + mm = re.match(rf'^See ({WORD_PATTERN})(?:\s+\(.*)?$', m.group(2)) + if mm: + params = mm.group(1) + continue + + m = re.match(r'^Load:(\s*)(.*)\s*$', line) + if m: + if m.group(1) != ' ': + reporter.error(f"Bad formatting in README ({linenum})") + if not overlay: + reporter.error(f"Load label with no Name? ({linenum})") + cmd = m.group(2) + has_params = False + if overlay != '': + if cmd == '': + # Ignore this overlay + ignore_missing[overlay] = True + ignore_vestigial[overlay] = True + overlay = None + elif cmd == '': + # Not a real overlay - a placeholder for common documentation + documentation[overlay] = True + ignore_missing[overlay] = True + ignore_vestigial[overlay] = True + has_params = True + else: + mm = re.match( + rf'^dtoverlay=({WORD_PATTERN})(,(?:=|\[=\])?)?$', cmd) + if not mm: + reporter.error(f"Invalid Load example ({linenum})") + elif mm.group(1) != overlay: + reporter.error(f"Wrong overlay name in Load example ({linenum})") + if mm and mm.group(2): + has_params = True + continue + + m = re.match(r'^Params:(.*)$', line) if overlay is not None else None + if m: + blank_count = 0 + rol = m.group(1) + params = [] + in_params = True + + if rol: + if rol == ' ': + if has_params: + reporter.error(f"Parameter presence mismatch ({linenum})") + continue + mm = re.match(r'^ ([^ ]+)( *)', rol) + if not mm: + reporter.error(f"Bad formatting in README ({linenum})") + continue + if not has_params: + reporter.error(f"Parameter presence mismatch ({linenum})") + param, indent2 = mm.group(1), len(mm.group(2)) + if not re.fullmatch(README_PARAM_PATTERN, param): + reporter.error(f"Invalid parameter name '{param}' in README ({linenum})") + descr_indent = 8 + len(param) + indent2 + if descr_indent != _DESCR_COLUMN and indent2 != 0: + reporter.error(f"Bad formatting in README ({linenum})") + params.append(param) + continue + + m = re.match(r'^( +)([^ ]+)( *)', line) if in_params else None + if m: + indent, param, indent2 = len(m.group(1)), m.group(2), len(m.group(3)) + if indent == 8: + descr_indent = 8 + len(param) + indent2 + # Try to spot a comment + if ((indent2 == 1 or (indent2 == 0 and param.endswith(':'))) and + descr_indent < _DESCR_COLUMN): + in_params = False + continue + if not re.fullmatch(README_PARAM_PATTERN, param): + reporter.error(f"Invalid parameter name '{param}' in README ({linenum})") + if descr_indent != _DESCR_COLUMN and indent2 != 0: + reporter.error(f"Bad formatting in README ({linenum})") + params.append(param) + else: + if indent < 8 or (indent > 8 and indent < _DESCR_COLUMN): + reporter.error(f"Bad formatting in README ({linenum})") + + # Resolve inter-overlay ("See") references + for name in list(overlays.keys()): + src = overlays[name] + if not isinstance(src, list): + resolved = overlays.get(src) + if isinstance(resolved, list): + overlays[name] = resolved + else: + reporter.error(f"Chained 'See' link from {name} to {src}") + + return overlays + + +def parse_makefile(path, reporter): + """Returns the ordered list of overlay names built by the Makefile, + with '' prepended (matching the source-file set's + convention for the always-present base DTB entry).""" + overlays = [''] + + try: + fh = open(path) + except OSError: + reporter.error(f"Failed to open '{path}'") + return overlays + + last_overlay = '' + linenum = 0 + in_multiline = False + + with fh: + for line in fh: + linenum += 1 + line = line.rstrip('\n') + if re.search(r'\s$', line): + reporter.error(f"Trailing whitespace in Makefile ({linenum})") + overlay = None + + if in_multiline: + m = re.match(r'^\t(.+)\.dtbo( \\)?$', line) + if m: + overlay = m.group(1) + if not m.group(2): + in_multiline = False + else: + reporter.error(f"Syntax error in Makefile ({linenum})") + else: + m = re.match(r'^dtbo-\$\(RPI_DT_OVERLAYS\) \+= (.+)\.dtbo$', line) + if m: + overlay = m.group(1) + elif re.match(r'^dtbo-\$\(CONFIG_ARCH_BCM2835\) \+= \\$', line): + in_multiline = True + + if overlay: + if not (overlay > last_overlay): + reporter.error(f"Overlay '{overlay}' - order violation in Makefile ({linenum})") + overlays.append(overlay) + last_overlay = overlay + + return overlays + + +# Checker registration and dispatch +# --------------------------------------------------------------------------- +# A "checker" is a Checker subclass, one fresh instance of which is created +# for every tree walk - replacing the old approach of shelling out to +# dtc/ovmerge and regexing their serialized text output line-by-line. +# +# Instantiating fresh per walk means a checker's own attributes are already +# per-overlay state, there from on_node() to on_end() and gone again on the +# next overlay's walk - no need to stash anything on the shared CheckContext. + +_CHECKER_CLASSES = [] + + +class Checker: + """Base class for a checker. Subclass it and set exactly one of `path` + or `name` (regex strings, anchored with .match() - use '$' to anchor the + end too) to have on_node() called for every node whose full path / own + name matches; the regex Match object is passed in, so capture groups + (e.g. a fragment number) are available without re-deriving them. Leave + both unset to skip per-node dispatch entirely. + + on_end() is called once per walk, after every node has been visited + (and after any on_node() calls) - for checks that need to see the whole + tree, or to report on state on_node() accumulated across matching nodes. + Override either or both of on_node()/on_end(); the defaults do nothing. + + Subclasses are auto-registered and instantiated fresh (no constructor + arguments) for each call to walk().""" + + path = None + name = None + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if cls.path is not None and cls.name is not None: + raise ValueError(f"{cls.__name__}: set at most one of path/name") + cls._pattern = re.compile(cls.path if cls.path is not None else cls.name) \ + if (cls.path is not None or cls.name is not None) else None + cls._on_path = cls.path is not None + _CHECKER_CLASSES.append(cls) + + def on_node(self, node, ctx, m): + pass + + def on_end(self, ctx): + pass + + +class CheckContext: + """Passed to every checker: the parsed DT, which overlay/file it came + from, and the shared Reporter for surfacing errors/warnings.""" + + def __init__(self, dt, overlay_name, source_file, reporter): + self.dt = dt + self.overlay_name = overlay_name + self.source_file = source_file + self.reporter = reporter + + +def walk(root, ctx): + """Visit every node in the tree rooted at root, dispatching to every + registered checker (a fresh instance per call) whose pattern matches, + then let each checker report via on_end().""" + instances = [cls() for cls in _CHECKER_CLASSES] + + def visit(node): + path = None + for inst in instances: + pattern = inst._pattern + if pattern is None: + continue + if inst._on_path: + if path is None: + path = node_path(node) + m = pattern.match(path) + else: + m = pattern.match(node[NAME]) + if m: + inst.on_node(node, ctx, m) + for child in get_children(node): + visit(child) + + visit(root) + + for inst in instances: + inst.on_end(ctx) + + +# Override-parameter / platform extraction +# --------------------------------------------------------------------------- +# Extracts the information the old Perl tool's get_params() scraped from +# dtc's pretty-printed '-O dts' text output (override parameter names, and +# which platform an overlay's 'compatible' string declares) - reading it +# directly from the parsed tree instead. +# +# Note: the old get_params() also had a 'deadbeef' placeholder-value check +# that only made sense because it was scanning dtc's re-serialized text; +# with no text-scraping step to protect against, there's nothing for it to +# guard here, so it's intentionally not carried over. + +_PLATFORM_RE = re.compile(r'^brcm,(bcm2835|bcm2711|bcm2712)$') + + +class SourceInfo: + """What overlaycheck needs from one parsed base/overlay .dts file: its + __overrides__ parameter names, and (for overlays) which platform(s) its + 'compatible' string declares support for.""" + + def __init__(self, params, platforms): + self.params = params + self.platforms = platforms + + +def get_source_info(dt, source_file, reporter, is_overlay): + overrides = get_node(dt, '/__overrides__') + params = [] + for prop in (get_props(overrides) if overrides is not None else []): + name = prop[0] + if not re.fullmatch(WORD_PATTERN, name): + reporter.error(f"Invalid parameter name '{name}' in '{source_file}'") + params.append(name) + params.sort() + + platforms = set() + if is_overlay: + compat = get_prop_string(dt.root, 'compatible') + if compat is None: + reporter.error(f"Missing overlay compatible string in '{source_file}'") + else: + m = _PLATFORM_RE.match(compat) + if not m: + reporter.error(f"Invalid overlay compatible string '{compat}' in '{source_file}'") + else: + platforms.add(m.group(1)) + + return SourceInfo(params, platforms) + + +# GPIO/pinctrl checks +# --------------------------------------------------------------------------- +# Checks around GPIO-hog/pinctrl fragments in an overlay: +# A) a child node of a fragment targeting the literal 'gpio' label, that +# isn't a gpio-hog, must have a label (otherwise nothing can reference it) +# B) every such label must actually be referenced by some 'pinctrl-0' +# property or override declaration +# C) an __overrides__ declaration must not target a fragment's own label +# (only its 'target'/'target-path' properties are legitimate to retarget) +# +# Replaces the old pinctrl_checker(), which tracked fragment/node/brace +# depth by hand while reading 'ovmerge -N' text output line by line. +# Unlike that version, this checks every direct child of a matching +# fragment's payload (the old brace-depth tracker gave up on the rest of a +# fragment's siblings after the first child that didn't match its +# line-oriented pattern - e.g. any property appearing before a subnode, or a +# subnode name containing '@'); this is intentionally more thorough. +# +# One deliberate behaviour change found while testing this against real +# overlays: a labelled gpio-hog node no longer counts as needing a +# pinctrl-0 reference. The old check's line-oriented pattern also excluded +# '@'-addressed node names, which happened to hide this - a labelled hog +# node with a unit address (e.g. "hog: hog@1a { gpio-hog; ... };", used so +# an __overrides__ declaration can address it) was never actually reached by +# the old checker. Once every child is checked, the old rule as literally +# written would flag every such hog as an unreferenced pinctrl label, which +# is wrong: a hog self-activates and was never meant to need one. + +_GPIO_LABEL = '&gpio' + + +def _collect_pinctrl_refs(dt): + refs = set() + + def walk_refs(node): + prop = get_prop(node, 'pinctrl-0') + if prop is not None: + for chunk in prop[1:]: + if chunk[0] == '<': + for elem in chunk[2]: + if isinstance(elem, str) and elem.startswith('&'): + refs.add(elem[1:]) + for child in get_children(node): + walk_refs(child) + + walk_refs(dt.root) + + overrides = get_node(dt, '/__overrides__') + for prop in (get_props(overrides) if overrides is not None else []): + dtparam(dt, prop[0], None, pinctrl_refs=refs) + + return refs + + +def _collect_misused_fragment_labels(dt, fraglabels): + misused = [] + seen = set() + + def on_label_use(label, prop): + if label in fraglabels and not prop.startswith('target') and label not in seen: + misused.append(label) + seen.add(label) + + overrides = get_node(dt, '/__overrides__') + for prop in (get_props(overrides) if overrides is not None else []): + dtparam(dt, prop[0], None, label_use=on_label_use) + + return misused + + +class CheckPinctrl(Checker): + """Visits every fragment, tallying its label plus (for one targeting the + literal 'gpio' label) its unlabelled/labelled-non-hog children; reports + on the complete tallies once the whole tree has been seen.""" + + name = r'^fragment[@-](\d+)$' + + def __init__(self): + self.fraglabels = set() + self.gpio_labels = set() + self.unlabelled_nodes = [] + + def on_node(self, node, ctx, m): + self.fraglabels.update(get_labels(node)) + + target = get_prop(node, 'target') + if target is None: + return + if get_label_ref(target[1]) != _GPIO_LABEL: + return + + payload = get_child(node, '__overlay__') or get_child(node, '__dormant__') + if payload is None: + return + + for sub in get_children(payload): + is_hog = get_prop(sub, 'gpio-hog') is not None + labels = get_labels(sub) + if labels: + if not is_hog: + self.gpio_labels.update(labels) + elif not is_hog: + self.unlabelled_nodes.append(sub[NAME]) + + def on_end(self, ctx): + pinctrl_refs = _collect_pinctrl_refs(ctx.dt) + + for name in sorted(self.unlabelled_nodes): + ctx.reporter.error(f"gpio node {name} has no label") + + for label in sorted(self.gpio_labels): + if label not in pinctrl_refs: + ctx.reporter.error(f"gpio node label {label} is not referenced") + + for label in _collect_misused_fragment_labels(ctx.dt, self.fraglabels): + ctx.reporter.error(f"misused fragment label {label}") + + +# Container-overlay checks +# --------------------------------------------------------------------------- +# Checks that "container" overlays (i2c-rtc, i2c-fan, i2c-sensor: a menu of +# optional devices selected via boolean __overrides__ params) actually wire +# each parameter to the device node it's named after. +# +# Replaces the old container_checker(), which compiled the overlay to a +# real .dtbo, ran 'fdtget -t bx' on each override to find the ones in the +# fragment-toggle form (a raw byte match on the phandle cell being zero), +# applied each one via the external 'dtmerge' binary, and grepped 'dtdiff' +# text output for an added node. All of that is replaced by applying the +# override and merging the fragment in-process (dtparam, then ovapply1 to +# fold the newly-woken fragment into the local 'i2cbus' label these +# overlays target - see i2c-buses.dtsi - before ovapply2 merges the result +# onto a real base tree) and diffing node names directly, using the same +# fragment-toggle-form test as find_wakeable_fragments() (the override's +# first reference is the literal label '0', not a real node label). +# +# Only applies to CONTAINER_OVERLAYS, and needs ctx.info (the overlay's +# SourceInfo, for its __overrides__ param names) and ctx.base0_dts_path (set +# on the CheckContext before walking) - skipped like any other checker whose +# match fails when neither is available. + +CONTAINER_OVERLAYS = ('i2c-rtc', 'i2c-fan', 'i2c-sensor') + + +def _node_names(node, names): + names.add(node[NAME]) + for child in get_children(node): + _node_names(child, names) + + +class CheckContainer(Checker): + """Only applies to CONTAINER_OVERLAYS, and needs ctx.info (the overlay's + SourceInfo, for its __overrides__ param names) and ctx.base0_dts_path + (set on the CheckContext before walking) - skipped like any other + checker whose match fails when neither is available. Doesn't react to + individual nodes, so leaves on_node() unused and only does its work in + on_end().""" + + def on_end(self, ctx): + if ctx.overlay_name not in CONTAINER_OVERLAYS or ctx.info is None or not ctx.base0_dts_path: + return + + overlay_path = ctx.source_file + base_path = ctx.base0_dts_path + params = ctx.info.params + reporter = ctx.reporter + + before = dtparse(base_path, False) + before_names = set() + _node_names(before.root, before_names) + + for param in params: + ov = dtparse(overlay_path, False) + overrides = get_node(ov, '/__overrides__') + ovr = get_prop(overrides, param) + if ovr is None or len(ovr) < 2 or get_label_ref(ovr[1]) != '0': + continue + + dtparam(ov, param, '') + ovapply1(ov) # fold the newly-woken fragment into its (local) i2cbus label + + base = dtparse(base_path, False) + try: + ovapply2(base, ov) + except OvMergeError as ex: + reporter.error(f"failed to merge with param '{param}': {ex}") + continue + + after_names = set() + _node_names(base.root, after_names) + new_names = after_names - before_names + + paramx = re.sub(r'\d$', 'x', param) + wanted = re.compile(rf'^({re.escape(param)}|{re.escape(paramx)})@') + if not any(wanted.match(name) for name in new_names): + reporter.error(f"parameter '{param}' doesn't enable matching node") + + +# Dormant-fragment check +# --------------------------------------------------------------------------- +# Detects '__dormant__' fragment payloads that no __overrides__ declaration +# can ever wake up. Replaces the old dormant_checker(), which tracked +# fragment/__overrides__ structure by hand while reading 'ovmerge -N' text +# output line by line. +# +# ctx.wakeable (a set of fragment-number strings, from +# find_wakeable_fragments()) must be set on the CheckContext before walking +# an overlay's tree with this checker registered. + +class CheckDormantFragment(Checker): + name = r'^fragment[@-](\d+)$' + + def on_node(self, node, ctx, m): + if get_child(node, '__dormant__') is None: + return + num = m.group(1) + if num not in ctx.wakeable: + ctx.reporter.error(f"fragment {num} is dormant and cannot be activated") + + +# Main +# --------------------------------------------------------------------------- + +BASE_FILES = [ + "bcm2712-rpi-5-b", + "bcm2712-rpi-cm5-cm5io", + "bcm2711-rpi-4-b", + "bcm2711-rpi-cm4", + "bcm2711-rpi-cm4s", + "bcm2708-rpi-b", + "bcm2708-rpi-b-rev1", + "bcm2708-rpi-b-plus", + "bcm2708-rpi-zero", + "bcm2708-rpi-zero-w", + "bcm2708-rpi-cm", + "bcm2709-rpi-2-b", + "bcm2710-rpi-3-b", + "bcm2710-rpi-3-b-plus", + "bcm2710-rpi-cm3", + "bcm2710-rpi-zero-2-w", +] + +EXTRA_BASE_FILES = [ + "bcm2709-rpi-cm2", + "bcm2710-rpi-cm0", + "bcm2710-rpi-2-b", + "bcm2711-rpi-400", + "bcm2712-rpi-500", + "bcm2712-rpi-cm5-cm4io", + "bcm2712-rpi-cm5l-cm5io", + "bcm2712-rpi-cm5l-cm4io", +] + +PLATFORM_REPS = { + 'bcm2835': 'bcm2710-rpi-3-b', + 'bcm2711': 'bcm2711-rpi-4-b', + 'bcm2712': 'bcm2712-rpi-5-b', +} + +WARNINGS_TO_SUPPRESS = [ + 'unit_address_vs_reg', 'simple_bus_reg', 'unit_address_format', + 'interrupts_property', 'gpios_property', 'label_is_string', + 'unique_unit_address', 'avoid_unnecessary_addr_size', +] + +NON_STRICT_WARNINGS = [ + 'pci_device_reg', 'pci_device_bus_num', 'reg_format', + 'interrupt_provider', 'dma_ranges_format', 'avoid_default_addr_size', +] + +# dtmerge's exit code when a base is simply incompatible with an overlay +# (e.g. a missing target label) - expected/acceptable in the -t sweep. +DTMERGE_LABEL_NOT_FOUND = 254 + + +def usage(): + print("Usage: overlaycheck2 [] [ ...]") + print(" where can be any of:") + print() + print(" -h Show this help message") + print(" -s Show more strict/picky warnings") + print(" -t Try all overlays on all base files") + print(" -v Enable verbose output") + + +def fatal(msg): + print(f"* {msg}") + sys.exit(1) + + +def list_print(label, items, reporter): + if items: + print(f"{label}:") + for x in sorted(items): + print(f" {x}") + reporter.fail = True + + +def find_cpp(): + for cmd in ('arm-linux-gnueabihf-cpp', 'aarch64-linux-gnu-cpp', 'cpp'): + if shutil.which(cmd): + return cmd + fatal("no suitable cpp found") + + +def compute_dtc_opts(dtc, strict_dtc): + warnings = list(WARNINGS_TO_SUPPRESS) + if not strict_dtc: + warnings += NON_STRICT_WARNINGS + opts = [] + for warn in warnings: + probe = subprocess.run([dtc, '-W', f'no-{warn}', '-v'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if probe.returncode == 0: + opts += ['-W', f'no-{warn}'] + return opts + + +def dtc_cpp(cpp, dtc, dtc_opts, kerndir, cwd, infile, outfile, reporter): + tmpfile = outfile + '.tmp' + cpp_cmd = [cpp, '-nostdinc', '-I.', f'-I{kerndir}/include', '-Ioverlays', + '-undef', '-D__DTS__', '-x', 'assembler-with-cpp', '-o', tmpfile, infile] + if subprocess.run(cpp_cmd, cwd=cwd).returncode != 0: + reporter.error(f"Failed to CPP '{infile}'") + return + dtc_cmd = [dtc, *dtc_opts, '-i', 'overlays', '-@', '-I', 'dts', '-O', 'dtb', '-o', outfile, tmpfile] + if subprocess.run(dtc_cmd, cwd=cwd).returncode != 0: + reporter.error(f"Failed to compile '{infile}'") + if os.path.exists(tmpfile): + os.unlink(tmpfile) + + +def _ovmerge_cmd(*args): + # A found '...ovmerge.py' needs the interpreter spelled out explicitly; + # a found bare 'ovmerge' is an installed executable with its own shebang. + if OVMERGE_PY.endswith('.py'): + return [sys.executable, OVMERGE_PY, *args] + return [OVMERGE_PY, *args] + + +def check_redo_comment(file, overlay, reporter): + with open(file) as f: + first_line = f.readline() + if not re.match(r'^// redo: ', first_line): + return + if OVMERGE_PY is None: + reporter.warn(f"'{overlay}' has a redo comment but ovmerge could not be found to check it") + return + # The redo comment's arguments name sibling overlay files by relative + # path (e.g. "w1-gpio-overlay.dts,gpiopin=4"), so ovmerge must run with + # cwd set to this file's own directory to find them. + result = subprocess.run(_ovmerge_cmd('-r', file), + capture_output=True, text=True, cwd=os.path.dirname(file)) + if result.returncode != 0: + reporter.error(f"Failed to regenerate '{overlay}' from its redo comment") + return + with open(file) as f: + current = f.read() + if result.stdout != current: + reporter.warn(f"'{overlay}' overlay changed after redo") + + +def parse_source_files(overlaysdir, dtsgroups, reporter): + params_union = set() + + for group_dir, bases in dtsgroups: + for base in bases: + dts_file = os.path.join(group_dir, base + '.dts') + try: + dt = dtparse(dts_file, False) + except OvMergeError as ex: + reporter.error(f"Failed to parse base file {base}.dts: {ex}") + continue + info = get_source_info(dt, dts_file, reporter, is_overlay=False) + params_union.update(info.params) + + source = {'': SourceInfo(sorted(params_union), set())} + + for file in sorted(glob.glob(os.path.join(overlaysdir, '*-overlay.dts'))): + overlay = os.path.basename(file)[:-len('-overlay.dts')] + check_redo_comment(file, overlay, reporter) + try: + dt = dtparse(file, False) + except OvMergeError as ex: + reporter.error(f"Failed to parse {overlay}-overlay.dts: {ex}") + continue + source[overlay] = get_source_info(dt, file, reporter, is_overlay=True) + + return source + + +def expand_documentation_params(params, readme, documentation): + expanded = [] + for p in params: + if p in documentation: + expanded.extend(readme.get(p, [])) + else: + expanded.append(p) + return expanded + + +def resolve_placeholders(l, r): + l = set(l) + r = set(r) + for rv in list(r): + pattern, count = re.subn(r'<[i-z]>', '[0-9a-fA-F]+', rv) + if not count: + pattern, count = re.subn(r'<[a-h]>', '[a-z]', rv) + if not count: + continue + regex = re.compile(f'^{pattern}$') + matched = {lv for lv in l if regex.match(lv)} + if matched: + l -= matched + r.discard(rv) + return l, r + + +def check_readme_vs_source(source, readme, ignore_missing, ignore_vestigial, documentation, reporter): + overlays = set(source.keys()) + readme_overlays = set(readme.keys()) + + left_only = overlays - readme_overlays - set(ignore_missing.keys()) + right_only = readme_overlays - overlays - set(ignore_vestigial.keys()) + list_print("Overlays without documentation", left_only, reporter) + list_print("Vestigial overlay documentation", right_only, reporter) + + for overlay in sorted(overlays & readme_overlays): + readme_params = expand_documentation_params(readme[overlay], readme, documentation) + l = set(source[overlay].params) - set(readme_params) + r = set(readme_params) - set(source[overlay].params) + + l, r = resolve_placeholders(l, r) + + excl_missing = ignore_missing.get(overlay, []) + excl_vestigial = ignore_vestigial.get(overlay, []) + l -= set(excl_missing if isinstance(excl_missing, list) else []) + r -= set(excl_vestigial if isinstance(excl_vestigial, list) else []) + + list_print(f"{overlay} undocumented parameters", l, reporter) + list_print(f"{overlay} vestigial parameter documentation", r, reporter) + + +def check_makefile_vs_source(source, makefile, reporter): + overlays = set(source.keys()) + makefile_set = set(makefile) + list_print("Overlays missing from the Makefile", overlays - makefile_set, reporter) + list_print("Vestigial overlay Makefile entries", makefile_set - overlays, reporter) + + +def check_one_overlay(overlay, overlaysdir, tmpdir, merged_dtb, base0_dts_path, + source, reporter, verbose): + reporter.context = overlay + overlay_file = os.path.join(overlaysdir, overlay + '-overlay.dts') + + try: + dt = dtparse(overlay_file, False) + except OvMergeError: + return # already reported by parse_source_files + + info = source.get(overlay) + + ctx = CheckContext(dt, overlay, overlay_file, reporter) + ctx.wakeable = find_wakeable_fragments(dt) + ctx.info = info + ctx.base0_dts_path = base0_dts_path + walk(dt.root, ctx) + + platforms = info.platforms if info else set() + + base = None + for plat, rep_base in PLATFORM_REPS.items(): + if plat in platforms: + base = rep_base + break + + if base: + base_dtb = os.path.join(tmpdir, base + '.dtb') + overlay_dtbo = os.path.join(tmpdir, overlay + '.dtbo') + if os.path.isfile(base_dtb) and os.path.isfile(overlay_dtbo): + if verbose: + print(f"[ dtmerge {base_dtb} {merged_dtb} {overlay_dtbo} ]") + cmd = [DTMERGE, base_dtb, merged_dtb, overlay_dtbo] + if subprocess.run(cmd).returncode != 0: + reporter.error(f"Error in overlay {overlay}") + + +def run_try_all(overlays, source, tmpdir, merged_dtb, verbose, reporter): + for overlay in overlays: + if overlay.startswith('<'): + continue + reporter.context = overlay + info = source.get(overlay) + platforms = info.platforms if info else set() + overlay_dtbo = os.path.join(tmpdir, overlay + '.dtbo') + if not os.path.isfile(overlay_dtbo): + continue + + for base in BASE_FILES: + base_dtb = os.path.join(tmpdir, base + '.dtb') + if not os.path.isfile(base_dtb): + continue + if re.search(r'(wifi|bt)', overlay) and not re.match( + r'^(bcm2708-rpi-zero-w|bcm2710-rpi-zero-2|bcm2710-rpi-3-|' + r'bcm2711-rpi-(4|cm4$)|bcm2712-rpi-(5|cm5))', base): + continue + if overlay == 'vl805' and base == 'bcm2711-rpi-cm4s': + continue + if 'bcm2711' in platforms and not base.startswith('bcm2711'): + continue + if 'bcm2712' in platforms and not base.startswith('bcm2712'): + continue + + probe = subprocess.run([DTMERGE, base_dtb, merged_dtb, overlay_dtbo], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if probe.returncode == DTMERGE_LABEL_NOT_FOUND: + continue + + if verbose: + print(f"[ dtmerge {base_dtb} {merged_dtb} {overlay_dtbo} ]") + cmd = [DTMERGE, *(['-d'] if verbose else []), base_dtb, merged_dtb, overlay_dtbo] + if subprocess.run(cmd).returncode != 0: + reporter.error(f"Failed to merge {overlay} with {base}") + + +def find_dts_path(base, dtsgroups): + for group_dir, bases in dtsgroups: + if base in bases: + return os.path.join(group_dir, base + '.dts') + return None + + +def main(argv): + os.environ['LD_LIBRARY_PATH'] = os.path.join(os.path.expanduser('~'), 'lib') + + verbose = False + strict_dtc = False + try_all = False + + args = list(argv) + while args and args[0].startswith('-'): + arg = args.pop(0) + if arg == '-h': + usage() + return 0 + elif arg == '-s': + strict_dtc = True + elif arg == '-t': + try_all = True + elif arg == '-v': + verbose = True + else: + fatal(f"Unknown option '{arg}'") + + requested_overlays = args + reporter = Reporter() + + kerndir = subprocess.run(['git', 'rev-parse', '--show-toplevel'], + capture_output=True, text=True).stdout.strip() + if not kerndir or not os.path.isdir(os.path.join(kerndir, 'kernel')): + fatal("This isn't a Linux repository") + + dtspath = os.path.join(kerndir, 'arch/arm/boot/dts/broadcom') \ + if os.path.isfile(os.path.join(kerndir, 'arch/arm/boot/dts/broadcom/Makefile')) \ + else os.path.join(kerndir, 'arch/arm/boot/dts') + dts64path = os.path.join(kerndir, 'arch/arm64/boot/dts/broadcom') \ + if os.path.isfile(os.path.join(kerndir, 'arch/arm64/boot/dts/broadcom/Makefile')) \ + else os.path.join(kerndir, 'arch/arm64/boot/dts') + + base32, base64_, extra32, extra64 = [], [], [], [] + for base in BASE_FILES: + if os.path.isfile(os.path.join(dtspath, base + '.dts')): + base32.append(base) + elif os.path.isfile(os.path.join(dts64path, base + '.dts')): + base64_.append(base) + for base in EXTRA_BASE_FILES: + if os.path.isfile(os.path.join(dtspath, base + '.dts')): + extra32.append(base) + elif os.path.isfile(os.path.join(dts64path, base + '.dts')): + extra64.append(base) + + dtsgroups = [(dts64path, base64_), (dtspath, base32)] + extra_dtsgroups = [(dts64path, extra64), (dtspath, extra32)] + all_dtsgroups = dtsgroups + extra_dtsgroups + + overlaysdir = os.path.join(kerndir, 'arch/arm/boot/dts/overlays') + + cpp = find_cpp() + dtc = os.path.join(kerndir, 'scripts/dtc/dtc') + dtc_opts = compute_dtc_opts(dtc, strict_dtc) + + exclusions_file = os.path.join(_HERE, 'overlaycheck_exclusions.txt') + ignore_missing, ignore_vestigial = parse_exclusions(exclusions_file, reporter) + documentation = {} + + readme = parse_readme(os.path.join(overlaysdir, 'README'), ignore_missing, + ignore_vestigial, documentation, reporter) + + source = parse_source_files(overlaysdir, all_dtsgroups, reporter) + + tmpdir = tempfile.mkdtemp(prefix='overlaycheck2-') + merged_dtb = os.path.join(tmpdir, 'merged.dtb') + + try: + for group_dir, bases in all_dtsgroups: + for base in bases: + dtc_cpp(cpp, dtc, dtc_opts, kerndir, group_dir, base + '.dts', + os.path.join(tmpdir, base + '.dtb'), reporter) + + if not requested_overlays: + check_readme_vs_source(source, readme, ignore_missing, ignore_vestigial, + documentation, reporter) + makefile = parse_makefile(os.path.join(overlaysdir, 'Makefile'), reporter) + check_makefile_vs_source(source, makefile, reporter) + overlays = sorted(k for k in source.keys() if not k.startswith('<')) + else: + overlays = requested_overlays + + overlay_map = os.path.join(overlaysdir, 'overlay_map.dts') + if os.path.isfile(overlay_map): + dtc_cpp(cpp, dtc, dtc_opts, kerndir, overlaysdir, 'overlay_map.dts', + os.path.join(tmpdir, 'overlay_map.dtb'), reporter) + + for overlay in overlays: + if overlay.startswith('<'): + continue + dtc_cpp(cpp, dtc, dtc_opts, kerndir, overlaysdir, overlay + '-overlay.dts', + os.path.join(tmpdir, overlay + '.dtbo'), reporter) + + base0_dts_path = find_dts_path(BASE_FILES[0], all_dtsgroups) + + for overlay in overlays: + if overlay.startswith('<'): + continue + check_one_overlay(overlay, overlaysdir, tmpdir, merged_dtb, base0_dts_path, + source, reporter, verbose) + + if try_all: + run_try_all(overlays, source, tmpdir, merged_dtb, verbose, reporter) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + print("Failed" if reporter.fail else "OK") + return 1 if reporter.fail else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/ovmerge/ovmerge.py b/ovmerge/ovmerge.py new file mode 100755 index 0000000..3ac181f --- /dev/null +++ b/ovmerge/ovmerge.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 + +# Author: Phil Elwell +# Copyright (c) 2018-2026, Raspberry Pi Ltd. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer, +# without modification. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The names of the above-listed copyright holders may not be used +# to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Python port of the Perl ovmerge tool. CLI wrapper - the DT tokeniser, +# parser, and merge/apply engine live in ovmerge_engine.py. + +import re +import sys + +from ovmerge_engine import ( + S, OvMergeError, dtparse, dtdump, open_source_lines, + get_prop_string, get_child, get_prop, set_prop, add_label, resolve_label, + dtparam, get_node, ovapply1, get_fragments, delete_node, delete_prop, + adj_ref, get_labels, ovstrip, renumber_fragments, ovmerge, ovapply2, + is_node_empty, add_node, PROPS, +) + + +# Main +# --------------------------------------------------------------------------- + +def usage(): + w = sys.stderr.write + w("Usage: ovmerge \n") + w(" where is the name of an overlay, optionally followed by\n") + w(" a comma-separated list of parameters, each with optional '='\n") + w(" assignments. The presence of any parameters, or a comma followed by\n") + w(" no parameters, removes the parameter declarations from the merged\n") + w(" overlay to avoid a potential name clash.\n") + w(" and are any of:\n") + w(" -b Read files from specified git branch\n") + w(" -B Set a debug breakpoint on the specified token number\n") + w(" -c Include 'redo' comment with command line (c.f. '-r')\n") + w(" -e Expand mode - list non-skipped lines in order of inclusion\n") + w(" -f Force some errors to be ignored\n") + w(" -h Display this help info\n") + w(" -i Show include hierarchy for each file\n") + w(" -l Like expand mode, but labels each line with source file\n") + w(" -n No .dts file header (just parsing .dtsi files)\n") + w(" -N Don't renumber overlay fragments (not guaranteed to work)\n") + w(" -p Emulate Pi firmware manipulation\n") + w(" -q Query mode (no output, just the success/failure return code)\n") + w(" -r Redo command comment in named files (c.f. '-c')\n") + w(" -s Sort nodes and properties (for easy comparison)\n") + w(" -S Instead of tabs, use 'n' spaces for indentation\n") + w(" -t Trace the tree changes\n") + w(" -T Trace the parsing process\n") + w(" -w Show warnings\n") + sys.exit(1) + + +def run(argv): + args = list(argv) + cmdline = [] + redo_comments = [] + + while args and args[0].startswith('-') and args[0] != '-': + arg = args.pop(0) + + if arg == '-b': + if not args: + print("* Branch parameter missing", file=sys.stderr) + usage() + S.branch = args.pop(0) + cmdline += [arg, S.branch] + elif arg == '-B': + if not args: + print("* Breakpoint number missing", file=sys.stderr) + usage() + S.bkpt = int(args.pop(0)) + elif arg == '-c': + S.comment = True + elif arg == '-e': + S.expand = True + elif arg == '-h': + usage() + elif arg == '-i': + S.show_includes = True + elif arg == '-l': + S.expand = True + S.expand_label = True + elif arg == '-n': + S.no_dts = True + elif arg == '-p': + S.pi_extras = True + cmdline.append(arg) + elif arg == '-r': + if args: + lines = open_source_lines(args[0]) + else: + lines = sys.stdin.readlines() + it = iter(lines) + firstline = next(it, '') + m = re.match(r'^// redo: ovmerge (.*)', firstline) + if not m: + print("* Redo but input has no 'redo:' comment", file=sys.stderr) + usage() + args = re.split(r'\s+', m.group(1).strip()) + for line in it: + line = line.rstrip('\n') + if re.match(r'^/dts-v1/', line): + break + redo_comments.append(line) + elif arg == '-s': + S.sort = True + cmdline.append(arg) + elif arg == '-t': + S.trace_tree = True + elif arg == '-T': + S.trace_parse = True + elif arg == '-w': + S.warnings = True + elif arg == '-q': + S.query = True + elif arg == '-f': + S.force = True + elif arg == '-N': + S.no_renumber = True + elif arg == '-S': + if not args: + print("* Spaces count parameter missing", file=sys.stderr) + usage() + indent_spaces = args.pop(0) + if not re.fullmatch(r'\d+', indent_spaces): + print("* Invalid spaces count parameter. Expected an integer.", file=sys.stderr) + usage() + S.indent_str = ' ' * int(indent_spaces) + cmdline += [arg, indent_spaces] + else: + print(f"* Unknown option '{arg}'", file=sys.stderr) + usage() + + if not args: + usage() + + cmdline += args + + overlays = [] + + for overlay in args: + if S.trace_tree: + print(f"[ overlay {overlay} ]") + + m = re.match(r'^([^,:]+)', overlay) + ovname = m.group(1) if m else '' + overlay_rest = overlay[len(ovname):] + dt = dtparse(ovname, S.no_dts) + apply_params = overlay_rest.startswith(',') + + if S.show_includes or S.expand: + continue + + model = get_prop_string(dt.root, 'model') + + S.cur_dt = dt + + if model and re.match(r'^Raspberry Pi', model) and S.pi_extras: + aliases = get_child(dt.root, 'aliases') + i2c = get_prop(aliases, 'i2c1')[1] + set_prop(aliases, 'i2c', i2c) + set_prop(aliases, 'i2c_arm', i2c) + + i2c_node = resolve_label(dt, i2c[1]) + add_label(dt, i2c_node, 'i2c_arm') + + i2c = get_prop(aliases, 'i2c0')[1] + set_prop(aliases, 'i2c_vc', i2c) + + i2c_node = resolve_label(dt, i2c[1]) + add_label(dt, i2c_node, 'i2c_vc') + + overrides = get_child(dt.root, '__overrides__') + i2c_prop = get_prop(overrides, 'i2c1') + prop_rest = i2c_prop[1:] + set_prop(overrides, 'i2c', *prop_rest) + set_prop(overrides, 'i2c_arm', *prop_rest) + + i2c_prop = get_prop(overrides, 'i2c0') + prop_rest = i2c_prop[1:] + set_prop(overrides, 'i2c_vc', *prop_rest) + + if apply_params: + seg_re = re.compile(r'[,:]([^=,]+)(?:=([^,]+))?') + pos = 0 + while True: + mm = seg_re.match(overlay_rest, pos) + if not mm: + break + pos = mm.end() + dtparam(dt, mm.group(1), mm.group(2) if mm.group(2) is not None else '') + if dt.plugin: + ovapply1(dt) + + exports = get_node(dt, '/__exports__') + if exports: + for symbol in exports[PROPS]: + expname = symbol[0] + adj_ref(1, expname) + + if apply_params: + delete_node(get_node(dt, '/__overrides__')) + for fragment in get_fragments(dt): + if get_child(fragment, '__dormant__'): + delete_node(fragment) + + payload = get_child(fragment, '__overlay__') + if payload is not None and get_prop(payload, 'dtoverlay,preserve-phandle'): + for label in get_labels(payload): + adj_ref(1, label) + delete_prop(payload, 'dtoverlay,preserve-phandle') + + S.cur_dt = None + + if dt.plugin and not S.no_renumber: + ovstrip(dt) + + overlays.append(dt) + + if not overlays: + sys.exit(0) + + if overlays[0].plugin: + if not S.no_renumber: + renumber_fragments(overlays[0], 0) + + for i in range(1, len(overlays)): + ovmerge(overlays[0], overlays[i]) + else: + base = overlays[0] + + if len(overlays) > 1: + symbols = get_child(base.root, '__symbols__') + if symbols is None: + symbols = add_node(base.root, '__symbols__') + + renumber_fragments(overlays[1], 0) + + for i in range(2, len(overlays)): + ovmerge(overlays[1], overlays[i]) + + ovapply2(base, overlays[1]) + + if is_node_empty(symbols): + delete_node(symbols) + + out = sys.stdout + + if S.comment: + parts = ['// redo: ovmerge -c'] + for opt in cmdline: + if re.search(r'\s', opt): + parts.append(f" '{opt}'") + else: + parts.append(f" {opt}") + out.write(''.join(parts) + "\n") + if redo_comments: + out.write("\n".join(redo_comments)) + out.write("\n") + + if not S.query: + dtdump(overlays[0], out) + + sys.exit(S.retcode) + + +def main(): + try: + run(sys.argv[1:]) + except OvMergeError as e: + print(str(e), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/ovmerge/ovmerge_engine.py b/ovmerge/ovmerge_engine.py new file mode 100644 index 0000000..e45eb60 --- /dev/null +++ b/ovmerge/ovmerge_engine.py @@ -0,0 +1,1728 @@ +# Author: Phil Elwell +# Copyright (c) 2018-2026, Raspberry Pi Ltd. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer, +# without modification. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The names of the above-listed copyright holders may not be used +# to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Device Tree engine: tokeniser, parser, in-memory node/property tree, and +# overlay merge/apply logic used by ovmerge.py. Split out into its own module +# so other tools (e.g. overlaycheck) can drive the same DT model in-process +# instead of shelling out to 'ovmerge' and parsing its text output. +# Fundamental types +# '' - string +# '#' - 64-bit +# ':' - 32-bit integer +# ';' - 16-bit integer +# '.' - 8-bit integer (byte) +# '?' - boolean +# '!' - inverted boolean +# '[' - byte array // Byte array syntax accepts but doesn't require colons between bytes +# 'prop[' // Interpret value as byte array and assign to prop +# 'prop[=00:01:02' // property = literal byte array +# +# Operations on a fundamental type: +# // N.B. The type must go before the list so we know how to interpret it. +# +# 'reg:0' // set reg and unit address to value +# 'reg:0=0' // set reg and unit address to the supplied literal +# // The reg property is only set if it already exists. +# +# 'name' // Assigning to name property automatically sets the node name +# // The name property is only set if it already exists. +# +# '=' - literal assignment (if a string, the literal after the =), if an integer, +# either the in-band integer or the next cell (useful for phandles). +# "prop=foo" // String literal assignment +# "prop=", &spi // String path literal assignment (the path to the node is substituted, used for aliases) +# "prop:0=0" // Integer literal assignment +# "prop:0=", <&spi>; // Integer cell assignment +# +# '{...}' - use the value as the key to an element in the set. The usual type indicators apply. +# 'prop{a='alpha',b='bravo',c='charlie'}"; +# 'prop:0{0=",<&i2c0>,"1=",<&i2c1>,"3=0x2a}"; + +import os +import re +import sys +import subprocess + +ELEM_SIZES = { + '"': 0, # string + '.': 1, # byte + ';': 2, # 16-bit int + ':': 4, # 32-bit int + '#': 8, # 64-bit int +} + + +class OvMergeError(Exception): + pass + + +class State: + def __init__(self): + self.branch = None + self.comment = False + self.expand = False + self.expand_label = False + self.show_includes = False + self.pi_extras = False + self.sort = False + self.warnings = False + self.no_dts = False + self.no_renumber = False + self.cur_dt = None + self.retcode = 0 + self.query = False + self.force = False + self.trace_parse = False + self.trace_tree = False + self.trace_prop = '' + self.trace_label = '' + self.bkpt = 0 + self.indent_str = "\t" + + +S = State() + +# Node layout: [name, props, children, labels, parent, depth] +NAME, PROPS, CHILDREN, LABELS, PARENT, DEPTH = range(6) + + +def new_node(name): + return [name, [], [], [], None, 0] + + +class DT: + def __init__(self): + self.root = None + self.plugin = False + self.labels = {} + self.refcount = {} + self.includes = [] + self.memreserves = [] + self.defines = {} + self.frag_count = 0 + + +class FileMarker: + __slots__ = ('filename',) + + def __init__(self, filename): + self.filename = filename + + +class PState: + __slots__ = ('tokens', 'pos', 'file') + + def __init__(self, tokens): + self.tokens = tokens + self.pos = 0 + self.file = None + + +# --------------------------------------------------------------------------- +# Small value helpers +# --------------------------------------------------------------------------- + +def byte_array_value(value): + arr = [] + for val in re.split(r'[: ]', value): + if not re.fullmatch(r'([0-9a-fA-F][0-9a-fA-F])*', val): + raise OvMergeError(f"* invalid bytestring at '{val}'") + for i in range(0, len(val), 2): + arr.append(int(val[i:i + 2], 16)) + return arr + + +def byte_array_string(arr): + return ' '.join('%02x' % v for v in arr) + + +_EXPR_TOKEN_RE = re.compile( + r"\s*(0[xX][0-9a-fA-F]+|0[bB][01]+|0[0-7]+|[0-9]+|<<|>>|[()+\-*/%&|^~])") + + +def _tokenize_expr(s): + tokens = [] + i = 0 + n = len(s) + while i < n: + m = _EXPR_TOKEN_RE.match(s, i) + if not m: + if s[i:].strip() == '': + break + raise OvMergeError(f"* Bad integer value '{s}'") + tokens.append(m.group(1)) + i = m.end() + return tokens + + +def _parse_expr_literal(tok): + if re.match(r'^0[xX]', tok): + return int(tok, 16) + if re.match(r'^0[bB]', tok): + return int(tok, 2) + if re.match(r'^0[0-7]+$', tok): + return int(tok, 8) + return int(tok, 10) + + +def _c_div(a, b): + q = abs(a) // abs(b) + if (a < 0) != (b < 0): + q = -q + return q + + +def _c_mod(a, b): + return a - _c_div(a, b) * b + + +class _ExprParser: + def __init__(self, tokens): + self.tokens = tokens + self.i = 0 + + def peek(self): + return self.tokens[self.i] if self.i < len(self.tokens) else None + + def take(self): + t = self.tokens[self.i] + self.i += 1 + return t + + def parse(self): + v = self.parse_or() + if self.i != len(self.tokens): + raise OvMergeError("* Trailing tokens in integer expression") + return v + + def parse_or(self): + v = self.parse_xor() + while self.peek() == '|': + self.take() + v = v | self.parse_xor() + return v + + def parse_xor(self): + v = self.parse_and() + while self.peek() == '^': + self.take() + v = v ^ self.parse_and() + return v + + def parse_and(self): + v = self.parse_shift() + while self.peek() == '&': + self.take() + v = v & self.parse_shift() + return v + + def parse_shift(self): + v = self.parse_add() + while self.peek() in ('<<', '>>'): + op = self.take() + r = self.parse_add() + v = (v << r) if op == '<<' else (v >> r) + return v + + def parse_add(self): + v = self.parse_mul() + while self.peek() in ('+', '-'): + op = self.take() + r = self.parse_mul() + v = v + r if op == '+' else v - r + return v + + def parse_mul(self): + v = self.parse_unary() + while self.peek() in ('*', '/', '%'): + op = self.take() + r = self.parse_unary() + if op == '*': + v = v * r + elif op == '/': + v = _c_div(v, r) + else: + v = _c_mod(v, r) + return v + + def parse_unary(self): + if self.peek() == '-': + self.take() + return -self.parse_unary() + if self.peek() == '~': + self.take() + return ~self.parse_unary() + if self.peek() == '+': + self.take() + return self.parse_unary() + return self.parse_primary() + + def parse_primary(self): + t = self.take() + if t == '(': + v = self.parse_or() + if self.take() != ')': + raise OvMergeError("* Mismatched parens in integer expression") + return v + return _parse_expr_literal(t) + + +def eval_expr(s): + tokens = _tokenize_expr(s) + if not tokens: + raise OvMergeError(f"* Bad integer value '{s}'") + return _ExprParser(tokens).parse() + + +def integer_value(value, size): + if value is None: + return None + if re.fullmatch(r'(y|yes|on|true|down)?', value): + return 1 + if re.fullmatch(r'n|no|off|false|none', value): + return 0 + if re.fullmatch(r'up', value): + return 2 + if value.startswith('&'): + if size != 4: + raise OvMergeError(f"* Label '{value}' used as non-32-bit integer") + return value + if re.match(r'^[0-9]', value): + mask = {1: 0xff, 2: 0xffff, 4: 0xffffffff, 8: 0xffffffffffffffff}.get(size) + if mask is None: + raise OvMergeError(f"* Bad size '{size}' for integer") + return eval_expr(value) & mask + if re.fullmatch(r'[A-Z][A-Z0-9_]+', value): + return value + if re.fullmatch(r'\(.+\)', value): + return value + raise OvMergeError(f"* Bad integer value '{value}'") + + +def boolean_value(value, strict): + if re.fullmatch(r'(y|yes|on|true|okay)?', value): + return True + if re.fullmatch(r'n|no|off|false|disabled', value): + return False + if not re.match(r'^[0-9]', value): + if strict: + raise OvMergeError(f"* Bad boolean value '{value}'") + return value != "" + m = re.match(r'[0-9]+', value) + return int(m.group(0)) != 0 + + +# --------------------------------------------------------------------------- +# Node / label / property helpers +# --------------------------------------------------------------------------- + +def add_node(parent, name): + node = name if isinstance(name, list) else new_node(name) + if S.trace_tree: + print("[ add_node %s -> %s ]" % (node[NAME] if node[NAME] is not None else "?", + parent[NAME] if parent else "-")) + node[PARENT] = parent + if parent is not None: + node[DEPTH] = parent[DEPTH] + 1 + parent[CHILDREN].append(node) + else: + if name != '/': + raise OvMergeError(f"* Invalid root node '{name}'") + node[DEPTH] = 0 + S.cur_dt.root = node + return node + + +def get_node(dt, path): + node = dt.root + m = re.match(r'^([^/]+)(/|$)', path) + if m: + path = '/' + path[m.end():] + node = resolve_alias(dt, m.group(1)) + if path == '/': + return node + pos = 0 + seg_re = re.compile(r'/([-a-zA-Z0-9,._+#@]+)') + while node is not None: + m2 = seg_re.match(path, pos) + if not m2: + break + node = get_child(node, m2.group(1)) + pos = m2.end() + return node + + +def is_node_empty(node): + return not get_children(node) and not get_props(node) + + +def get_child(node, name): + if node is not None: + for child in node[CHILDREN]: + if child[NAME] == name or ('@' not in name and re.match(name + '@', child[NAME])): + return child + return None + else: + if name == '/': + return S.cur_dt.root if S.cur_dt else None + return None + + +def _lenient_hex(s): + # Mirrors Perl's hex(), which parses a leading run of hex digits and + # warns (but doesn't fail) about anything else, defaulting to 0. + m = re.match(r'[0-9a-fA-F]*', s) + digits = m.group(0) if m else '' + return int(digits, 16) if digits else 0 + + +def _by_addr_key(node): + m = re.search(r'@(.*)$', node[NAME]) + return _lenient_hex(m.group(1)) if m else None + + +import functools + + +def _by_addr_cmp(a, b): + a_addr = _by_addr_key(a) + b_addr = _by_addr_key(b) + if a_addr is not None and b_addr is not None: + return (a_addr > b_addr) - (a_addr < b_addr) + if a_addr is not None: + return -1 + if b_addr is not None: + return 1 + return (a[NAME] > b[NAME]) - (a[NAME] < b[NAME]) + + +def get_children(node): + if S.sort: + return sorted(node[CHILDREN], key=functools.cmp_to_key(_by_addr_cmp)) + return list(node[CHILDREN]) + + +def get_labels(node): + if S.sort: + return sorted(node[LABELS]) + return list(node[LABELS]) + + +def node_path(node): + if node[NAME] == '/': + return '/' + parent_path = node_path(node[PARENT]) + if parent_path == '/': + parent_path = '' + return parent_path + '/' + node[NAME] + + +def add_label(dt, node, label, move_from=None): + if S.trace_tree: + print("[ add_label %s -> %s ]" % (label, node[NAME])) + old_value = dt.labels.get(label) + if old_value is not None: + if old_value is node: + return + if move_from is not None and old_value is not move_from: + raise OvMergeError(f"* Label '{label}' redefined") + if move_from is not None: + move_from[LABELS] = [l for l in move_from[LABELS] if l != label] + dt.labels[label] = node + node[LABELS].append(label) + if S.warnings and len(node[LABELS]) > 1: + print("* Multiple labels on '" + node_path(node) + "'") + + +def resolve_label(dt, label): + return dt.labels.get(label) + + +def resolve_alias(dt, alias): + aliases = get_node(dt, '/aliases') + prop = get_prop(aliases, alias) + if prop is None: + return None + if prop[1][0] == '&': + return resolve_label(dt, prop[1][1]) + else: + return get_node(dt, prop[1][1]) + + +def fragment_of(node): + if node is None: + return None + if re.match(r'^fragment@', node[NAME]): + return node + return fragment_of(node[PARENT]) + + +# --------------------------------------------------------------------------- +# Property helpers +# --------------------------------------------------------------------------- + +def get_prop_string(node, name): + prop = get_prop(node, name) + if prop is None: + return None + if len(prop) != 2: + return None + if prop[1][0] != '"': + return None + return prop[1][1] + + +def get_prop(node, name): + if node is None: + return None + for prop in node[PROPS]: + if prop[0] == name: + return prop + return None + + +def get_props(node): + if S.sort: + return sorted(node[PROPS], key=lambda p: p[0]) + return list(node[PROPS]) + + +def add_prop(node, name, *vals): + new = [name] + list(vals) + node[PROPS].append(new) + return new + + +def set_prop(node, name, *vals): + if name == S.trace_prop: + pass + adj_val_refs(1, vals) + for prop in node[PROPS]: + if prop[0] == name: + adj_val_refs(-1, prop[1:]) + prop[1:] = list(vals) + return prop + return add_prop(node, name, *vals) + + +def apply_prop(node, name, *vals): + vals = list(vals) + if name == 'status': + vals = [['"', 'okay' if boolean_value(vals[0][1], True) else 'disabled']] + elif name == 'bootargs': + vals = [['"', get_prop(node, name)[1][1] + ' ' + vals[0][1]]] + return set_prop(node, name, *vals) + + +def delete_prop(node, name): + if name == S.trace_prop: + pass + for i, prop in enumerate(node[PROPS]): + if prop[0] == name: + adj_val_refs(-1, prop[1:]) + return node[PROPS].pop(i) + return None + + +def find_prop_chunk(node, propname, offset, size, ovrname, create): + prop = get_prop(node, propname) + if prop is None and create: + prop = set_prop(node, propname, ['<', size, []]) + if prop is None: + return (None, 0) + + chunk = None + pos = 0 + for i in range(1, len(prop)): + chunk = prop[i] + typ = chunk[0] + if typ == '"': + end = pos + len(chunk[1]) + 1 + elif typ == '[': + end = pos + len(chunk[2]) + else: + end = pos + chunk[1] * len(chunk[2]) + if offset < end: + break + pos = end + + if chunk is None and create: + chunk = ['<', size, []] + prop.append(chunk) + + offset -= pos + if size and offset % size: + raise OvMergeError(f"* Unaligned override '{ovrname}', property {propname}") + return (chunk, offset // size if size else 0) + + +def adj_val_refs(inc, vals): + for val in vals: + if val[0] == '&': + adj_ref(inc, val[1]) + elif val[0] == '<': + for elem in val[2]: + if isinstance(elem, str) and elem.startswith('&'): + adj_ref(inc, elem[1:]) + + +def adj_ref(inc, label): + if S.cur_dt is None: + return + S.cur_dt.refcount[label] = S.cur_dt.refcount.get(label, 0) + inc + if label == S.trace_label: + print("[ ref %s -> %s ]" % (label, S.cur_dt.refcount[label])) + + +# --------------------------------------------------------------------------- +# Vectors / overrides +# --------------------------------------------------------------------------- + +def get_vector(p, size, length=None): + if p is None: + return None + if (p[0] == '<' or p[0] == '[') and (p[1] == size or (p[1] == 4 and size == 8)) and \ + (length is None or length == len(p[2])): + return p[2] + return None + + +def get_label_ref(p): + vector = get_vector(p, 4, 1) + if vector is not None and re.fullmatch(r'&.*|0', vector[0]): + return vector[0] + return None + + +def _aget(lst, idx): + return lst[idx] if 0 <= idx < len(lst) else None + + +def parse_lookup_table(table, ovr, ppos, value): + val = None + have_val = False + pos = 0 + pattern = re.compile(r"(?:'([^']*)'|([^=,}]*))(?:=(?:'([^']*)'|([^,}]*)))?([,}])?") + + while True: + m = pattern.match(table, pos) + if not m: + break + if m.end() == pos and m.group(0) == '': + break + pos = m.end() + g1, g2, g3, g4, sep = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5) + key = g1 if g1 is not None else g2 + sub = g3 if g3 is not None else g4 + if sep is None: + p = _aget(ovr, ppos[0]) + ppos[0] += 1 + vec = get_vector(p, 4, 1) + sub = vec[0] if vec is not None else None + nxt = _aget(ovr, ppos[0]) + ppos[0] += 1 + if nxt is None: + sep = '}' + else: + if nxt[0] != '"': + raise OvMergeError("* Expected string in lookup table") + table = nxt[1] + pos = 0 + mm = re.match(r'(\})', table) + if mm: + sep = mm.group(1) + if value is not None: + if key == '': + if not have_val: + val = sub if sub is not None else value + have_val = True + elif key == value: + val = sub if sub is not None else key + have_val = True + if (sep or '') == '}': + break + + if value is not None and val is None: + raise OvMergeError(f"* No match for '{value}'") + + return val + + +def parse_fragment_ops(decl): + ops = [] + p = 0 + while True: + m = re.match(r'([=!+-])(\d+)', decl[p:]) + if not m: + break + ops.append((m.group(1), m.group(2))) + p += m.end() + return ops + + +def dtparam(dt, param, value, wakeable=None, pinctrl_refs=None, label_use=None): + overrides = get_node(dt, '/__overrides__') + if overrides is None: + raise OvMergeError("* No overrides found") + ovr = get_prop(overrides, param) + if ovr is None: + raise OvMergeError(f"* dtparam '{param}' not found") + + pos = 1 + n = len(ovr) + while pos < n: + p = ovr[pos] + pos += 1 + label = get_label_ref(p) + if label is None: + raise OvMergeError(f"* Invalid override 1: {param}") + p = _aget(ovr, pos) + pos += 1 + if p is None or p[0] != '"': + raise OvMergeError(f"* Invalid override 2: {param}") + decl = p[1] + + m_label = re.match(r'^&(.*)', label) + if m_label: + node = resolve_label(dt, m_label.group(1)) + if node is None: + raise OvMergeError(f"* Override '{param}' targets unknown label '{m_label.group(1)}'") + + if label_use is not None: + pm = re.match(r'^([-a-zA-Z0-9_,]+)', decl) + if pm: + label_use(m_label.group(1), pm.group(1)) + + m = re.match(r'^([-a-zA-Z0-9_,]+)([.;:#])(\d+)(?:(=|\{)(.*))?$', decl) + if m: + prop, typ, offset, op, opdata = m.group(1), m.group(2), int(m.group(3)), \ + (m.group(4) or ''), m.group(5) + size = ELEM_SIZES[typ] + val = value + if op == '=': + if opdata: + val = opdata + else: + vector = get_vector(_aget(ovr, pos), 4, 1) + pos += 1 + if vector is None: + raise OvMergeError(f"* Expected cell value in parameter '{param}'") + val = vector[0] + elif op == '{': + ppos = [pos] + val = parse_lookup_table(opdata, ovr, ppos, value) + pos = ppos[0] + + intval = integer_value(val, size) + if (pinctrl_refs is not None and prop == 'pinctrl-0' and + isinstance(intval, str) and intval.startswith('&')): + pinctrl_refs.add(intval[1:]) + if value is not None and prop == 'reg': + regval = format(intval & 0xffffffff, 'x') if isinstance(intval, int) else '0' + node[NAME] = re.sub(r'@[0-9a-fA-F]*$', '@' + regval, node[NAME]) + + chunk, chunk_idx = find_prop_chunk( + node, prop, offset, size, param, (value is not None) and prop != 'reg') + + if chunk is not None: + vector = get_vector(chunk, size) + if vector is None: + raise OvMergeError(f"* Probably incorrect override property type for '{prop}'") + + if value is not None: + for i in range(len(vector), chunk_idx): + vector.append(0) + if chunk_idx < len(vector): + vector[chunk_idx] = intval + else: + vector.append(intval) + + elif re.match(r'^([-a-zA-Z0-9_,]+)([?!])(?:(=|\{)(.*))?$', decl): + mm = re.match(r'^([-a-zA-Z0-9_,]+)([?!])(?:(=|\{)(.*))?$', decl) + prop, sense, op, opdata = mm.group(1), mm.group(2), (mm.group(3) or ''), mm.group(4) + val = value + if op == '=': + if opdata: + val = opdata + else: + vector = get_vector(_aget(ovr, pos), 4, 1) + pos += 1 + if vector is None: + raise OvMergeError(f"* Expected cell value in parameter '{param}'") + val = vector[0] + elif op == '{': + ppos = [pos] + val = parse_lookup_table(opdata, ovr, ppos, value) + pos = ppos[0] + + if value is not None: + boolval = boolean_value(val, True) + if sense == '!': + boolval = not boolval + if boolval: + set_prop(node, prop) + else: + delete_prop(node, prop) + + elif re.match(r'^([-a-zA-Z0-9_,]+)\[(?:(=|\{)(.*))?$', decl): + mm = re.match(r'^([-a-zA-Z0-9_,]+)\[(?:(=|\{)(.*))?$', decl) + prop, op, opdata = mm.group(1), (mm.group(2) or ''), mm.group(3) + val = value + if op == '=': + val = opdata + elif op == '{': + ppos = [pos] + val = parse_lookup_table(opdata, ovr, ppos, value) + pos = ppos[0] + if value is not None: + apply_prop(node, prop, ['[', 1, byte_array_value(val)]) + + elif re.match(r'^([-a-zA-Z0-9_,]+)(?:(=|\{)(.*))?$', decl): + mm = re.match(r'^([-a-zA-Z0-9_,]+)(?:(=|\{)(.*))?$', decl) + prop, op, opdata = mm.group(1), (mm.group(2) or ''), mm.group(3) + val = value + if op == '=': + if opdata: + val = opdata + elif pos == n: + val = '' + else: + val = _aget(ovr, pos) + pos += 1 + if val is None or (val[0] != '"' and val[0] != '&'): + raise OvMergeError( + f"* Expected a string or label reference in parameter '{param}'") + elif op == '{': + ppos = [pos] + val = parse_lookup_table(opdata, ovr, ppos, value) + pos = ppos[0] + if value is not None: + if prop == 'name': + node[NAME] = val + else: + apply_prop(node, prop, val if isinstance(val, list) else ['"', val]) + + else: + raise OvMergeError(f"* Invalid parameter declaration '{decl}'") + else: + for op, num in parse_fragment_ops(decl): + frag = get_node(dt, '/fragment-' + num) or get_node(dt, '/fragment@' + num) + if frag is None: + raise OvMergeError(f"* Param {param}: no fragment {num}") + if wakeable is not None and op != '-': + wakeable.add(num) + if value is not None: + boolval = boolean_value(value, False) + if op == '!': + boolval = not boolval + elif op == '+': + boolval = True + elif op == '-': + boolval = False + frag[CHILDREN][0][NAME] = '__overlay__' if boolval else '__dormant__' + + +def find_wakeable_fragments(dt): + """Fragment numbers ('0', '1', ...) reachable from a __dormant__ state via + some __overrides__ declaration (any op other than '-' on that fragment).""" + wakeable = set() + overrides = get_node(dt, '/__overrides__') + for prop in (get_props(overrides) if overrides is not None else []): + dtparam(dt, prop[0], None, wakeable=wakeable) + return wakeable + + +# --------------------------------------------------------------------------- +# Tree mutation +# --------------------------------------------------------------------------- + +def remove_node(node): + parent = node[PARENT] + if S.trace_tree: + print("[ remove_node %s ]" % node[NAME]) + if parent is None: + return + node[PARENT] = None + found = None + for i, child in enumerate(parent[CHILDREN]): + if child is node: + found = i + break + if found is None: + raise OvMergeError("* Internal error - wrong parent/missing child") + del parent[CHILDREN][found] + + +def delete_node(node): + if node is None: + return None + + if S.trace_tree: + print("[ delete node %s ]" % node[NAME]) + remove_node(node) + + for label in get_labels(node): + S.cur_dt.labels.pop(label, None) + + for prop in get_props(node): + adj_val_refs(-1, prop[1:]) + + while node[CHILDREN]: + delete_node(node[CHILDREN][0]) + + return True + + +def relabel_node(node, transform, depth): + for prop in get_props(node): + if depth > 0: + for chunk in prop[1:]: + if chunk[0] == '<': + vals = chunk[2] + for idx in range(len(vals)): + term = vals[idx] + if isinstance(term, str): + m = re.match(r'^&(.*)', term) + if m: + newlabel = transform.get(m.group(1)) + if newlabel: + adj_ref(-1, m.group(1)) + adj_ref(1, newlabel) + vals[idx] = '&' + newlabel + elif chunk[0] == '&': + newlabel = transform.get(chunk[1]) + if newlabel: + adj_ref(-1, chunk[1]) + adj_ref(1, newlabel) + chunk[1] = newlabel + + for subnode in get_children(node): + relabel_node(subnode, transform, depth + 1) + + +def apply_node(base, dst, src): + for prop in get_props(src): + apply_prop(dst, prop[0], *prop[1:]) + + for label in get_labels(src): + add_label(base, dst, label, src) + + for subsrc in get_children(src): + subdst = get_child(dst, subsrc[NAME]) + if subdst and not S.force: + raise OvMergeError(f"* Subnode {subsrc[NAME]} already exists") + else: + subdst = add_node(dst, subsrc[NAME]) + apply_node(base, subdst, subsrc) + + +# --------------------------------------------------------------------------- +# Overlay merging +# --------------------------------------------------------------------------- + +def get_fragments(ov): + fragments = [] + for child in get_children(ov.root): + if re.match(r'^fragment[@-](\d+)$', child[NAME]): + fragments.append(child) + return fragments + + +def renumber_fragments(ov, offset): + fragments = [] + remap = {} + count = 0 + overrides = None + + for child in get_children(ov.root): + m = re.fullmatch(r'fragment([@-])(\d+)', child[NAME]) + if m: + sep, num = m.group(1), int(m.group(2)) + remap[num] = count + offset + child[NAME] = 'fragment%s%d' % (sep, count + offset) + fragments.append(child) + count += 1 + elif child[NAME] == '__overrides__': + overrides = child + + ov.frag_count = count + + if overrides is None: + return + + for ovr in overrides[PROPS]: + pos = 1 + n = len(ovr) + while pos + 1 < n: + if (get_label_ref(ovr[pos]) or '') == '0': + pos += 1 + chunk = ovr[pos] + decl = chunk[1] + + p = 0 + while True: + m2 = re.match(r'[=!+-](\d+)', decl[p:]) + if not m2: + break + fragnum = int(m2.group(1)) + if fragnum not in remap: + raise OvMergeError( + "* override '" + str(ovr[0]) + + "}' references missing fragment " + str(fragnum)) + p += m2.end() + + out = [] + p = 0 + while True: + m2 = re.match(r'([=!+-])(\d+)', decl[p:]) + if not m2: + break + out.append(m2.group(1) + str(remap[int(m2.group(2))])) + p += m2.end() + out.append(decl[p:]) + chunk[1] = ''.join(out) + pos += 1 + + +def ovmerge(base, ov): + if not base.plugin or not ov.plugin: + raise OvMergeError("* Cannot merge a non-overlay") + + for inc in list(ov.includes): + set_add(base.includes, inc) + + renumber_fragments(ov, base.frag_count) + + transform = {} + base_labels = base.labels + ov_labels = ov.labels + + for l in list(ov_labels.keys()): + nl = l + n = ov_labels[l] + if base_labels.get(l): + i = 1 + while True: + nl = f"{l}_{i}" + if not base_labels.get(nl): + break + i += 1 + transform[l] = nl + for idx, ol in enumerate(n[LABELS]): + if ol == l: + n[LABELS][idx] = nl + base_labels[nl] = n + + relabel_node(ov.root, transform, 0) + + base_overrides = get_node(base, '/__overrides__') + ov_overrides = get_node(ov, '/__overrides__') + + if base_overrides: + remove_node(base_overrides) + + for child in get_fragments(ov): + add_node(base.root, child) + base.frag_count += 1 + + if ov_overrides: + if not base_overrides: + base_overrides = new_node('__overrides__') + for ovr in ov_overrides[PROPS]: + if get_prop(base_overrides, ovr[0]): + raise OvMergeError(f"* Duplicate parameter '{ovr[0]}'") + set_prop(base_overrides, ovr[0], *ovr[1:]) + + if base_overrides: + add_node(base.root, base_overrides) + + +def ovstrip(dt): + if S.trace_tree: + print("[ ovstrip ]") + S.cur_dt = dt + + unused = [key for key, value in dt.labels.items() if not dt.refcount.get(key)] + + for label in unused: + node = dt.labels[label] + del dt.labels[label] + for i, l in enumerate(node[LABELS]): + if l == label: + del node[LABELS][i] + break + + S.cur_dt = None + + +def ovapply1(ov): + if not ov.plugin: + raise OvMergeError("* Cannot apply a non-overlay") + + for fragment in get_fragments(ov): + overlay = get_child(fragment, '__overlay__') + if overlay is None: + continue + if S.trace_tree: + print("[ apply fragment %s ]" % fragment[NAME]) + target_node = None + target = get_prop(fragment, 'target') + if target: + label = get_label_ref(target[1]) + m = re.match(r'^&(.*)', label) if label is not None else None + if not m: + raise OvMergeError("* Invalid target reference") + target_node = ov.labels.get(m.group(1)) + if target_node: + apply_node(ov, target_node, overlay) + overlay[NAME] = '__dormant__' + + +def ovapply2(base, ov): + if not ov.plugin: + raise OvMergeError("* Cannot apply a non-overlay") + if base.plugin: + raise OvMergeError("* Cannot apply an overlay to an overlay") + + for inc in list(ov.includes): + set_add(base.includes, inc) + + for fragment in get_fragments(ov): + overlay = get_child(fragment, '__overlay__') + if overlay is None: + continue + if S.trace_tree: + print("[ apply fragment %s to base tree]" % fragment[NAME]) + target_node = None + target = get_prop(fragment, 'target') + if target: + label = get_label_ref(target[1]) + m = re.match(r'^&(.*)', label) if label is not None else None + if not m: + raise OvMergeError("* Invalid target reference") + target_node = base.labels.get(m.group(1)) + if target_node is None: + raise OvMergeError(f"* Label '{m.group(1)}' not found in base") + else: + target = get_prop(fragment, 'target-path') + if target[1][0] != '"': + raise OvMergeError("* Invalid target-path") + target_node = get_node(base, target[1][1]) + if target_node is None: + raise OvMergeError(f"* Path '{target[1][1]}' not found in base") + + apply_node(base, target_node, overlay) + + +def ordercheck(ov, applied): + for fragment in get_fragments(ov): + overlay = get_child(fragment, '__overlay__') or get_child(fragment, '__dormant__') + if overlay is None: + continue + target = get_prop(fragment, 'target') + if target: + label = get_label_ref(target[1]) + m = re.match(r'^&(.*)', label) if label is not None else None + if not m: + raise OvMergeError("* Invalid target reference") + target_node = ov.labels.get(m.group(1)) + if target_node: + target_fragment = fragment_of(target_node) + if applied.get(id(target_node)): + raise OvMergeError( + "* %s should precede %s" % + (fragment[NAME], target_fragment[NAME] if target_fragment else "fragment@?")) + set_applied(overlay, applied) + + +def set_applied(node, applied): + applied[id(node)] = True + for subnode in get_children(node): + set_applied(subnode, applied) + + +# --------------------------------------------------------------------------- +# Sets +# --------------------------------------------------------------------------- + +def set_add(lst, val): + if isinstance(val, list): + for existing in lst: + if existing is val: + return + lst.append(val) + else: + if val not in lst: + lst.append(val) + + +# --------------------------------------------------------------------------- +# Dump +# --------------------------------------------------------------------------- + +def dtdump(dt, out): + out.write("/dts-v1/;\n") + if dt.plugin: + out.write("/plugin/;\n") + out.write("\n") + + if dt.includes: + for inc in dt.includes: + out.write(f"#include {inc}\n") + out.write("\n") + + if dt.defines: + for name, value in dt.defines.items(): + out.write(f"#define {name} {value}\n") + out.write("\n") + + if dt.memreserves: + for res in dt.memreserves: + out.write(f"/memreserve/ {res[0]} {res[1]};\n") + out.write("\n") + + dump_node(dt.root, 0, out) + + +def dump_node(node, depth, out): + indent = S.indent_str * depth + + out.write(indent + ': '.join(get_labels(node) + [node[NAME]]) + " {\n") + + for prop in get_props(node): + terms = [] + out.write(indent + S.indent_str + prop[0]) + for chunk in prop[1:]: + if chunk[0] == '"': + terms.append('"' + chunk[1] + '"') + elif chunk[0] == '&': + terms.append('&' + chunk[1]) + elif chunk[0] == '<': + prefix = ("/bits/ %d " % (chunk[1] * 8)) if chunk[1] != 4 else '' + terms.append(prefix + '<' + ' '.join(str(v) for v in chunk[2]) + '>') + elif chunk[0] == '[': + terms.append('[' + byte_array_string(chunk[2]) + ']') + else: + terms.append('?') + if terms: + out.write(' = ' + ', '.join(terms)) + out.write(";\n") + + for subnode in get_children(node): + dump_node(subnode, depth + 1, out) + + out.write(indent + "};\n") + + +# --------------------------------------------------------------------------- +# Tokenizer +# --------------------------------------------------------------------------- + +_IF_RE = re.compile(r'^\s*#if(def|ndef)?\s+(\w+)') +_ELSE_RE = re.compile(r'^\s*#else') +_ENDIF_RE = re.compile(r'^\s*#endif') +_INCLUDE_RE = re.compile(r'^\s*(?:#include|/include/)\s+(["<][^">]+[">])\s*$') +_DEFINE_RE = re.compile(r'^\s*#define\s+(\w+)(?:\s+([^\r\n]+))?') +_UNDEF_RE = re.compile(r'^\s*#undef\s+(\w+)') +_BKPT_RE = re.compile(r'^\s*#bkpt') +_WS_RE = re.compile(r'\s*') +_WORD_RE = re.compile(r'\b(\w+)\b') +_PAREN_RE = re.compile(r'(.*?)([()])(\s*)') +_TAIL_RE = re.compile(r'(.*?)\s*$') +_COMMENT_END_RE = re.compile(r'.*?\*/') +_BLOCK_COMMENT_RE = re.compile(r'.*?\*/\s*') + +_TOKEN_RE = re.compile( + r'((?:/(?:dts-v1|plugin|memreserve|bits|delete-node|delete-property)/)' + r'|&[a-zA-Z_][a-zA-Z0-9_]*' + r'|[a-zA-Z_][a-zA-Z0-9_]*:' + r'|[-a-zA-Z0-9,._+#@]+' + r'|\(' + r'|"(?:[^\\"]|\\.)*"' + r"|'(?:[^']|\\.)*'" + r'|//' + r'|/\*' + r'|[/{};=<>,\[\]])') + + +def search_path(fname): + if S.branch: + r = subprocess.run(['git', 'cat-file', '-e', f'{S.branch}:./{fname}'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if r.returncode == 0: + return fname + if os.access(fname, os.R_OK): + return fname + return None + + +def open_source_lines(filename): + if S.branch: + proc = subprocess.run(['git', 'show', f'{S.branch}:./{filename}'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise OvMergeError(f"* Failed to open '{filename}'") + return proc.stdout.decode('utf-8', errors='replace').splitlines(keepends=True) + else: + try: + with open(filename, 'r') as f: + return f.readlines() + except OSError: + raise OvMergeError(f"* Failed to open '{filename}'") + + +def read_tokens(filename, depth, defines): + linenum = 0 + tokens = [FileMarker(filename)] + in_comment = False + if_count = 0 + hidden_count = 0 + expr = None + expr_level = 0 + filepath = re.sub(r'/?[^/]*$', '', filename) + if filepath: + filepath += '/' + + if S.show_includes: + print(" " * depth + filename) + if S.trace_parse: + print(f"[read_tokens '{filename}']") + if S.expand and not S.expand_label: + print(f"#### Start of '{filename}'") + + lines = open_source_lines(filename) + + for line in lines: + linenum += 1 + + if in_comment: + m = _COMMENT_END_RE.match(line) + if not m: + continue + line = line[m.end():] + in_comment = False + + m = _IF_RE.match(line) + if m: + mode = m.group(1) + defined = m.group(2) in defines + if_count += 1 + if hidden_count or not mode or (mode == 'def' and not defined) or \ + (mode == 'ndef' and defined): + hidden_count += 1 + continue + + if _ELSE_RE.match(line): + if hidden_count == 0: + hidden_count = 1 + elif hidden_count == 1: + hidden_count = 0 + continue + + if _ENDIF_RE.match(line): + if if_count == 0: + raise OvMergeError(f"* Unmatched #endif ({filename}:{linenum})") + if_count -= 1 + if hidden_count: + hidden_count -= 1 + continue + + if hidden_count: + continue + + m = _INCLUDE_RE.match(line) + if m: + incfile = m.group(1) + if re.search(r'\.h.$', incfile): + tokens.append('#include') + tokens.append(incfile) + elif re.search(r'\.dtsi?.$', incfile): + dtsfile = search_path(filepath + incfile[1:-1]) + if not dtsfile: + raise OvMergeError(f"* Failed to find include file '{incfile}'") + inc_tokens = read_tokens(dtsfile, depth + 1, defines) + tokens.extend(inc_tokens) + tokens.append(FileMarker(filename)) + if S.expand and not S.expand_label: + print(f"#### Continue '{filename}'") + else: + raise OvMergeError(f"* Invalid include file '{incfile}'") + continue + + m = _DEFINE_RE.match(line) + if m: + symbol = m.group(1) + val = m.group(2) if m.group(2) is not None else '' + val = re.sub(r'//.*', '', val) + val = re.sub(r'\s+$', '', val) + defines[symbol] = val + continue + + m = _UNDEF_RE.match(line) + if m: + defines.pop(m.group(1), None) + continue + + if _BKPT_RE.match(line): + continue + + if S.expand_label: + sys.stdout.write(f"{filename}:{linenum}: ") + if S.expand: + sys.stdout.write(line) + + pos = _WS_RE.match(line).end() + + if expr_level: + while expr_level: + mm = _PAREN_RE.match(line, pos) + if not mm: + break + pos = mm.end() + expr_level += 1 if mm.group(2) == '(' else -1 + expr += mm.group(1) + mm.group(2) + if expr_level: + expr += mm.group(3) + if expr_level: + mm = _TAIL_RE.match(line, pos) + if mm: + expr += mm.group(1) + pos = mm.end() + continue + tokens.append(expr) + + while True: + m = _TOKEN_RE.match(line, pos) + if not m: + break + tok = m.group(1) + pos = m.end() + wsm = _WS_RE.match(line, pos) + pos = wsm.end() + + if tok == '//': + pos = len(line) + break + elif tok == '/*': + mm = _BLOCK_COMMENT_RE.match(line, pos) + if mm: + pos = mm.end() + continue + else: + in_comment = True + pos = len(line) + break + elif tok == '(': + expr_level = 1 + expr = '(' + while expr_level: + mm = _PAREN_RE.match(line, pos) + if not mm: + break + pos = mm.end() + expr_level += 1 if mm.group(2) == '(' else -1 + expr += mm.group(1) + mm.group(2) + if expr_level: + expr += mm.group(3) + if expr_level: + mm = _TAIL_RE.match(line, pos) + if mm: + expr += mm.group(1) + pos = mm.end() + break + tok = expr + + wm = _WORD_RE.search(tok) + if wm: + sym = wm.group(1) + newsym = defines.get(sym) + if newsym is not None: + if S.trace_parse: + print(f"['{sym}' -> '{newsym}']") + tok = re.sub(r'\b' + re.escape(sym) + r'\b', lambda _m: newsym, tok, count=1) + tokens.append(tok) + + rest = line[pos:] + if not re.fullmatch(r'[\r\n]*', rest): + raise OvMergeError(f"* Bad token at '{rest}'") + + if S.expand and not S.expand_label: + print(f"#### End of '{filename}'") + + return tokens + + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- + +def get_head(ps): + while True: + if ps.pos >= len(ps.tokens): + return None + head = ps.tokens[ps.pos] + if isinstance(head, FileMarker): + ps.file = head.filename + if S.trace_parse: + print(f"[file {head.filename}]") + ps.pos += 1 + continue + return head + + +def get_next(ps): + ps.pos += 1 + return get_head(ps) + + +def match(ps, tok): + head = get_head(ps) + if S.trace_parse: + print(f"[match '{tok}' @ {ps.pos}]") + if head != tok: + raise OvMergeError(f"* Unexpected token '{head}' - expected '{tok}'") + return get_next(ps) + + +def get_int(ps): + head = get_head(ps) + if head is None or not re.match(r'^[0-9]', head): + return None + get_next(ps) + return head + + +def parse_node(ps, parent, depth, node, *newlabels): + next_tok = match(ps, '{') + + if not isinstance(node, list): + child = get_child(parent, node) + node = child if child is not None else add_node(parent, node) + + if S.trace_parse: + print(f"parse_node({node[NAME]}, {depth} ...)") + + while next_tok != '}': + childlabels = [] + + if next_tok == '/delete-node/': + next_tok = match(ps, next_tok) + if next_tok is not None and re.fullmatch(r'[-a-zA-Z0-9,._+#@]+', next_tok): + delete_node(get_child(node, next_tok)) + match(ps, next_tok) + next_tok = match(ps, ';') + continue + elif next_tok == '/delete-property/': + next_tok = match(ps, next_tok) + if next_tok is not None and re.fullmatch(r'[-a-zA-Z0-9,._+#@]+', next_tok): + delete_prop(node, next_tok) + match(ps, next_tok) + next_tok = match(ps, ';') + continue + elif next_tok == '#include': + next_tok = match(ps, next_tok) + set_add(S.cur_dt.includes, next_tok) + next_tok = match(ps, next_tok) + continue + + while next_tok is not None and re.fullmatch(r'(\w+):', next_tok): + childlabels.append(next_tok[:-1]) + if S.trace_parse: + print(f"[Label: {next_tok[:-1]}]") + next_tok = match(ps, next_tok) + + if next_tok is not None and re.fullmatch(r'[-a-zA-Z0-9,._+#@]+', next_tok): + name = next_tok + if re.search(r'@0[0-9a-fA-F]', name): + print(f"* Leading zero in node name '{name}'") + next_tok = match(ps, next_tok) + if next_tok == '{': + next_tok = parse_node(ps, node, depth + 1, name, *childlabels) + elif next_tok == '=': + prop = [] + if childlabels and S.warnings: + print(f"* Ignoring label on property '{name}'") + while True: + next_tok = match(ps, next_tok) + m = re.fullmatch(r'"(.*)"', next_tok) if next_tok is not None else None + if m: + prop.append(['"', m.group(1)]) + next_tok = match(ps, next_tok) + elif next_tok is not None and re.fullmatch(r'&(.*)', next_tok): + prop.append(['&', next_tok[1:]]) + next_tok = match(ps, next_tok) + elif next_tok == '<' or next_tok == '/bits/': + elemsize = 4 + if next_tok == '/bits/': + next_tok = match(ps, next_tok) + if next_tok not in ('8', '16', '32', '64'): + raise OvMergeError(f"* Invalid /bits/ value '{next_tok}'.") + elemsize = int(next_tok) // 8 + match(ps, next_tok) + next_tok = match(ps, '<') + vals = [] + while next_tok != '>': + vals.append(next_tok) + next_tok = match(ps, next_tok) + prop.append(['<', elemsize, vals]) + next_tok = match(ps, '>') + else: + vals = [] + next_tok = match(ps, '[') + while next_tok != ']': + vals.extend(byte_array_value(next_tok)) + next_tok = match(ps, next_tok) + next_tok = match(ps, ']') + prop.append(['[', 1, vals]) + if next_tok != ',': + break + next_tok = match(ps, ';') + set_prop(node, name, *prop) + else: + if childlabels and S.warnings: + print(f"* Ignoring label on property '{name}'") + next_tok = match(ps, ';') + set_prop(node, name) + else: + raise OvMergeError(f"* Unexpected token '{next_tok}'") + + labels = S.cur_dt.labels + for newlabel in newlabels: + labelled_node = labels.get(newlabel) + if labelled_node is not None: + if labelled_node is not node: + print(f"* Duplicated label '{newlabel}' - '{labelled_node[NAME]}' and '{node[NAME]}'", + file=sys.stderr) + else: + if S.warnings: + print(f"* Replicated label '{newlabel}' (on the same node)") + add_label(S.cur_dt, node, newlabel) + + match(ps, '}') + return match(ps, ';') + + +def dtparse(filename, got_header): + defines = {} + tokens = read_tokens(filename, 0, defines) + ps = PState(tokens) + + dt = DT() + dt.defines = defines + + next_tok = get_head(ps) + + while next_tok is not None and re.fullmatch(r'/.+/|#include', next_tok): + typ = next_tok + next_tok = match(ps, next_tok) + if typ == '#include': + if S.trace_parse: + print(f"[#include {next_tok}]") + set_add(dt.includes, next_tok) + next_tok = match(ps, next_tok) + else: + if not got_header: + if typ != '/dts-v1/': + raise OvMergeError("* File missing /dts-v1/ tag") + got_header = True + elif typ == '/dts-v1/': + if S.warnings: + print("* Ignoring duplicate /dts-v1/ tag") + elif typ == '/plugin/': + dt.plugin = True + elif typ == '/memreserve/': + start = get_int(ps) + length = get_int(ps) + set_add(dt.memreserves, [start, length]) + else: + raise OvMergeError(f"* Unexpected token '{typ}'") + next_tok = match(ps, ';') + + S.cur_dt = dt + + while next_tok is not None: + if next_tok == '/': + match(ps, '/') + next_tok = parse_node(ps, None, 0, '/') + else: + newlabels = [] + while next_tok is not None and re.fullmatch(r'(\w+):', next_tok): + newlabels.append(next_tok[:-1]) + if S.trace_parse: + print(f"[Label: {next_tok[:-1]}]") + next_tok = match(ps, next_tok) + + m = re.fullmatch(r'&(\w+)', next_tok) if next_tok is not None else None + if m: + label = m.group(1) + subnode = dt.labels.get(label) + match(ps, next_tok) + if subnode is not None: + next_tok = parse_node(ps, subnode[PARENT], subnode[DEPTH], subnode, *newlabels) + else: + print(f"* Unknown label '{label}'", file=sys.stderr) + next_tok = parse_node(ps, None, 0, '/', *newlabels) + elif next_tok == '/delete-node/': + next_tok = match(ps, next_tok) + m2 = re.fullmatch(r'&(\w+)', next_tok) if next_tok is not None else None + if m2: + label = m2.group(1) + subnode = dt.labels.get(label) + match(ps, next_tok) + if subnode is not None: + delete_node(subnode) + else: + print(f"* Unknown label '{label}'", file=sys.stderr) + next_tok = match(ps, ';') + elif next_tok == '#include': + next_tok = match(ps, next_tok) + set_add(dt.includes, next_tok) + next_tok = match(ps, next_tok) + else: + raise OvMergeError(f"* Unexpected token '{next_tok}'") + + S.cur_dt = None + + if ps.pos != len(ps.tokens): + print(f"* Junk at the end - {get_head(ps)} ...") + + for key, value in list(dt.refcount.items()): + if value < 0: + raise OvMergeError(f"* Internal error - negative refcount on '{key}'") + elif value > 0 and key not in dt.labels and not dt.plugin: + print(f"* symbol '{key}' is undefined in '{filename}'", file=sys.stderr) + S.retcode = 1 + + if dt.plugin: + ordercheck(dt, {}) + + overrides = get_node(dt, '/__overrides__') + for param in (overrides[PROPS] if overrides is not None else []): + dtparam(dt, param[0], None) + + return dt