From 815d15ee8b41392db5271025cc59f4ae63ec3624 Mon Sep 17 00:00:00 2001 From: o-murphy Date: Mon, 27 Jul 2026 22:50:29 +0000 Subject: [PATCH] mip: Support optional per-entry native code compatibility tags. Related to #1140 / micropython#19478 / micropython#19479: those add URL-templating for single-file natmod installs, but don't help when a single manifest needs to serve different files per architecture. Adds an optional third element to hashes/urls entries: a raw sys.implementation._mpy-shaped integer. Entries without it install unconditionally (backward compatible); entries with it only install if _mpy_tag_ok() matches - exact match on version/sub-version/arch, subset match on arch-flags (a variant that doesn't require an extension must still install on hardware that supports it). Signed-off-by: o-murphy --- micropython/mip/mip/__init__.py | 33 +++++++++++++++++++++++++++++++-- micropython/mip/test_mip_tag.py | 31 +++++++++++++++++++++++++++++++ tools/build.py | 4 ++++ tools/ci.sh | 2 ++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 micropython/mip/test_mip_tag.py diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 22982fe9a..fec1c0e62 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -24,6 +24,29 @@ _ALLOWED_MIP_URL_PREFIXES = const(("http://", "https://", "codeberg:", "github:", "gitlab:")) +# Bits 0-15 of sys.implementation._mpy: version/sub-version/arch (exact +# match required). Bits 16+: optional arch-flags (e.g. RV32 extensions) - +# a tag's flags must be a subset of what this device supports, not equal. +_MPY_VERSION_MASK = const(0xFF) +_MPY_BASE_MASK = const(0xFFFF) + + +def _mpy_tag_ok(tag, device_mpy=None): + if device_mpy is None: + device_mpy = getattr(sys.implementation, "_mpy", None) + if device_mpy is None: + return False + # A bytecode-only tag (arch nibble 0) only needs the major version to + # match - mirrors mp_raw_code_load()'s own arch == MP_NATIVE_ARCH_NONE + # fast path in py/persistentcode.c, which skips the sub-version check + # entirely for non-native code (sub-version only encodes native-code ABI + # changes and is meaningless for portable bytecode). + if (tag >> 10) & 0xF == 0: + return (tag & _MPY_VERSION_MASK) == (device_mpy & _MPY_VERSION_MASK) + if (tag & _MPY_BASE_MASK) != (device_mpy & _MPY_BASE_MASK): + return False + return (tag >> 16) & (device_mpy >> 16) == (tag >> 16) + # This implements os.makedirs(os.dirname(path)) def _ensure_path_exists(path): @@ -112,7 +135,10 @@ def _install_json(package_json_url, index, target, version, mpy): package_json = response.json() finally: response.close() - for target_path, short_hash in package_json.get("hashes", ()): + for entry in package_json.get("hashes", ()): + target_path, short_hash = entry[0], entry[1] + if len(entry) > 2 and not _mpy_tag_ok(entry[2]): + continue fs_target_path = target + "/" + target_path if _check_exists(fs_target_path, short_hash): print("Exists:", fs_target_path) @@ -122,7 +148,10 @@ def _install_json(package_json_url, index, target, version, mpy): print("File not found: {} {}".format(target_path, short_hash)) return False base_url = package_json_url.rpartition("/")[0] - for target_path, url in package_json.get("urls", ()): + for entry in package_json.get("urls", ()): + target_path, url = entry[0], entry[1] + if len(entry) > 2 and not _mpy_tag_ok(entry[2]): + continue fs_target_path = target + "/" + target_path is_full_url = any(url.startswith(p) for p in _ALLOWED_MIP_URL_PREFIXES) if base_url and not is_full_url: diff --git a/micropython/mip/test_mip_tag.py b/micropython/mip/test_mip_tag.py new file mode 100644 index 000000000..24d4c61fe --- /dev/null +++ b/micropython/mip/test_mip_tag.py @@ -0,0 +1,31 @@ +from mip import _mpy_tag_ok + +# Exact match on version/sub-version/arch (bits 0-15), no arch-flags. +assert _mpy_tag_ok(0x0A06, 0x0A06) is True + +# Different version/sub-version/arch -> reject. +assert _mpy_tag_ok(0x0A06, 0x0B06) is False +assert _mpy_tag_ok(0x0A06, 0x0A07) is False + +# Arch-flags (bits 16+): tag's required flags must be a subset of the +# device's, not an exact match. +assert _mpy_tag_ok(0x0A06 | (0b001 << 16), 0x0A06 | (0b011 << 16)) is True +assert _mpy_tag_ok(0x0A06 | (0b011 << 16), 0x0A06 | (0b001 << 16)) is False +assert _mpy_tag_ok(0x0A06, 0x0A06 | (0b111 << 16)) is True # tag needs nothing +assert _mpy_tag_ok(0x0A06 | (0b111 << 16), 0x0A06) is False # device supports nothing + +# No _mpy support on the device (e.g. bytecode-only build). +assert _mpy_tag_ok(0x0A06, None) is False + +# Bytecode-only tag (arch nibble 0): only the major version must match -- +# sub-version and arch are ignored, mirroring mp_raw_code_load()'s own +# fast path for non-native code in py/persistentcode.c. +_BYTECODE_TAG = 0x0206 # version=6, sub-version=2, arch=NONE(0) +assert _mpy_tag_ok(_BYTECODE_TAG, 0x0A06) is True # same major, different sub/arch on device +assert _mpy_tag_ok(_BYTECODE_TAG, 0x0107) is False # different major version + +# Omitting device_mpy falls back to sys.implementation._mpy; just check it +# runs without raising, since that value depends on the build running the test. +_mpy_tag_ok(0x0A06) + +print("PASS") diff --git a/tools/build.py b/tools/build.py index 442cf2121..fdd82bcb5 100755 --- a/tools/build.py +++ b/tools/build.py @@ -99,6 +99,10 @@ # "v": 1, <-- file format version # "hashes": [ # ["aioble/server.mpy", "e39dbf64"], +# ["aioble/server.mpy", "e39dbf64", 2822], # optional 3rd element: only +# # installed if it matches the +# # device's sys.implementation._mpy +# # (not emitted by this script) # ... # ], # "urls": [ <-- not used by micropython-lib packages diff --git a/tools/ci.sh b/tools/ci.sh index 4c56fb9c5..dcf7ca91a 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -50,6 +50,7 @@ function ci_package_tests_setup_lib { $CP -r python-stdlib/unittest/unittest "${VIRTUAL_ENV}/lib/" $CP -r python-stdlib/unittest-discover/unittest "${VIRTUAL_ENV}/lib/" $CP unix-ffi/ffilib/ffilib.py "${VIRTUAL_ENV}/lib/" + $CP -r python-ecosys/requests/requests "${VIRTUAL_ENV}/lib/" tree "${VIRTUAL_ENV}" } @@ -57,6 +58,7 @@ function ci_package_tests_run { export MICROPYPATH for test in \ micropython/drivers/storage/sdcard/sdtest.py \ + micropython/mip/test_mip_tag.py \ micropython/umqtt.simple/test_umqtt_simple.py \ micropython/xmltok/test_xmltok.py \ python-ecosys/requests/test_requests.py \