From a27da962481fe04c0a5712c5539ca3aca5662459 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 09:38:12 -0500 Subject: [PATCH 01/53] rooter: opt-in next-hop egress primitive (fail-closed, per-task, vendor-neutral) Adds an opt-in [nexthop]/[gwX] routing mode that binds each analysis task's guest source IP to a chosen egress gateway via a dedicated policy-routing table, MASQUERADing onto that gateway's interface. Disabled by default: with [nexthop] enabled=off the existing routing paths are byte-for-byte unchanged. - conf/default/routing.conf.default: [nexthop] + [gwX] profile sections; the "nexthop" sentinel route is the pool default (default_policy). - lib/cuckoo/core/rooter.py: gateways registry + _select_gateway (explicit id / roundrobin / random; liveness-gated; fail-closed on None). - lib/cuckoo/core/startup.py: load_nexthop_profiles + validate_default_route (section-header id; required-option validation; each [gwX] rt_table must be a dedicated unique table -- rejects reserved/fail/duplicate/VPN/dirty-line; accepts gateway id / roundrobin / random / nexthop as a VPN-less default route; intra-subnet exception installed whenever nexthop enabled, blackhole only when fail_closed). - lib/cuckoo/core/analysis_manager.py: route_network resolves tunX before nexthop and lets nexthop consume the route (bind or fail-closed drop); reserved routes never resolve to a gateway; unroute mirrors the persisted tuple + guest ingress. - utils/rooter.py: nexthop_init/enable/disable/intra_exception/fail_closed/teardown (reserved-table guard; per-task forward ACCEPT via CAPE_ACCEPTED_SEGMENTS, constrained to the guest ingress iface (anti-spoof) and idempotent delete-until-gone; intra-subnet exception just below the per-task band; teardown sweeps only vm_net-sourced band rules by full selector; SIGTERM no-op when disabled). - web/submission: the route selector lists the "nexthop" pool + each configured [gwX] gateway when nexthop is enabled (defensive; no-op when disabled). - tests: selection, route/unroute, rooter primitives, netns integration. Fail-closed throughout: any unresolved/typo'd/reserved route drops rather than leaking onto the host default route. --- conf/default/routing.conf.default | 30 ++ lib/cuckoo/core/analysis_manager.py | 112 +++++- lib/cuckoo/core/rooter.py | 37 ++ lib/cuckoo/core/startup.py | 187 +++++++++- tests/integration/__init__.py | 0 tests/integration/test_nexthop_netns.py | 313 ++++++++++++++++ tests/test_nexthop_selection.py | 463 ++++++++++++++++++++++++ tests/test_rooter_nexthop.py | 233 ++++++++++++ tests/test_route_network_nexthop.py | 227 ++++++++++++ utils/rooter.py | 185 ++++++++++ web/submission/views.py | 20 + web/templates/submission/index.html | 8 + 12 files changed, 1801 insertions(+), 14 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_nexthop_netns.py create mode 100644 tests/test_nexthop_selection.py create mode 100644 tests/test_rooter_nexthop.py create mode 100644 tests/test_route_network_nexthop.py diff --git a/conf/default/routing.conf.default b/conf/default/routing.conf.default index fd4e281efad..e4d9b8adea3 100644 --- a/conf/default/routing.conf.default +++ b/conf/default/routing.conf.default @@ -132,6 +132,36 @@ interface = tun0 rt_table = tun0 +[nexthop] +# Decoupled next-hop egress to a shared gateway pool. Default OFF (no-regress). +enabled = no +# Comma list of [gwX] profile ids to load. +gateways = gw1 +# Selection used when a task resolves to the pool: roundrobin | random | . Set +# [routing] route = nexthop (the pool sentinel) as the node default so tasks that DON'T name a route +# use this. A task may also name a route directly -- a , roundrobin, random, or nexthop -- +# from the web submission UI route selector (which lists the nexthop pool + each configured [gwX]) +# or via API/CLI/broker submissions. (Advertising nexthop routes across a distributed multi-node +# CAPE via the exitnodes API is a separate follow-up.) +default_policy = roundrobin +# Drop guest egress when no live gateway (strongly recommended yes). +fail_closed = yes +# Whole guest subnet the blackhole covers (CIDR). MUST match the libvirt guest net. +vm_net = 192.168.100.0/24 + +[gw1] +# Interface that reaches the gateway network (provided by the deployment). +interface = ens6 +# Next-hop gateway IP reachable via `interface`, or 'onlink'. +next_hop = onlink +# Dedicated policy routing table id for this gateway. MUST be unique across the WHOLE system: +# not a reserved table (main/local/default/253-255/0), not another [gwX]'s table, not a VPN's +# rt_table, and not the [routing] dirty-line rt_table -- startup rejects those it can see. Note a +# name<->id alias counts as the same table (e.g. do NOT use 201 here if a VPN uses a name mapped to +# 201 in /etc/iproute2/rt_tables); keeping gateway tables to their own numeric range avoids this. +rt_table = 201 +description = vpn-router-1 + [socks5] # By default we disable socks5 support as it requires running utils/rooter.py as # root next to cuckoo.py (which should run as regular user). diff --git a/lib/cuckoo/core/analysis_manager.py b/lib/cuckoo/core/analysis_manager.py index 0dc4fd428f9..d56327d6ae1 100644 --- a/lib/cuckoo/core/analysis_manager.py +++ b/lib/cuckoo/core/analysis_manager.py @@ -29,7 +29,7 @@ from lib.cuckoo.core.machinery_manager import MachineryManager from lib.cuckoo.core.plugins import RunAuxiliary from lib.cuckoo.core.resultserver import ResultServer -from lib.cuckoo.core.rooter import _load_socks5_operational, rooter, vpns +from lib.cuckoo.core.rooter import _load_socks5_operational, _select_gateway, gateways, rooter, vpns log = logging.getLogger(__name__) @@ -53,6 +53,11 @@ def is_network_interface(intf: str): return intf in network_interfaces +def _nexthop_enabled(routing): + """True when the [nexthop] section exists and is enabled (review M4 hasattr guard).""" + return hasattr(routing, "nexthop") and routing.nexthop.enabled + + class CuckooDeadMachine(Exception): """Exception thrown when a machine turns dead. @@ -134,6 +139,11 @@ def __init__( self.reject_segments = None self.reject_hostports = None self.no_local_routing = None + # per-task [nexthop] binding (None unless this task is bound to a gateway profile) + self.nexthop_id = None + self.nexthop_interface = None + self.nexthop_rt_table = None + self.nexthop_priority = None @main_thread_only def prepare_task_and_machine_to_start(self) -> None: @@ -536,6 +546,85 @@ def _rooter_response_check(self): if self.rooter_response and self.rooter_response["exception"] is not None: raise CuckooCriticalError(f"Error execution rooter command: {self.rooter_response['exception']}") + def _resolve_nexthop(self, routing): + """Resolve self.route to a LIVE gateway profile and bind this task to it. + + Returns True if it OWNS the route (bound a profile); False if the id is + unresolved/typo'd or the pool is empty/all-down, in which case it forces + self.route="drop" so the existing none/drop/false dispatch drops the VM + (review B1: never fall through to host default forwarding). + + Always sets self.interface = None and self.rt_table = None so the generic + forward/srcroute block is skipped (review H2 option b) regardless of outcome. + """ + self.interface = None # nexthop owns enable/disable; skip the generic block + self.rt_table = None + # Resolve the selector, FAILING CLOSED on a typo'd/unknown route (review B1 + gemini #14): + # - an explicit gateway id, or a policy token (roundrobin/random), selects directly; + # - ONLY the node's configured DEFAULT route falls back to default_policy; + # - anything else (a typo'd/unconfigured id like "gw9"/"vpn9") DROPS rather than + # silently egressing via some live gateway (that would be an isolation-boundary bypass). + # A reserved route (none/drop/false/internet/tor/inetsim) must NEVER resolve to a gateway + # even if it is the configured default — those are owned by earlier route_network branches; + # reaching here with one means drop, not gateway (gemini #15 security-critical). + if self.route in ("none", "drop", "false", "internet", "tor", "inetsim"): + self.route = "drop" + return False + if self.route in gateways or self.route in ("roundrobin", "random"): + sel = self.route + elif self.route == "nexthop" or self.route == routing.routing.route: + # the "nexthop" sentinel (explicit) OR the node's configured default route falls back + # to default_policy and picks from the live pool. "nexthop" is a reserved token (a + # [gwX] may not be named it), so this is not the gemini-#15 typo'd-route leak. + sel = routing.nexthop.default_policy + else: + self.route = "drop" + return False + profile = _select_gateway(sel) + if profile is None: + # empty/all-down pool => FAIL CLOSED (review B1) + self.route = "drop" + return False + self.nexthop_id = profile.name + self.nexthop_interface = str(profile.interface) + self.nexthop_rt_table = str(profile.rt_table) + # deterministic per-task priority: 10000 + last octet of the VM IP (review M6). + # Unique within one guest /24 (the shipped vm_net); across multiple /24s two VMs + # could share a priority number but keep distinct `from ` selectors, so no + # leak — teardown's by-priority band sweep still clears them all. + self.nexthop_priority = str(10000 + int(self.machine.ip.rsplit(".", 1)[1])) + return True + + def _dispatch_nexthop(self): + """Issue the per-task nexthop bind (source rule + SNAT). No-op unless this + task is bound to a gateway profile. Every rooter arg is str() (review B3).""" + if not self.nexthop_id: + return + self.rooter_response = rooter( + "nexthop_enable", + str(self.machine.ip), + str(self.machine.interface), # guest ingress bridge -- constrains forwarding (anti-spoof) + self.nexthop_interface, + self.nexthop_rt_table, + self.nexthop_priority, + ) + self._rooter_response_check() + + def _unroute_nexthop(self): + """Mirror-teardown the per-task bind with the PERSISTED self.nexthop_* tuple; + never re-run the selector (review M5). No-op unless bound to a gateway.""" + if not self.nexthop_id: + return + self.rooter_response = rooter( + "nexthop_disable", + str(self.machine.ip), + str(self.machine.interface), # mirror the ingress bridge enable used (persisted machine) + self.nexthop_interface, + self.nexthop_rt_table, + self.nexthop_priority, + ) + self._rooter_response_check() + def route_network(self): """Enable network routing if desired.""" # Determine the desired routing strategy (none, internet, VPN). @@ -566,8 +655,19 @@ def route_network(self): elif self.route in self.socks5s: self.interface = "" elif self.route[:3] == "tun" and is_network_interface(self.route): - # tunnel interface starts with "tun" and interface exists on machine + # tunnel interface starts with "tun" and interface exists on machine. Checked BEFORE + # the nexthop branch: _resolve_nexthop rewrites self.route="drop" for any route it does + # not own, which would otherwise clobber an explicit tunX route into a drop when + # [nexthop] is enabled (Copilot). Legacy explicit routes win over nexthop resolution. self.interface = self.route + elif _nexthop_enabled(routing): + # When [nexthop] is enabled it OWNS every route not claimed by a legacy branch above: + # it binds a gateway profile (setting self.nexthop_* and self.interface=None so the + # generic forward/srcroute block is skipped) or, for an unresolved/typo'd id, forces + # self.route="drop". Consuming the route here (rather than an `and`-guarded `pass`) + # means a nexthop-dropped task falls into the none/drop/false dispatch below and fails + # closed cleanly, instead of the misleading "Unknown ... ignoring routing" else (Copilot). + self._resolve_nexthop(routing) else: self.log.warning("Unknown network routing destination specified, ignoring routing for this analysis: %s", self.route) self.interface = None @@ -622,6 +722,10 @@ def route_network(self): self._rooter_response_check() + # nexthop bind (self.interface is None for a gateway route, so the generic + # block below is skipped; this issues the per-task source rule + SNAT). + self._dispatch_nexthop() + # check if the interface is up if HAVE_NETWORKIFACES and routing.routing.verify_interface and self.interface and self.interface not in network_interfaces: self.log.info("Network interface %s not found, falling back to dropping network traffic", self.interface) @@ -757,6 +861,10 @@ def unroute_network(self): self.log.info("Disable tunnel interface: %s", self.interface) self.rooter_response = rooter("interface_route_tun_disable", self.machine.ip, self.route, str(self.task.id)) + # nexthop teardown with the PERSISTED tuple (the generic disable block above + # was skipped because self.interface is None). No-op unless bound (review M5). + self._unroute_nexthop() + self._rooter_response_check() def get_machine_specific_options(self, task_opts: str) -> str: diff --git a/lib/cuckoo/core/rooter.py b/lib/cuckoo/core/rooter.py index 0afbf2d7ca7..4c3e93b3e70 100644 --- a/lib/cuckoo/core/rooter.py +++ b/lib/cuckoo/core/rooter.py @@ -4,6 +4,8 @@ # See the file 'docs/LICENSE' for copying permission. import logging +import random +import threading from contextlib import suppress from lib.cuckoo.common.config import Config @@ -15,6 +17,9 @@ vpns = {} socks5s = {} +gateways = {} # profile-id -> profile object (.name/.interface/.rt_table/.priority) +_gw_cursor = 0 # process-global round-robin cursor +_gw_lock = threading.Lock() def _load_socks5_operational(): @@ -59,3 +64,35 @@ def rooter(command, *args, **kwargs): if ret and ret.get("exception"): log.warning("Rooter returned error: %s", ret["exception"]) return ret + + +def _gw_live(profile): + """True if the profile's egress interface exists and is up. Delegates to the rooter + nic_available check; overridable in tests.""" + resp = rooter("nic_available", str(profile.interface)) + return bool(resp and resp.get("output")) + + +def _select_gateway(route): + """Resolve a task route to a LIVE gateway profile, or None (=> caller fails closed). + route: an explicit profile id, or a default_policy token 'roundrobin'/'random'.""" + global _gw_cursor + if not gateways: + return None + if route in gateways: + p = gateways[route] + return p if _gw_live(p) else None + live = [gateways[k] for k in gateways if _gw_live(gateways[k])] + if not live: + return None + if route == "random": + return random.choice(live) + if route != "roundrobin": + # not a gateway id, not 'random', not 'roundrobin' => unknown selector, FAIL CLOSED + # rather than silently treating it as roundrobin (gemini #15). + return None + # roundrobin: advance the process-global cursor under the lock + with _gw_lock: + p = live[_gw_cursor % len(live)] + _gw_cursor += 1 + return p diff --git a/lib/cuckoo/core/startup.py b/lib/cuckoo/core/startup.py index 8ebf21b3a16..bb1ffcf9bad 100644 --- a/lib/cuckoo/core/startup.py +++ b/lib/cuckoo/core/startup.py @@ -56,7 +56,7 @@ from lib.cuckoo.core.data.task import TASK_FAILED_ANALYSIS, TASK_RUNNING from lib.cuckoo.core.log import init_logger from lib.cuckoo.core.plugins import import_package, import_plugin, list_plugins -from lib.cuckoo.core.rooter import rooter, socks5s, vpns +from lib.cuckoo.core.rooter import gateways, rooter, socks5s, vpns log = logging.getLogger() @@ -67,6 +67,172 @@ auxconf = Config("auxiliary") dist_conf = Config("distributed") +# --------------------------------------------------------------------------- +# Next-hop egress primitive — constants used by loader + SIGTERM path +# --------------------------------------------------------------------------- +NEXTHOP_FAIL_TABLE = "250" +NEXTHOP_PRIORITY_LOW = "30000" +NEXTHOP_BAND_LO = "10000" +NEXTHOP_BAND_HI = "10255" +# Route keywords a [gwX] gateway id must never collide with. "nexthop" is the sentinel default +# route that maps to [nexthop] default_policy (pool selection), so a gateway named "nexthop" +# would be ambiguous — reserve it too (Copilot). +_RESERVED_ROUTE_NAMES = {"none", "internet", "tor", "inetsim", "drop", "false", "nexthop"} +# Pool-policy selector tokens: a task route of roundrobin/random means "pick from the live +# pool", so a [gwX] must not be *named* one of these (it could never be explicitly selected +# and would collide with the policy token in _resolve_nexthop/_select_gateway). +_POLICY_TOKENS = ("roundrobin", "random") +# Linux reserved/system routing tables. A [gwX] rt_table must never be one of these: it is fed +# straight into `ip route flush table ` and, if it were `main`, the per-task rule would +# route the VM out the host's own default route (fail-OPEN) instead of the blackhole. Mirrors the +# defence-in-depth guard in utils.rooter nexthop_init/teardown. Kernel-fixed ids. +_RESERVED_RT_TABLES = ("local", "main", "default", "0", "253", "254", "255") + + +def load_nexthop_profiles(routing_cfg): + """Parse [nexthop]/[gwX] sections into the rooter.gateways global, sweep stale + policy-routing state, then arm fail-closed. No-op when [nexthop] is absent + or disabled (review M4 hasattr guard).""" + if not hasattr(routing_cfg, "nexthop") or not routing_cfg.nexthop.enabled: + return + # [nexthop] is enabled: it MUST define gateways + vm_net (gemini #14 MEDIUM). CAPE's config + # Dictionary returns None (not AttributeError) for a missing key, so without this a missing + # option slips through as None and misbehaves downstream — fail with a clear error instead. + for opt in ("gateways", "vm_net"): + if getattr(routing_cfg.nexthop, opt, None) is None: + raise CuckooStartupError(f"[nexthop] is enabled but missing the required '{opt}' option in routing.conf") + # Routing tables already OWNED by another primitive -- a [gwX] must not reuse one. nexthop_init + # flush/replaces each gateway table, so a shared table silently clobbers (or is clobbered by) the + # other primitive's route, misrouting its traffic with no error. Two sources, both knowable now: + # - VPN tables: init_routing builds vpns[*].rt_table BEFORE calling us, so nexthop_init would + # wipe a just-built VPN table and point its default out the gateway NIC (VPN-isolation break). + # - the [routing] dirty-line table (routing.routing.rt_table): its init_rttable runs AFTER us, + # so it would repopulate a gateway table with the dirty-line interface. + # Coerce to str: a VPN rt_table (or the dirty-line one) may be an int or a name in config. + owned_tables = {} + for vname, ventry in vpns.items(): + vrt = getattr(ventry, "rt_table", None) + if vrt is not None: + owned_tables[str(vrt)] = f"VPN '{vname}'" + dirty_rt = getattr(getattr(routing_cfg, "routing", None), "rt_table", None) + if dirty_rt is not None and str(dirty_rt): + owned_tables.setdefault(str(dirty_rt), "the [routing] dirty-line table") + # Pass 1: parse + validate + register profiles (no rooter side effects yet). + profiles = [] + claimed_tables = {} # rt_table -> gateway id, to reject duplicates (each [gwX] needs its own) + for name in routing_cfg.nexthop.gateways.split(","): + name = name.strip() + if not name: + continue + if name in _RESERVED_ROUTE_NAMES or name in _POLICY_TOKENS or name in vpns or name in socks5s or name[:3] == "tun": + raise CuckooStartupError(f"nexthop gateway id '{name}' collides with a reserved/route/policy name") + if not hasattr(routing_cfg, name): + raise CuckooStartupError(f"nexthop gateway '{name}' has no [{name}] section in routing.conf") + entry = routing_cfg.get(name) + # A [gwX] section MUST define interface/next_hop/rt_table (gemini #14 MEDIUM). Config + # Dictionary returns None for a missing key, so validate explicitly — otherwise None + # (or str(None)=="None") would slip into the rooter commands below. + for opt in ("interface", "next_hop", "rt_table"): + if getattr(entry, opt, None) is None: + raise CuckooStartupError(f"nexthop gateway '{name}' is missing the required '{opt}' option in [{name}]") + entry.rt_table = str(entry.rt_table) # coerce: config may produce int (review B3) + # A reserved rt_table (main/local/default/...) must be rejected HERE, at load, not just + # skipped in nexthop_init: an unbuilt custom table is empty so its per-task rule falls + # through to the fail-closed blackhole, but `main` already holds the host default route, + # so a per-task `from vm_ip lookup main priority 100xx` (below the 30000 blackhole) would + # route the VM straight out the control-plane NIC -- an isolation bypass. Fail loudly. + if entry.rt_table in _RESERVED_RT_TABLES: + raise CuckooStartupError( + f"nexthop gateway '{name}' uses reserved routing table '{entry.rt_table}' in [{name}]; " + "pick a dedicated custom table id (e.g. 201)" + ) + # rt_table must not be the fail-closed table: nexthop_init builds the gateway default there, + # then nexthop_fail_closed_enable does `ip route replace blackhole default table `, + # overwriting that default with a blackhole -> every task bound to the gateway silently drops. + if entry.rt_table == NEXTHOP_FAIL_TABLE: + raise CuckooStartupError( + f"nexthop gateway '{name}' uses rt_table '{entry.rt_table}' in [{name}], which is the " + "reserved fail-closed table; the blackhole would overwrite the gateway's default route. " + "Pick a different custom table id." + ) + # rt_table must not be owned by a VPN or the [routing] dirty-line (see owned_tables above): + # nexthop_init would clobber that primitive's table and silently misroute its traffic. This + # catches DIRECT string collisions; a name<->id alias (a VPN using the name "tun0" mapped to + # id 201 in /etc/iproute2/rt_tables while a [gwX] uses 201) resolves to the same kernel table + # but different config strings -- that stays the operator's "unique across the system" + # responsibility, as documented for VPN rt_table in routing.conf. + if entry.rt_table in owned_tables: + raise CuckooStartupError( + f"nexthop gateway '{name}' rt_table '{entry.rt_table}' in [{name}] collides with " + f"{owned_tables[entry.rt_table]}; each routing primitive needs a unique table id" + ) + # rt_table must be UNIQUE across [gwX]: nexthop_init flush/replaces a single table per profile, + # so a shared table leaves the last gateway's default winning while the earlier gateway stays + # selectable -> its per-task rule looks up a table pointing at the wrong egress interface. + if entry.rt_table in claimed_tables: + raise CuckooStartupError( + f"nexthop gateways '{claimed_tables[entry.rt_table]}' and '{name}' both use rt_table " + f"'{entry.rt_table}'; each [gwX] needs a unique routing table id" + ) + claimed_tables[entry.rt_table] = name + # [gwX] sections carry no `name =` field (unlike [vpnX]/[socks5]), so config + # Dictionary.__getattr__ returns None for entry.name. Carry the section header as + # the profile id: analysis_manager._resolve_nexthop reads profile.name into + # self.nexthop_id, and a None id makes _dispatch_nexthop silently no-op (the + # per-task rule never installs and every task fails closed). Verified live. + entry.name = name + gateways[name] = entry + profiles.append(entry) + # [nexthop] enabled but nothing usable parsed (e.g. gateways = "" or all-blank): without + # this every analysis task would silently fall through to the fail-closed blackhole. Fail + # loudly at startup instead (gemini medium). + if not profiles: + raise CuckooStartupError("[nexthop] is enabled but no gateways were configured in routing.conf") + vm_net = str(routing_cfg.nexthop.vm_net) + tables_csv = ",".join(p.rt_table for p in profiles) + # Record sweep state for SIGTERM, then sweep any STALE state from a prior run + # BEFORE building fresh tables. nexthop_teardown flushes the gateway tables, so it + # MUST run before nexthop_init (which builds them) or it wipes the just-built routes. + rooter("nexthop_configure", tables_csv, vm_net, NEXTHOP_FAIL_TABLE, + NEXTHOP_PRIORITY_LOW, NEXTHOP_BAND_LO, NEXTHOP_BAND_HI) + rooter("nexthop_teardown", tables_csv, vm_net, NEXTHOP_FAIL_TABLE, + NEXTHOP_PRIORITY_LOW, NEXTHOP_BAND_LO, NEXTHOP_BAND_HI) + # Pass 2: build fresh profile tables, then arm the intra-subnet exception + (optionally) fail-closed. + for entry in profiles: + rooter("nexthop_init", str(entry.rt_table), str(entry.interface), str(entry.next_hop)) + # The intra-subnet exception (keep guest<->host + guest<->guest vm_net traffic on main) is a + # CONNECTIVITY guarantee, independent of the blackhole -- install it whenever nexthop is enabled. + # Otherwise, with fail_closed=no, a bound VM's per-task rule would send its intra-vm_net traffic + # (siblings / non-host-local services) out the gateway instead of the guest network (codex P2). + rooter("nexthop_intra_exception_enable", vm_net, NEXTHOP_BAND_LO) + if routing_cfg.nexthop.fail_closed: + rooter("nexthop_fail_closed_enable", vm_net, NEXTHOP_FAIL_TABLE, NEXTHOP_PRIORITY_LOW) + + +def validate_default_route(routing_cfg): + """Check that the configured default route is valid. Accepts gateway ids + (from gateways global) when nexthop is enabled, bypassing the vpn.enabled gate. + Extracted from init_routing so it is independently unit-testable (review H3).""" + route = routing_cfg.routing.route + if route in ("none", "internet", "tor", "inetsim"): + return + nexthop_on = hasattr(routing_cfg, "nexthop") and routing_cfg.nexthop.enabled + if nexthop_on and (route in gateways or route in _POLICY_TOKENS or route == "nexthop"): + # A concrete gateway id, a pool-policy token (roundrobin/random), or the "nexthop" sentinel + # is a valid default route when nexthop is on -- _resolve_nexthop maps the token/sentinel to + # default_policy and picks from the live pool. Accept it here so the documented pool default + # (route = nexthop) works without a VPN; otherwise startup wrongly raises the vpn-not-enabled + # error and the default_policy fallback is unreachable in production (codex P2 / Copilot). + return # skip the vpn.enabled gate + if not routing_cfg.vpn.enabled: + raise CuckooStartupError( + "A VPN has been configured as default routing interface for VMs, but VPNs have not been enabled in routing.conf" + ) + if route not in vpns and route not in socks5s: + raise CuckooStartupError( + "The VPN/Socks5 defined as default routing target has not been configured in routing.conf. You should use name field" + ) + def check_python_version(): """Checks if Python version is supported by Cuckoo. @@ -449,12 +615,15 @@ def init_rooter(): connect to it.""" # The default configuration doesn't require the rooter to be ran. + # A nexthop-only node still needs the rooter for forward_drop() + fail-closed arm. + _nexthop_enabled = hasattr(routing, "nexthop") and routing.nexthop.enabled if ( not routing.vpn.enabled and not routing.tor.enabled and not routing.inetsim.enabled and not routing.socks5.enabled and routing.routing.route == "none" + and not _nexthop_enabled ): return @@ -573,21 +742,15 @@ def init_routing(): rooter("flush_rttable", entry.rt_table) rooter("init_rttable", entry.rt_table, entry.interface) + # Load [gwX] next-hop egress profiles, arm fail-closed (no-op when [nexthop] absent/disabled). + load_nexthop_profiles(routing) + # If we are storage and webgui only but using as default route one of the workers exitnodes if dist_conf.distributed.master_storage_only: return - # Check whether the default VPN exists if specified. - if routing.routing.route not in ("none", "internet", "tor", "inetsim"): - if not routing.vpn.enabled: - raise CuckooStartupError( - "A VPN has been configured as default routing interface for VMs, but VPNs have not been enabled in routing.conf" - ) - - if routing.routing.route not in vpns and routing.routing.route not in socks5s: - raise CuckooStartupError( - "The VPN/Socks5 defined as default routing target has not been configured in routing.conf. You should use name field" - ) + # Check whether the default VPN/gateway exists if specified. + validate_default_route(routing) # Check whether the dirty line exists if it has been defined. if routing.routing.internet != "none": diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_nexthop_netns.py b/tests/integration/test_nexthop_netns.py new file mode 100644 index 00000000000..8f5a24fa6e5 --- /dev/null +++ b/tests/integration/test_nexthop_netns.py @@ -0,0 +1,313 @@ +# tests/integration/test_nexthop_netns.py +# +# Root-gated netns integration test for the nexthop egress primitive. +# Requires: root + CAPE_NETNS_TESTS=1 env var. +# Run via: SUDO=1 bash run-cape-tests-on-box.sh tests/integration/test_nexthop_netns.py -v +# +# What this tests (semantic, not just argv): +# 1. nexthop_init installs a forced-default in the profile's routing table +# 2. nexthop_enable installs a policy rule pointing vm_ip -> that table +# 3. Two concurrent profiles resolve to DISTINCT tables/interfaces +# 4. nexthop_fail_closed_enable installs a blackhole in the fail table and +# a low-priority subnet rule so an unbound guest source hits the blackhole +# +# Assertion strategy: +# - "ip rule show" to verify policy rules exist at the right priority +# - "ip route show table " to verify the forced-default points to the right egress_if +# - The combination proves a packet from vm_ip would be source-routed through the correct table. +# - For fail-closed: verify the blackhole route is in table 250 and the subnet rule is present. +# - We do NOT use "ip route get ... from " because that command can refuse +# to execute (EHOSTUNREACH) when the source is not locally assigned, even under root. +# The rule+table combination is the authoritative semantic check. + +import os +import subprocess +import pytest +import threading +import utils.rooter as rooter + +pytestmark = pytest.mark.skipif( + os.geteuid() != 0 or os.environ.get("CAPE_NETNS_TESTS") != "1", + reason="needs root + CAPE_NETNS_TESTS=1 (creates network namespaces)", +) + +# ─── ip path ───────────────────────────────────────────────────────────────── +# /sbin/ip and /usr/sbin/ip both exist on this Ubuntu box (usrmerge symlinks); +# using the PATH-resolved "ip" is most portable. Override in the fixture so the +# rooter functions use the same binary as our assertions. +_IP = "/sbin/ip" + + +def sh(*args, check=True): + """Run an ip/iptables command, raising on failure unless check=False.""" + result = subprocess.run(list(args), capture_output=True, text=True, check=False) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, args, result.stdout, result.stderr + ) + return result.stdout, result.stderr + + +def ip_rule_exists(from_addr, table, priority): + """Return True if 'ip rule show' contains a rule matching from/table/priority.""" + out, _ = sh(_IP, "rule", "show") + for line in out.splitlines(): + # Line format: "10041: from 192.168.100.41 lookup 201" + head = line.split(":", 1)[0].strip() + if head == str(priority) and f"from {from_addr}" in line and f"lookup {table}" in line: + return True + return False + + +def ip_route_table_has_default_via_if(table, iface): + """Return True if the routing table has a default route dev iface.""" + out, _ = sh(_IP, "route", "show", "table", table) + for line in out.splitlines(): + if "default" in line and iface in line: + return True + return False + + +def ip_route_table_has_blackhole(table): + """Return True if the routing table has a blackhole default.""" + out, _ = sh(_IP, "route", "show", "table", table) + for line in out.splitlines(): + if "blackhole" in line and "default" in line: + return True + return False + + +def subnet_rule_exists(subnet, table, priority): + """Return True if a rule for the whole subnet exists in ip rule show.""" + out, _ = sh(_IP, "rule", "show") + for line in out.splitlines(): + head = line.split(":", 1)[0].strip() + if head == str(priority) and subnet in line and f"lookup {table}" in line: + return True + return False + + +# ─── fixture ───────────────────────────────────────────────────────────────── + +@pytest.fixture(autouse=False) +def netns(): + """Build two veth egress interfaces + two dummy gateway namespaces. + + Topology: + default ns gw1 ns gw2 ns + egress_if1 <-veth-> gwp1 (acts as a real next-hop) + egress_if2 <-veth-> gwp2 + + Each egress_if is the "egress interface" we hand to nexthop_init. + The gateway peer address (10.k.0.2) is the next_hop argument. + + Teardown is run unconditionally (try/except each step) so a failed run + does not leave stale state that blocks a re-run. + """ + # Wire the real ip binary into rooter so nexthop_* functions hit the kernel. + class _Settings: + ip = _IP + rooter.settings = _Settings + rooter.ServicePaths.ip = _IP + # These tests verify policy ROUTING (ip rule/route), not iptables filtering, and no packets + # traverse. STUB run_iptables so nexthop_enable's MASQUERADE/ACCEPT rules never touch the real + # host firewall -- otherwise the root-gated suite would leave CAPE-rooter NAT rules behind + # (codex/Copilot). Saved + restored around the test. + rooter.ServicePaths.iptables = "/usr/sbin/iptables" + _orig_run_iptables = rooter.run_iptables + # Return non-empty stderr for `-D` so the idempotent delete-until-gone loop terminates after one + # call instead of spinning to its bound; empty stderr (success) for everything else. + rooter.run_iptables = lambda *a, **k: (("", "gone") if "-D" in a else ("", "")) + + NEXTHOP_VM_NET = "192.168.100.0/24" # every test source IP lives here + created_ns = [] + created_links = [] + created_tables = [] # table ids with content to flush + + def _cleanup(): + # Best-effort teardown; each step is independent. + # Use the PRODUCTION filtered teardown for ip rules/routes/tables: it deletes only band rules + # whose source is inside vm_net (never an unrelated host rule at a band priority), flushes the + # gateway tables, and removes the blackhole + intra-subnet exception (codex/Copilot). + try: + rooter.nexthop_teardown(",".join(created_tables), NEXTHOP_VM_NET, "250", "30000", "10000", "10255") + except Exception: + pass + # bring down/del veth links (deleting one side removes the peer too) + for link in created_links: + subprocess.run([_IP, "link", "del", link], capture_output=True) + # delete namespaces + for ns in created_ns: + subprocess.run([_IP, "netns", "del", ns], capture_output=True) + rooter.run_iptables = _orig_run_iptables + + try: + for k in (1, 2): + ns = f"gw{k}" + link = f"egress_if{k}" + peer = f"gwp{k}" + sh(_IP, "netns", "add", ns) + created_ns.append(ns) + sh(_IP, "link", "add", link, "type", "veth", "peer", "name", peer, "netns", ns) + created_links.append(link) + sh(_IP, "addr", "add", f"10.{k}.0.1/24", "dev", link) + sh(_IP, "link", "set", link, "up") + sh(_IP, "netns", "exec", ns, _IP, "addr", "add", f"10.{k}.0.2/24", "dev", peer) + sh(_IP, "netns", "exec", ns, _IP, "link", "set", peer, "up") + created_tables.append(str(200 + k)) # tables 201, 202 + except Exception: + _cleanup() + raise + + yield + + _cleanup() + + +# ─── tests ─────────────────────────────────────────────────────────────────── + +def test_two_profiles_route_to_distinct_interfaces(netns): + """Two VM source IPs bound to two different profiles route via distinct egress interfaces. + + Verification: + (a) Each profile's routing table has a default pointing to its own egress_if. + (b) Each VM's source IP has a policy rule pointing it to its table. + Together these prove that a packet from vm_ip1 would be looked up in table 201 + (-> egress_if1) and a packet from vm_ip2 in table 202 (-> egress_if2). + """ + VM_IP1 = "192.168.100.41" + VM_IP2 = "192.168.100.42" + PRIO1 = "10041" # 10000 + last octet + PRIO2 = "10042" + + # Initialize profiles: forced-default routes into tables 201 / 202 + rooter.nexthop_init("201", "egress_if1", "10.1.0.2") + rooter.nexthop_init("202", "egress_if2", "10.2.0.2") + + # Bind the two VM IPs + rooter.nexthop_enable(VM_IP1, "lo", "egress_if1", "201", PRIO1) + rooter.nexthop_enable(VM_IP2, "lo", "egress_if2", "202", PRIO2) + + # (a) Tables have forced defaults via the correct interface + assert ip_route_table_has_default_via_if("201", "egress_if1"), ( + "table 201 should have a default via egress_if1" + ) + assert ip_route_table_has_default_via_if("202", "egress_if2"), ( + "table 202 should have a default via egress_if2" + ) + + # (b) Source policy rules point each VM IP to its table + assert ip_rule_exists(VM_IP1, "201", PRIO1), ( + f"ip rule should map {VM_IP1} -> table 201 at priority {PRIO1}" + ) + assert ip_rule_exists(VM_IP2, "202", PRIO2), ( + f"ip rule should map {VM_IP2} -> table 202 at priority {PRIO2}" + ) + + # (c) Profiles are DISTINCT: VM_IP1 is NOT in table 202, VM_IP2 is NOT in table 201 + assert not ip_rule_exists(VM_IP1, "202", PRIO1), ( + f"{VM_IP1} must NOT be in table 202" + ) + assert not ip_rule_exists(VM_IP2, "201", PRIO2), ( + f"{VM_IP2} must NOT be in table 201" + ) + + +def test_concurrent_profile_setup_is_consistent(netns): + """Two threads calling nexthop_enable concurrently produce the correct rule set. + + This exercises the real kernel's netlink serialization (iproute2 calls are + serialized by the kernel, not by us — we verify no rules are duplicated or lost). + """ + VM_IP1 = "192.168.100.51" + VM_IP2 = "192.168.100.52" + PRIO1 = "10051" + PRIO2 = "10052" + + rooter.nexthop_init("201", "egress_if1", "10.1.0.2") + rooter.nexthop_init("202", "egress_if2", "10.2.0.2") + + errors = [] + + def bind1(): + try: + rooter.nexthop_enable(VM_IP1, "lo", "egress_if1", "201", PRIO1) + except Exception as e: + errors.append(e) + + def bind2(): + try: + rooter.nexthop_enable(VM_IP2, "lo", "egress_if2", "202", PRIO2) + except Exception as e: + errors.append(e) + + t1 = threading.Thread(target=bind1) + t2 = threading.Thread(target=bind2) + t1.start() + t2.start() + t1.join() + t2.join() + + assert not errors, f"Concurrent nexthop_enable raised: {errors}" + + # Both rules must be present, independently + assert ip_rule_exists(VM_IP1, "201", PRIO1), "Concurrent bind: VM_IP1 rule missing" + assert ip_rule_exists(VM_IP2, "202", PRIO2), "Concurrent bind: VM_IP2 rule missing" + + +def test_unbound_source_is_blackholed(netns): + """An unbound guest-subnet source (no per-task rule) resolves to the fail-closed blackhole. + + After nexthop_fail_closed_enable: + - table 250 has `blackhole default` + - ip rule has a low-priority (30000) rule routing the entire guest subnet (192.168.100.0/24) + to table 250 + + An unbound IP (e.g. 192.168.100.99) has no higher-priority per-task rule, so it + hits the subnet rule -> table 250 -> blackhole. + + We verify the state is correctly installed; we do NOT try to inject a test packet + (that would need an actual VM interface or raw-socket plumbing out of scope here). + """ + VM_NET = "192.168.100.0/24" + FAIL_TABLE = "250" + PRIORITY_LOW = "30000" + BAND_LO = "10000" + + # intra-subnet exception is now a separate primitive (always installed); the blackhole is + # fail_closed_enable (3 args). Install both, as load_nexthop_profiles does when fail_closed=yes. + rooter.nexthop_intra_exception_enable(VM_NET, BAND_LO) + rooter.nexthop_fail_closed_enable(VM_NET, FAIL_TABLE, PRIORITY_LOW) + + # (a) Blackhole default is in table 250 + assert ip_route_table_has_blackhole(FAIL_TABLE), ( + "table 250 should have a blackhole default after nexthop_fail_closed_enable" + ) + + # (b) Subnet rule at priority 30000 routes the guest /24 to table 250 + assert subnet_rule_exists(VM_NET, FAIL_TABLE, PRIORITY_LOW), ( + f"ip rule should route {VM_NET} -> table {FAIL_TABLE} at priority {PRIORITY_LOW}" + ) + + # (c) No per-task rule for the unbound IP (192.168.100.99) exists — + # so it falls through to the subnet rule -> blackhole. + assert not ip_rule_exists("192.168.100.99", FAIL_TABLE, PRIORITY_LOW), ( + "unbound IP must NOT have its own rule — it is covered only by the subnet rule" + ) + + +def test_enable_then_disable_removes_rules(netns): + """nexthop_disable is the mirror of nexthop_enable: all rules are removed.""" + VM_IP = "192.168.100.61" + PRIO = "10061" + + rooter.nexthop_init("201", "egress_if1", "10.1.0.2") + rooter.nexthop_enable(VM_IP, "lo", "egress_if1", "201", PRIO) + + assert ip_rule_exists(VM_IP, "201", PRIO), "Rule must exist after enable" + + rooter.nexthop_disable(VM_IP, "lo", "egress_if1", "201", PRIO) + + assert not ip_rule_exists(VM_IP, "201", PRIO), ( + "Rule must be removed after disable (mirror teardown)" + ) diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py new file mode 100644 index 00000000000..7da2a9697ab --- /dev/null +++ b/tests/test_nexthop_selection.py @@ -0,0 +1,463 @@ +# tests/test_nexthop_selection.py +import pytest +import threading +import lib.cuckoo.core.rooter as core_rooter + + +class _Profile: + def __init__(self, name, interface, rt_table, priority): + self.name, self.interface, self.rt_table, self.priority = name, interface, rt_table, priority + + +def _seed(monkeypatch, live=("gw1", "gw2", "gw3")): + gws = {n: _Profile(n, f"ens{6+i}", str(201 + i), 0) for i, n in enumerate(("gw1", "gw2", "gw3"))} + monkeypatch.setattr(core_rooter, "gateways", gws, raising=False) + monkeypatch.setattr(core_rooter, "_gw_cursor", 0, raising=False) + monkeypatch.setattr(core_rooter, "_gw_live", lambda p: p.name in live) # liveness shim + return gws + + +def test_explicit_id_resolves(monkeypatch): + _seed(monkeypatch) + assert core_rooter._select_gateway("gw2").name == "gw2" + + +def test_explicit_id_down_fails_closed(monkeypatch): + _seed(monkeypatch, live=("gw1", "gw3")) + assert core_rooter._select_gateway("gw2") is None # named-but-down => caller drops + + +def test_roundrobin_cycles_over_live(monkeypatch): + _seed(monkeypatch, live=("gw1", "gw3")) # gw2 down + picks = [core_rooter._select_gateway("roundrobin").name for _ in range(4)] + assert picks == ["gw1", "gw3", "gw1", "gw3"] + + +def test_empty_pool_fails_closed(monkeypatch): + monkeypatch.setattr(core_rooter, "gateways", {}, raising=False) + assert core_rooter._select_gateway("roundrobin") is None + + +def test_roundrobin_threadsafe(monkeypatch): + _seed(monkeypatch) + out = [] + lock = threading.Lock() + + def worker(): + p = core_rooter._select_gateway("roundrobin") + with lock: + out.append(p.name) + threads = [threading.Thread(target=worker) for _ in range(300)] + for t in threads: + t.start() + for t in threads: + t.join() + # even distribution across 3 live gateways, no crash + assert all(out.count(n) == 100 for n in ("gw1", "gw2", "gw3")) + + +def test_unknown_selector_fails_closed(monkeypatch): + # gemini #15: a value that is neither a known gateway id nor 'random'/'roundrobin' must NOT + # silently fall through to roundrobin — it fails closed (returns None so the caller drops). + _seed(monkeypatch) + assert core_rooter._select_gateway("bogus") is None + assert core_rooter._select_gateway("roundrobin").name in ("gw1", "gw2", "gw3") # policy still works + + +# --------------------------------------------------------------------------- +# _FakeRouting helper shared by T7 and T8 tests +# --------------------------------------------------------------------------- + +class _FakeGw1: + """Fake [gw1] section: rt_table is an int (as config.getint would produce).""" + name = "gw1" + interface = "ens6" + next_hop = "onlink" + rt_table = 201 # int — loader must coerce to str + + +class _FakeNexthop: + def __init__(self, enabled=True, route="gw1"): + self.enabled = enabled + self.gateways = "gw1" + self.default_policy = "roundrobin" + self.fail_closed = True + self.vm_net = "192.168.100.0/24" + + +class _FakeRoutingSection: + """Fake top-level routing config section (routing.routing.route).""" + def __init__(self, route="none"): + self.route = route + + +class _FakeVpn: + enabled = False + + +class _FakeRouting: + """Minimal fake routing config for T7/T8 tests. + + Exposes: + .nexthop — _FakeNexthop (enabled/disabled, gateways="gw1") + .gw1 — _FakeGw1 (interface, next_hop, rt_table as int) + .routing — .route + .vpn — .enabled = False + .get(name) -> attribute named `name` + """ + def __init__(self, nexthop_enabled=True, route="none"): + self.nexthop = _FakeNexthop(enabled=nexthop_enabled, route=route) + self.gw1 = _FakeGw1() + self.routing = _FakeRoutingSection(route=route) + self.vpn = _FakeVpn() + + def get(self, name): + return getattr(self, name) + + +# --------------------------------------------------------------------------- +# T7: [gwX] loader — populates gateways global + coerces rt_table to str +# --------------------------------------------------------------------------- + +def test_gwx_loader_populates_and_coerces(monkeypatch): + import lib.cuckoo.core.startup as startup + recorded = [] + # startup.gateways is the dict object the loader writes into (imported reference, same object + # as core_rooter.gateways unless replaced). Clear it in place so both refs see the reset. + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append((cmd, a)) or {}, raising=False) + startup.load_nexthop_profiles(_FakeRouting()) + # Check via startup.gateways (the dict the function mutated) + assert "gw1" in startup.gateways + assert startup.gateways["gw1"].rt_table == "201" # coerced to str + assert ("nexthop_init", ("201", "ens6", "onlink")) in recorded + assert any(c == "nexthop_fail_closed_enable" for c, _ in recorded) + assert any(c == "nexthop_teardown" for c, _ in recorded) + # ORDER MATTERS: nexthop_teardown flushes the gateway tables, so it must run + # BEFORE nexthop_init (which builds them) — otherwise it wipes the fresh routes. + # And fail-closed arms last. (Regression guard for the loader ordering bug found + # in the live FakeNet detonation on 2026-07-01.) + cmds = [c for c, _ in recorded] + assert cmds.index("nexthop_teardown") < cmds.index("nexthop_init"), \ + f"teardown must precede init, got order: {cmds}" + assert cmds.index("nexthop_init") < cmds.index("nexthop_fail_closed_enable"), \ + f"init must precede fail_closed arm, got order: {cmds}" + + +class _DictSection(dict): + """Faithful stand-in for CAPE's config Dictionary: attribute access maps to dict + keys and a MISSING key returns None (not AttributeError). This is exactly how a + real [gwX] section behaves for the absent `name` field.""" + def __getattr__(self, k): + return self.get(k) + + def __setattr__(self, k, v): + self[k] = v + + +def test_gwx_loader_stamps_profile_name_from_section_header(monkeypatch): + """Regression (live FakeNet detonation, 2026-07-01): [gwX] sections carry no `name =` + field (unlike [vpnX]/[socks5]), so config Dictionary.__getattr__ returns None for + entry.name. The loader MUST stamp the section header as the profile id — otherwise + analysis_manager._resolve_nexthop sets self.nexthop_id = profile.name = None, and + _dispatch_nexthop's `if not self.nexthop_id: return` silently no-ops: the per-task + source rule never installs and every real task falls through to the fail-closed + blackhole. Unit tests missed it because the fakes hard-coded .name; the real config + does not.""" + import lib.cuckoo.core.startup as startup + + class _NamelessRouting(_FakeRouting): + def __init__(self): + super().__init__() + # a [gw1] section with NO `name` key — the real routing.conf shape + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + routing = _NamelessRouting() + # precondition: the section reports no name (mimics config Dictionary -> None) + assert routing.gw1.name is None + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: {}, raising=False) + startup.load_nexthop_profiles(routing) + # postcondition: loader stamped the id so nexthop_id resolves to a real value + assert startup.gateways["gw1"].name == "gw1" + + +# --------------------------------------------------------------------------- +# T8: validate_default_route — gateway route accepted; unknown raises +# --------------------------------------------------------------------------- + +def test_nexthop_default_route_boots_without_vpn(monkeypatch): + import lib.cuckoo.core.startup as startup + # Seed gateways so "gw1" is known. validate_default_route reads the module-global + # `gateways` in startup's namespace (bound via `from ... import gateways`), so patch + # THAT binding — patching core_rooter.gateways would be invisible to startup. + monkeypatch.setattr(startup, "gateways", {"gw1": _FakeGw1()}) + # routing.route = "gw1", nexthop enabled, vpn disabled -> must NOT raise + startup.validate_default_route(_FakeRouting(route="gw1")) + + +def test_nexthop_policy_token_default_route_boots_without_vpn(monkeypatch): + # codex P2: a pool-policy token (roundrobin/random) is a valid DEFAULT route when nexthop is + # on — _resolve_nexthop maps it to default_policy and picks from the live pool. validate must + # accept it WITHOUT a VPN, even though it is not a concrete gateway id; otherwise the + # documented pool default raises the vpn-not-enabled error at startup. (gateways empty on + # purpose to prove acceptance is by-token, not by-id.) + import lib.cuckoo.core.startup as startup + monkeypatch.setattr(startup, "gateways", {}) + for tok in ("roundrobin", "random"): + startup.validate_default_route(_FakeRouting(route=tok)) # must NOT raise + + +def test_nexthop_sentinel_default_route_boots_without_vpn(monkeypatch): + # Copilot: `[routing] route = nexthop` is the documented pool default (maps to default_policy). + # validate_default_route must accept the sentinel WITHOUT a VPN, else the default_policy path is + # unreachable in production. gateways empty on purpose (sentinel is not a gateway id). + import lib.cuckoo.core.startup as startup + monkeypatch.setattr(startup, "gateways", {}) + startup.validate_default_route(_FakeRouting(route="nexthop")) # must NOT raise + + +def test_gateway_named_nexthop_raises(monkeypatch): + # Copilot: "nexthop" is the pool sentinel, so a [gwX] must not be named it (ambiguous with the + # default-policy route). Rejected via _RESERVED_ROUTE_NAMES. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwNamedNexthop(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="nexthop", default_policy="roundrobin", + fail_closed=True, vm_net="192.168.100.0/24") + self.nexthop_section = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + def get(self, name): + return self.nexthop_section if name == "nexthop" else getattr(self, name) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwNamedNexthop()) + + +def test_unknown_gateway_default_route_raises(monkeypatch): + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + # gateways is empty — "gw9" is unknown (patch startup's binding, not core_rooter's) + monkeypatch.setattr(startup, "gateways", {}) + with pytest.raises(CuckooStartupError): + startup.validate_default_route(_FakeRouting(route="gw9")) + + +# --------------------------------------------------------------------------- +# T11: no-regress — disabled [nexthop] is a no-op (empty gateways, no rooter calls) +# --------------------------------------------------------------------------- + +def test_nexthop_disabled_is_noop(monkeypatch): + import lib.cuckoo.core.startup as startup + import lib.cuckoo.core.rooter as core_rooter + monkeypatch.setattr(core_rooter, "gateways", {}, raising=False) + called = [] + monkeypatch.setattr(startup, "rooter", lambda *a, **k: called.append(a) or {}, raising=False) + startup.load_nexthop_profiles(_FakeRouting(nexthop_enabled=False)) + assert core_rooter.gateways == {} and called == [] + + +# --------------------------------------------------------------------------- +# gemini #14 MEDIUM: [nexthop]/[gwX] required-option validation (clear startup error) +# --------------------------------------------------------------------------- + +def test_nexthop_missing_vm_net_raises(monkeypatch): + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _NexthopNoVmNet(_FakeRouting): + def __init__(self): + super().__init__() + # enabled + gateways set, but vm_net absent (config Dictionary -> None) + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="roundrobin", fail_closed=True) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_NexthopNoVmNet()) + + +def test_gwx_missing_interface_raises(monkeypatch): + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwNoInterface(_FakeRouting): + def __init__(self): + super().__init__() + # [gw1] with next_hop + rt_table but NO interface (config Dictionary -> None) + self.gw1 = _DictSection(next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwNoInterface()) + + +def test_gwx_reserved_rt_table_raises(monkeypatch): + # ADVERSARIAL-REVIEW HIGH (2026-07-08): the nexthop_init reserved-table guard alone leaves a + # fail-OPEN — a [gwX] with rt_table=main is still registered/selectable, and its per-task rule + # `from vm_ip lookup main priority 100xx` (below the 30000 blackhole) routes the VM out the host + # default route. Reject a reserved rt_table at LOAD so the profile can never be dispatched. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + for bad in ("main", "local", "default", "254", "255", "253", "0"): + class _GwReservedTable(_FakeRouting): + def __init__(self, rt=bad): + super().__init__() + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=rt) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwReservedTable()) + assert "gw1" not in startup.gateways, f"reserved-table gw1 (rt={bad}) leaked into gateways" + + +def test_gwx_rt_table_is_fail_table_raises(monkeypatch): + # codex P2: a [gwX] rt_table == NEXTHOP_FAIL_TABLE (250) is not a kernel-reserved table so it + # passes the reserved-table check, but nexthop_fail_closed_enable's `ip route replace blackhole + # default table 250` would overwrite the gateway's default with a blackhole → every task on that + # gateway silently drops. Reject it at load. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwFailTable(_FakeRouting): + def __init__(self): + super().__init__() + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=startup.NEXTHOP_FAIL_TABLE) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwFailTable()) + assert "gw1" not in startup.gateways + + +def test_gwx_duplicate_rt_table_raises(monkeypatch): + # codex P2: two [gwX] sharing an rt_table → nexthop_init flush/replaces the one table per profile + # so the last gateway's default wins, but the earlier gateway stays selectable with a per-task rule + # pointing at a table for the WRONG egress interface → misroute/drop. Fail startup on duplicates. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwDupTable(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1,gw2", default_policy="roundrobin", + fail_closed=True, vm_net="192.168.100.0/24") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + self.gw2 = _DictSection(interface="ens7", next_hop="onlink", rt_table=201) # same table -> reject + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwDupTable()) + + +def test_nexthop_intra_exception_installed_even_when_fail_closed_off(monkeypatch): + # codex P2: the intra-subnet exception is a CONNECTIVITY guarantee, separate from the blackhole. + # With [nexthop] fail_closed=no it MUST still be installed (else a bound VM's intra-vm_net traffic + # misroutes to the gateway), while the blackhole (nexthop_fail_closed_enable) must NOT be. + import lib.cuckoo.core.startup as startup + recorded = [] + + class _NexthopNoFailClosed(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="roundrobin", + fail_closed=False, vm_net="192.168.100.0/24") + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {}, raising=False) + startup.load_nexthop_profiles(_NexthopNoFailClosed()) + assert "nexthop_intra_exception_enable" in recorded + assert "nexthop_fail_closed_enable" not in recorded + + +def test_gwx_rt_table_collides_with_vpn_raises(monkeypatch): + # adversarial-review HIGH: a [gwX] rt_table equal to a configured VPN's rt_table would clobber the + # VPN's just-built routing table (nexthop_init flush/replace) -> VPN tasks silently egress the + # gateway NIC instead of the tunnel (VPN-isolation break). Reject at load. The VPN rt_table is an + # int here to prove the str-coercion works (config may give int or name). + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwVpnTable(_FakeRouting): + def __init__(self): + super().__init__() + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {"vpn0": type("V", (), {"rt_table": 201})()}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwVpnTable()) + assert "gw1" not in startup.gateways + + +def test_gwx_rt_table_collides_with_dirty_line_raises(monkeypatch): + # adversarial-review HIGH: a [gwX] rt_table equal to routing.routing.rt_table (the internet + # dirty-line table) would be clobbered by the dirty-line init_rttable that runs AFTER us -> + # gateway tasks egress the dirty line instead of the gateway egress_if. Reject at load. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _GwDirtyLineTable(_FakeRouting): + def __init__(self): + super().__init__() + self.routing = _DictSection(route="none", rt_table="201") # dirty-line table == gw table + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwDirtyLineTable()) + assert "gw1" not in startup.gateways + + +def test_gateway_named_like_policy_token_raises(monkeypatch): + # A [gwX] must not be named a pool-policy token (roundrobin/random): _select_gateway would + # treat the name as the policy, not the id, so the profile could never be explicitly selected. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + for tok in ("roundrobin", "random"): + class _GwPolicyName(_FakeRouting): + def __init__(self, name=tok): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways=name, default_policy="roundrobin", + fail_closed=True, vm_net="192.168.100.0/24") + setattr(self, name, _DictSection(interface="ens6", next_hop="onlink", rt_table=201)) + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_GwPolicyName()) + + +def test_nexthop_enabled_empty_pool_raises(monkeypatch): + # gemini medium: [nexthop] enabled with an empty/blank gateways list parses zero profiles. + # Without a guard, every task would silently fall through to the fail-closed blackhole — + # so fail loudly at startup instead. `gateways = ""` passes the not-None required-option + # check (empty string != None) but yields no profiles. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _NexthopEmptyPool(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways=" , ", default_policy="roundrobin", + fail_closed=True, vm_net="192.168.100.0/24") + + startup.gateways.clear() + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_NexthopEmptyPool()) diff --git a/tests/test_rooter_nexthop.py b/tests/test_rooter_nexthop.py new file mode 100644 index 00000000000..b0de9b333e6 --- /dev/null +++ b/tests/test_rooter_nexthop.py @@ -0,0 +1,233 @@ +# tests/test_rooter_nexthop.py +import pytest +import utils.rooter as rooter + + +@pytest.fixture +def rec(monkeypatch): + """Record every run()/run_iptables() invocation as a list of argv tuples. + + utils.rooter functions reference settings.ip / ServicePaths.ip (both the same + path at runtime, defined only in __main__), so inject both for unit tests. + """ + calls = {"run": [], "iptables": []} + + def fake_run(*args): + calls["run"].append(tuple(str(a) for a in args)) + return ("", "") # (stdout, stderr); real run() never raises + + def fake_run_iptables(*args, **kwargs): + calls["iptables"].append(tuple(str(a) for a in args)) + # A `-D` of an absent rule returns non-empty stderr; report "gone" so the idempotent + # delete-until-gone loop (_iptables_delete_all) terminates after one delete in tests. + if "-D" in args: + return ("", "iptables: Bad rule (does a matching rule exist in that chain?).") + return ("", "") + + class _Settings: + ip = "ip" + monkeypatch.setattr(rooter, "settings", _Settings, raising=False) + monkeypatch.setattr(rooter.ServicePaths, "ip", "ip", raising=False) + monkeypatch.setattr(rooter.ServicePaths, "iptables", "iptables", raising=False) + monkeypatch.setattr(rooter, "run", fake_run) + monkeypatch.setattr(rooter, "run_iptables", fake_run_iptables) + return calls + + +def test_recorder_captures(rec): + rooter.run("ip", "route", "show") + assert rec["run"] == [("ip", "route", "show")] + + +# --------------------------------------------------------------------------- +# Task 1: nexthop_init +# --------------------------------------------------------------------------- + +def test_nexthop_init_onlink(rec): + rooter.nexthop_init("201", "ens6", "onlink") + assert rec["run"] == [ + ("ip", "route", "flush", "table", "201"), + ("ip", "route", "replace", "default", "dev", "ens6", "onlink", "table", "201"), + ] + + +def test_nexthop_init_via(rec): + rooter.nexthop_init("202", "ens7", "10.30.72.1") + assert rec["run"] == [ + ("ip", "route", "flush", "table", "202"), + ("ip", "route", "replace", "default", "via", "10.30.72.1", "dev", "ens7", "table", "202"), + ] + + +def test_nexthop_init_skips_reserved_table(rec): + # codex P1 / gemini critical: load_nexthop_profiles feeds each [gwX] rt_table straight into + # nexthop_init's flush at startup, so a misconfigured reserved table (main/254/...) must NOT + # be flushed — doing so would wipe the host's own routing. nexthop_teardown already guards + # this set; the init path must too. No ip command runs at all for a reserved table. + for bad in ("main", "local", "default", "254", "255", "253", "0"): + rec["run"].clear() + rooter.nexthop_init(bad, "ens6", "onlink") + assert rec["run"] == [], f"nexthop_init flushed reserved table {bad!r}: {rec['run']}" + + +# --------------------------------------------------------------------------- +# Task 2: nexthop_enable +# --------------------------------------------------------------------------- + +def test_nexthop_enable_argv(rec): + # signature: (vm_ip, ingress_if, egress_if, rt_table, priority) + rooter.nexthop_enable("192.168.100.42", "virbr0", "ens6", "201", "10042") + # iproute2 + conntrack go through run(); nat/filter through run_iptables() + assert rec["run"] == [ + ("conntrack", "-D", "-s", "192.168.100.42"), # pre-bind flush + ("ip", "rule", "del", "from", "192.168.100.42", "lookup", "201", "priority", "10042"), # idempotent pre-clean + ("ip", "rule", "add", "from", "192.168.100.42", "lookup", "201", "priority", "10042"), + ] + # Each iptables rule is delete-until-gone (idempotent; Copilot) then added once. The forward + # ACCEPT goes into CAPE_ACCEPTED_SEGMENTS (jumped first in FORWARD), NOT a tail `-A FORWARD` + # (else a libvirt `-i virbr* -j REJECT` would shadow it), and is constrained to the guest + # ingress interface `-i virbr0` so a spoofed source from another NIC isn't forwarded (codex). + assert rec["iptables"] == [ + ("-t", "nat", "-D", "POSTROUTING", "-s", "192.168.100.42", "-o", "ens6", "-j", "MASQUERADE"), + ("-t", "nat", "-A", "POSTROUTING", "-s", "192.168.100.42", "-o", "ens6", "-j", "MASQUERADE"), + ("-D", "CAPE_ACCEPTED_SEGMENTS", "-i", "virbr0", "-s", "192.168.100.42", "-o", "ens6", "-j", "ACCEPT"), + ("-I", "CAPE_ACCEPTED_SEGMENTS", "-i", "virbr0", "-s", "192.168.100.42", "-o", "ens6", "-j", "ACCEPT"), + ] + # never a raw tail FORWARD append (regression guard for the shadowing bug) + assert not any(a[:2] == ("-A", "FORWARD") for a in rec["iptables"]) + + +# --------------------------------------------------------------------------- +# Task 3: nexthop_disable +# --------------------------------------------------------------------------- + +def test_nexthop_disable_argv(rec): + # signature: (vm_ip, ingress_if, egress_if, rt_table, priority) + rooter.nexthop_disable("192.168.100.42", "virbr0", "ens6", "201", "10042") + assert rec["run"] == [ + ("ip", "rule", "del", "from", "192.168.100.42", "lookup", "201", "priority", "10042"), + ("conntrack", "-D", "-s", "192.168.100.42"), + ] + # mirror-delete (until gone) from the same chain/rules enable installed, incl. the -i ingress + assert rec["iptables"] == [ + ("-t", "nat", "-D", "POSTROUTING", "-s", "192.168.100.42", "-o", "ens6", "-j", "MASQUERADE"), + ("-D", "CAPE_ACCEPTED_SEGMENTS", "-i", "virbr0", "-s", "192.168.100.42", "-o", "ens6", "-j", "ACCEPT"), + ] + + +# --------------------------------------------------------------------------- +# Task 4: nexthop_intra_exception_enable + nexthop_fail_closed_enable (split; codex P2) +# --------------------------------------------------------------------------- + +def test_nexthop_intra_exception_argv(rec): + # band_lo=10000 => exception at band_lo-1 = 9999, BELOW the per-task band so it wins over a bound + # VM's `from lookup ` rule for intra-vm_net destinations. Installed regardless + # of fail_closed (connectivity, not fail-closed) -- see test_nexthop_disabled_is_noop siblings. + rooter.nexthop_intra_exception_enable("192.168.100.0/24", "10000") + assert rec["run"] == [ + ("ip", "rule", "del", "from", "192.168.100.0/24", "to", "192.168.100.0/24", "lookup", "main", "priority", "9999"), + ("ip", "rule", "add", "from", "192.168.100.0/24", "to", "192.168.100.0/24", "lookup", "main", "priority", "9999"), + ] + # ORDERING INVARIANT: the intra-subnet exception MUST sit below the per-task band (10000), + # otherwise a bound VM's per-task rule shadows it and intra-vm_net traffic leaks to the gateway. + intra = [r for r in rec["run"] if r[:3] == ("ip", "rule", "add") and "to" in r] + assert intra and all(int(r[-1]) < 10000 for r in intra), intra + + +def test_nexthop_fail_closed_argv(rec): + # fail_closed_enable now installs ONLY the blackhole (route + rule); the intra-subnet exception + # is a separate primitive installed regardless of fail_closed (codex P2). Signature dropped band_lo. + rooter.nexthop_fail_closed_enable("192.168.100.0/24", "250", "30000") + assert rec["run"] == [ + ("ip", "route", "replace", "blackhole", "default", "table", "250"), + ("ip", "rule", "del", "from", "192.168.100.0/24", "lookup", "250", "priority", "30000"), + ("ip", "rule", "add", "from", "192.168.100.0/24", "lookup", "250", "priority", "30000"), + ] + # the blackhole primitive must NOT install the intra-subnet (`to vm_net`) exception anymore + assert not any("to" in r for r in rec["run"]) + + +# --------------------------------------------------------------------------- +# Task 5: nexthop_teardown + nexthop_configure +# --------------------------------------------------------------------------- + +def test_nexthop_teardown_sweeps_policy_routing(rec, monkeypatch): + # `ip rule show` returns two in-band per-task rules (from within vm_net), an out-of-band rule, + # AND an UNRELATED host admin rule that happens to sit in the 10000-10255 band but whose source + # is OUTSIDE vm_net -- teardown must NOT delete that one (codex P2). + def fake_run(*args): + rec["run"].append(tuple(str(a) for a in args)) + if args[:3] == ("ip", "rule", "show"): + return ("10042: from 192.168.100.42 lookup 201\n" + "10043: from 192.168.100.43 lookup 202\n" + "10099: from 10.0.0.5 lookup 999\n" # unrelated host rule in the band + "32766: from all lookup main\n", "") + return ("", "") + monkeypatch.setattr(rooter, "run", fake_run) + + rooter.nexthop_teardown("201,202", "192.168.100.0/24", "250", "30000", "10000", "10255") + + assert ("ip", "route", "flush", "table", "201") in rec["run"] + assert ("ip", "route", "flush", "table", "202") in rec["run"] + assert ("ip", "route", "del", "blackhole", "default", "table", "250") in rec["run"] + assert ("ip", "rule", "del", "from", "192.168.100.0/24", "lookup", "250", "priority", "30000") in rec["run"] + # intra-subnet exception rule also removed on teardown + assert ("ip", "rule", "del", "from", "192.168.100.0/24", "to", "192.168.100.0/24", "lookup", "main", "priority", "9999") in rec["run"] + # in-band per-task rules (source in vm_net) swept -- by FULL selector (from + priority), not + # priority alone, so a host rule sharing the priority is never hit (codex). + assert ("ip", "rule", "del", "from", "192.168.100.42", "priority", "10042") in rec["run"] + assert ("ip", "rule", "del", "from", "192.168.100.43", "priority", "10043") in rec["run"] + # nothing deleted for the out-of-band 32766 main rule, nor for the in-band-but-NOT-ours rule + # (from 10.0.0.5, outside vm_net) -- neither its priority nor its source appears in any `del`. + dels = [r for r in rec["run"] if r[:3] == ("ip", "rule", "del")] + assert not any("32766" in r for r in dels) + assert not any(("10099" in r) or ("10.0.0.5" in r) for r in dels) + + +def test_nexthop_teardown_skips_reserved_tables(rec): + # gemini #14 HIGH: even if a gateway profile is misconfigured with a reserved/system table + # id, teardown must NOT flush it — doing so (at startup + SIGTERM) would wipe the host's own + # routing and take the box offline. The real gateway table IS still flushed. + rooter.nexthop_teardown("main,254,201", "192.168.100.0/24", "250", "30000", "10000", "10255") + assert ("ip", "route", "flush", "table", "201") in rec["run"] + assert ("ip", "route", "flush", "table", "main") not in rec["run"] + assert ("ip", "route", "flush", "table", "254") not in rec["run"] + + +def test_nexthop_configure_sets_globals(rec): + rooter.nexthop_configure("201,202", "192.168.100.0/24", "250", "30000", "10000", "10255") + assert rooter.GATEWAY_TABLES_CSV == "201,202" + assert rooter.NEXTHOP_VM_NET == "192.168.100.0/24" + assert rooter.NEXTHOP_FAIL_TABLE == "250" + assert rooter.NEXTHOP_PRIORITY_LOW == "30000" + assert rooter.NEXTHOP_BAND_LO == "10000" + assert rooter.NEXTHOP_BAND_HI == "10255" + + +# --------------------------------------------------------------------------- +# SIGTERM teardown gate — disabled node must be a no-op (Copilot C3) +# --------------------------------------------------------------------------- + +def test_sigterm_teardown_noop_when_nexthop_never_configured(rec, monkeypatch): + # On a node that never enabled [nexthop], GATEWAY_TABLES_CSV is "" and the SIGTERM teardown + # helper must issue ZERO ip commands — no host policy-routing mutation, and crucially no + # 10000-10255 band sweep (which could delete an unrelated host rule). Guards the disabled=no-op + # contract, which is otherwise untestable (handle_sigterm lives inside __main__). + monkeypatch.setattr(rooter, "GATEWAY_TABLES_CSV", "", raising=False) + rooter.nexthop_teardown_if_configured() + assert rec["run"] == [] + assert rec["iptables"] == [] + + +def test_sigterm_teardown_runs_when_configured(rec, monkeypatch): + # When the loader configured nexthop this session, the SIGTERM helper DOES tear down the + # gateway tables (positive control so the no-op test above can't pass by accident). + monkeypatch.setattr(rooter, "GATEWAY_TABLES_CSV", "201,202", raising=False) + monkeypatch.setattr(rooter, "NEXTHOP_VM_NET", "192.168.100.0/24", raising=False) + monkeypatch.setattr(rooter, "NEXTHOP_FAIL_TABLE", "250", raising=False) + monkeypatch.setattr(rooter, "NEXTHOP_PRIORITY_LOW", "30000", raising=False) + monkeypatch.setattr(rooter, "NEXTHOP_BAND_LO", "10000", raising=False) + monkeypatch.setattr(rooter, "NEXTHOP_BAND_HI", "10255", raising=False) + rooter.nexthop_teardown_if_configured() + assert ("ip", "route", "flush", "table", "201") in rec["run"] + assert ("ip", "route", "flush", "table", "202") in rec["run"] diff --git a/tests/test_route_network_nexthop.py b/tests/test_route_network_nexthop.py new file mode 100644 index 00000000000..59120a4927f --- /dev/null +++ b/tests/test_route_network_nexthop.py @@ -0,0 +1,227 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +"""Unit tests for the per-task [nexthop] resolve+bind branch in +AnalysisManager.route_network / unroute_network. + +We drive the extracted helpers (_resolve_nexthop / _dispatch_nexthop / +_unroute_nexthop) through a small `_route` shim so the branch is testable +without standing up the full route_network() (DB, Config, machinery).""" +import pytest + +import lib.cuckoo.core.analysis_manager as am + + +@pytest.fixture +def mgr(monkeypatch): + """A minimal AnalysisManager with a recording rooter and a fake machine/route.""" + m = am.AnalysisManager.__new__(am.AnalysisManager) + m.interface = m.rt_table = m.route = None + m.nexthop_id = m.nexthop_interface = m.nexthop_rt_table = m.nexthop_priority = None + m.no_local_routing = m.reject_segments = m.reject_hostports = None + m.rooter_response = "" + + class _Mach: + ip = "192.168.100.42" + interface = "virbr0" + + m.machine = _Mach() + calls = [] + monkeypatch.setattr(am, "rooter", lambda cmd, *a, **k: calls.append((cmd, a)) or {}, raising=False) + m._calls = calls + return m + + +def _fake_routing(default_policy="roundrobin", default_route="nexthop"): + """Minimal stand-in for Config('routing') exposing .nexthop.* and .routing.route + (the configured default route — _resolve_nexthop only falls back to default_policy + when self.route equals it; a typo'd explicit route drops instead).""" + return type( + "R", + (), + { + "nexthop": type("NH", (), {"enabled": True, "default_policy": default_policy})(), + "routing": type("RT", (), {"route": default_route})(), + }, + )() + + +def _route(mgr, route, nexthop_enabled, default_policy="roundrobin", default_route="nexthop", tun_iface_exists=False): + """Replicate ONLY route_network's tunX + nexthop branches (the two the C1 reorder concerns). + + This deliberately models just the tail of route_network's resolution chain: it captures the + RELATIVE ORDER of the tunX branch vs the nexthop branch and the nexthop-consumes-the-route + behaviour. It does NOT model the earlier legacy branches (none/inetsim/tor/internet/vpn/socks5) + or the post-resolution nic_available fallback -- in real route_network those legacy routes are + consumed by their own branches and never reach _resolve_nexthop. Tests that pass route="internet" + etc. here are exercising _resolve_nexthop's defensive reserved-DROP guard in isolation, not a + production route_network path. + + The modelled invariants (Copilot fix): the tunX branch is checked BEFORE the nexthop branch, + because _resolve_nexthop rewrites self.route="drop" for any route it does not own and would + otherwise clobber an explicit tun route; and when [nexthop] is enabled its branch CONSUMES the + route (bind a gateway, or force drop) rather than an `and`-guarded pass, so a nexthop drop falls + into the none/drop/false dispatch below instead of a misleading "Unknown route" else. + `nexthop_enabled` False => the branch is never entered (legacy paths byte-for-byte unchanged). + """ + routing = _fake_routing(default_policy, default_route) + mgr.route = route + # resolution chain: tun BEFORE nexthop + if str(mgr.route)[:3] == "tun" and tun_iface_exists: + mgr.interface = mgr.route + elif nexthop_enabled: + mgr._resolve_nexthop(routing) + # dispatch chain: forced-drop handled by the existing none/drop/false dispatch; tun by its own + if str(mgr.route).lower() in ("none", "drop", "false"): + am.rooter("drop_enable", mgr.machine.ip, "2042") + elif str(mgr.route)[:3] == "tun" and tun_iface_exists: + am.rooter("interface_route_tun_enable", mgr.machine.ip, mgr.route, "1") + mgr._dispatch_nexthop() + + +def test_tun_route_not_hijacked_when_nexthop_enabled(mgr, monkeypatch): + # Copilot: an explicit tunX route must be handled by the tun branch, NOT clobbered to drop by + # _resolve_nexthop, when [nexthop] is enabled. route_network checks tun BEFORE nexthop. + # _select_gateway WOULD bind a live profile if reached — the fix must not reach it for a tun route. + live = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: live, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": live}, raising=False) + _route(mgr, route="tun0", nexthop_enabled=True, tun_iface_exists=True) + cmds = [c for c, _ in mgr._calls] + assert mgr.route == "tun0" # NOT rewritten to drop + assert mgr.interface == "tun0" # handled by the tun branch + assert mgr.nexthop_id is None # nexthop never bound it + assert "interface_route_tun_enable" in cmds + assert "drop_enable" not in cmds and "nexthop_enable" not in cmds + + +def test_explicit_nexthop_sentinel_uses_default_policy(mgr, monkeypatch): + # Copilot: route="nexthop" (explicit) maps to default_policy pool selection regardless of the + # node's configured default route — this is what makes the documented pool default reachable. + prof = type("P", (), {"name": "gw2", "interface": "ens7", "rt_table": "202", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: prof if r in ("roundrobin", "random") else None, raising=False) + monkeypatch.setattr(am, "gateways", {"gw2": prof}, raising=False) + # node default is a DIFFERENT gateway, proving the "nexthop" sentinel is honored on its own + _route(mgr, route="nexthop", nexthop_enabled=True, default_policy="roundrobin", default_route="gw2") + cmds = [c for c, _ in mgr._calls] + assert "nexthop_enable" in cmds and "drop_enable" not in cmds + assert mgr.nexthop_id == "gw2" + + +def test_explicit_gateway_binds_and_skips_generic(mgr, monkeypatch): + prof = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: prof if r == "gw1" else None, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": prof}, raising=False) + _route(mgr, route="gw1", nexthop_enabled=True) + # bound to the profile, generic forward/srcroute NOT used + assert mgr.interface is None # generic block skipped (H2 option b) + assert mgr.rt_table is None # so the srcroute_enable elif is skipped too + assert mgr.nexthop_id == "gw1" + assert mgr.nexthop_interface == "ens6" + assert mgr.nexthop_rt_table == "201" + assert mgr.nexthop_priority == "10042" # 10000 + last octet + cmds = [c for c, _ in mgr._calls] + assert "nexthop_enable" in cmds + assert "forward_enable" not in cmds and "srcroute_enable" not in cmds + assert "drop_enable" not in cmds + + +def test_typo_gateway_fails_closed_to_drop(mgr, monkeypatch): + monkeypatch.setattr(am, "_select_gateway", lambda r: None, raising=False) + monkeypatch.setattr(am, "gateways", {}, raising=False) + _route(mgr, route="gw9", nexthop_enabled=True) + cmds = [c for c, _ in mgr._calls] + assert "drop_enable" in cmds # review B1: never fall through to no-op + assert "nexthop_enable" not in cmds + assert mgr.route == "drop" + assert mgr.nexthop_id is None + assert mgr.interface is None # not left on host default forwarding + + +def test_typod_route_drops_even_when_a_gateway_is_live(mgr, monkeypatch): + # gemini #14 HIGH: a typo'd/foreign route (e.g. "vpn9") that is NOT a gateway id, NOT a + # policy token, and NOT the configured default route must DROP — it must never fall back + # to default_policy and silently egress via a live gateway (isolation-boundary bypass). + live = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + # _select_gateway WOULD return a live profile if reached — the fix must not reach it. + monkeypatch.setattr(am, "_select_gateway", lambda r: live, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": live}, raising=False) + _route(mgr, route="vpn9", nexthop_enabled=True, default_route="nexthop") + cmds = [c for c, _ in mgr._calls] + assert "drop_enable" in cmds and "nexthop_enable" not in cmds + assert mgr.route == "drop" and mgr.nexthop_id is None + + +def test_configured_default_route_uses_default_policy(mgr, monkeypatch): + # When the task uses the node's CONFIGURED default route (self.route == routing.routing.route), + # fall back to default_policy (roundrobin/random) and pick from the live pool. + prof = type("P", (), {"name": "gw2", "interface": "ens7", "rt_table": "202", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: prof if r in ("roundrobin", "random") else None, raising=False) + monkeypatch.setattr(am, "gateways", {"gw2": prof}, raising=False) + _route(mgr, route="nexthop", nexthop_enabled=True, default_policy="roundrobin", default_route="nexthop") + cmds = [c for c, _ in mgr._calls] + assert "nexthop_enable" in cmds and "drop_enable" not in cmds + assert mgr.nexthop_id == "gw2" + + +def test_reserved_route_never_resolves_to_gateway(mgr, monkeypatch): + # gemini #15 (security-critical): a reserved route (none/drop/false/internet/tor/inetsim) must + # DROP, never resolve to a gateway — even if it is the node's configured default route. + live = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: live, raising=False) # would bind if reached + monkeypatch.setattr(am, "gateways", {"gw1": live}, raising=False) + _route(mgr, route="internet", nexthop_enabled=True, default_route="internet") + cmds = [c for c, _ in mgr._calls] + assert "drop_enable" in cmds and "nexthop_enable" not in cmds + assert mgr.route == "drop" and mgr.nexthop_id is None + + +def test_all_nexthop_args_are_str(mgr, monkeypatch): + prof = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: prof, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": prof}, raising=False) + _route(mgr, route="gw1", nexthop_enabled=True) + found = False + for cmd, args in mgr._calls: + if cmd == "nexthop_enable": + found = True + assert all(isinstance(a, str) for a in args) + assert found + + +def test_no_regress_vpn_and_none(mgr): + # nexthop DISABLED -> _resolve_nexthop must never run; route=vpn0/none unchanged. + # vpn0 path + _route(mgr, route="vpn0", nexthop_enabled=False) + assert mgr.nexthop_id is None + cmds = [c for c, _ in mgr._calls] + assert "nexthop_enable" not in cmds # no nexthop calls leak into the legacy path + # none path + mgr._calls.clear() + mgr.nexthop_id = None + _route(mgr, route="none", nexthop_enabled=False) + assert "nexthop_enable" not in [c for c, _ in mgr._calls] + + +def test_unroute_mirrors_persisted_tuple(mgr, monkeypatch): + prof = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + monkeypatch.setattr(am, "_select_gateway", lambda r: prof, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": prof}, raising=False) + _route(mgr, route="gw1", nexthop_enabled=True) + enable_args = next(a for c, a in mgr._calls if c == "nexthop_enable") + mgr._calls.clear() + mgr._unroute_nexthop() # extracted helper called from unroute_network + disable_args = next(a for c, a in mgr._calls if c == "nexthop_disable") + # disable deletes EXACTLY what enable created (vm_ip, ingress bridge, egress iface, rt_table, + # priority). The selector is NOT re-run: the persisted self.nexthop_* tuple + machine is used (M5). + assert disable_args == ("192.168.100.42", "virbr0", "ens6", "201", "10042") + assert disable_args == enable_args + + +def test_unroute_noop_when_not_nexthop(mgr): + # no nexthop_id => _unroute_nexthop must issue nothing (legacy paths untouched, + # and the generic disable block is skipped because self.interface is None). + mgr.nexthop_id = None + mgr._unroute_nexthop() + assert [c for c, _ in mgr._calls if c == "nexthop_disable"] == [] diff --git a/utils/rooter.py b/utils/rooter.py index 45f1cd018e8..39e1ed346f9 100644 --- a/utils/rooter.py +++ b/utils/rooter.py @@ -39,6 +39,17 @@ class ServicePaths: ip = None +# --------------------------------------------------------------------------- +# nexthop primitive — module-level state (loader sets these via nexthop_configure) +# --------------------------------------------------------------------------- +GATEWAY_TABLES_CSV = "" +NEXTHOP_VM_NET = "255.255.255.255/32" # matches nothing if mis-fired before configure +NEXTHOP_FAIL_TABLE = "250" # best-effort del is a no-op; table 250 is not kernel-reserved +NEXTHOP_PRIORITY_LOW = "30000" +NEXTHOP_BAND_LO = "10000" +NEXTHOP_BAND_HI = "10255" + + def run(*args): """Wrapper to subprocess.run.""" log.debug("Running command: %s", " ".join(args)) @@ -650,6 +661,169 @@ def srcroute_disable(rt_table, ipaddr): run(settings.ip, "route", "flush", "cache") +# --------------------------------------------------------------------------- +# nexthop primitive — argv builders +# --------------------------------------------------------------------------- + +# Linux reserved/system routing tables that a [gwX] rt_table must never target — flushing any of +# these would wipe the host's own routing and take the box offline. The startup loader rejects a +# reserved rt_table up front (fail loud); these guards are defence-in-depth for the rooter path. +# Kernel-fixed ids. +RESERVED_RT_TABLES = ("local", "main", "default", "0", "253", "254", "255") + + +def nexthop_init(rt_table, egress_if, next_hop): + """Idempotently build a next-hop profile's routing table: one forced default route. + next_hop == 'onlink' => default dev egress_if onlink; else default via next_hop dev egress_if. + NOTE: deliberately does NOT call init_rttable (that copies main's per-interface routes).""" + # NEVER flush a reserved/system routing table (codex P1 / gemini critical). See RESERVED_RT_TABLES. + if rt_table in RESERVED_RT_TABLES: + log.error("nexthop_init refusing to flush reserved routing table %r (fix the [gwX] rt_table)", rt_table) + return + run(settings.ip, "route", "flush", "table", rt_table) + if next_hop == "onlink": + run(settings.ip, "route", "replace", "default", "dev", egress_if, "onlink", "table", rt_table) + else: + run(settings.ip, "route", "replace", "default", "via", next_hop, "dev", egress_if, "table", rt_table) + + +def _iptables_delete_all(*args): + """Best-effort delete an iptables rule until it is no longer present, so a retried bind can't + leave stacked duplicates (a single -D would remove only one copy). run_iptables never raises; + a non-empty stderr from `-D` means the rule is gone. Bounded so a persistent error can't spin.""" + for _ in range(32): + _, err = run_iptables(*args) + if err: + break + + +def nexthop_enable(vm_ip, ingress_if, egress_if, rt_table, priority): + """Per task: source-route the VM into its profile table and SNAT onto egress_if. Idempotent -- + a retried bind (or route_network called twice) must NOT stack rules, so the ip rule and both + iptables rules are delete-then-add'd; the iptables deletes loop until gone.""" + run("conntrack", "-D", "-s", vm_ip) # drop stale flows so a recycled IP starts clean (best-effort) + run(settings.ip, "rule", "del", "from", vm_ip, "lookup", rt_table, "priority", priority) # idempotent + run(settings.ip, "rule", "add", "from", vm_ip, "lookup", rt_table, "priority", priority) + # SNAT the VM out its gateway. Delete-until-gone first so a retry can't stack duplicate NAT rules + # that nexthop_disable's delete would then leave behind (Copilot). + _iptables_delete_all("-t", "nat", "-D", "POSTROUTING", "-s", vm_ip, "-o", egress_if, "-j", "MASQUERADE") + run_iptables("-t", "nat", "-A", "POSTROUTING", "-s", vm_ip, "-o", egress_if, "-j", "MASQUERADE") + # Accept the VM's forwarded egress via CAPE_ACCEPTED_SEGMENTS -- the chain FORWARD jumps to FIRST + # (cleanup_rooter) -- so it precedes any libvirt default `-i virbr* -j REJECT` still in the raw + # FORWARD chain (a tail `-A FORWARD ... ACCEPT` would be shadowed on libvirt guests). Constrain -i + # to the guest ingress interface so a spoofed source from another host NIC can't be forwarded and + # MASQUERADEd through the gateway -- parity with the legacy forward_enable path (codex). ACCEPT + # terminates traversal, so return traffic is covered by the ESTABLISHED,RELATED accept in the same + # chain. Delete-until-gone before insert (idempotent, Copilot). + _iptables_delete_all("-D", "CAPE_ACCEPTED_SEGMENTS", "-i", ingress_if, "-s", vm_ip, "-o", egress_if, "-j", "ACCEPT") + run_iptables("-I", "CAPE_ACCEPTED_SEGMENTS", "-i", ingress_if, "-s", vm_ip, "-o", egress_if, "-j", "ACCEPT") + + +def nexthop_disable(vm_ip, ingress_if, egress_if, rt_table, priority): + """Mirror-delete the per-task state from nexthop_enable, then flush conntrack. The iptables + deletes loop until gone so any duplicate left by a retried bind is fully removed (no stale egress).""" + run(settings.ip, "rule", "del", "from", vm_ip, "lookup", rt_table, "priority", priority) + _iptables_delete_all("-t", "nat", "-D", "POSTROUTING", "-s", vm_ip, "-o", egress_if, "-j", "MASQUERADE") + _iptables_delete_all("-D", "CAPE_ACCEPTED_SEGMENTS", "-i", ingress_if, "-s", vm_ip, "-o", egress_if, "-j", "ACCEPT") + run("conntrack", "-D", "-s", vm_ip) + + +def nexthop_intra_exception_enable(vm_net, band_lo): + """Keep intra-vm_net traffic on the main table: `from vm_net to vm_net lookup main`. + + vm_net includes the host's own guest-bridge IP and sibling guests. A bound VM's per-task rule is + `from lookup ` and the gateway table holds only a default route, so without this + exception a bound VM's traffic to a vm_net address that is NOT the host-local bridge IP (a sibling + guest, or a ResultServer that is not the host itself) would be captured by the per-task rule and + SNAT'd out the gateway instead of staying on the guest network (codex). Host-local ResultServer + already resolves via the kernel priority-0 local table; this covers the non-host-local + + guest<->guest cases, for bound and unbound VMs alike. + + Installed at band_lo-1 -- JUST BELOW the per-task band (band_lo..band_hi) -- so it wins over a + bound VM's in-band per-task rule. It only matches `to vm_net`, so external egress is unaffected. + This is a CONNECTIVITY guarantee, installed whenever nexthop is enabled and independent of + fail_closed (which only governs the blackhole). Idempotent (del+add, no stacked duplicates).""" + intra_prio = str(int(band_lo) - 1) + run(settings.ip, "rule", "del", "from", vm_net, "to", vm_net, "lookup", "main", "priority", intra_prio) + run(settings.ip, "rule", "add", "from", vm_net, "to", vm_net, "lookup", "main", "priority", intra_prio) + + +def nexthop_fail_closed_enable(vm_net, fail_table, priority_low): + """Arm once at startup (only when fail_closed=yes): any guest-subnet source with no + higher-priority per-task rule is blackholed (dropped), never routed by main out the control-plane + NIC. `route replace` is idempotent; the rule is del+add'd so a restart does not stack duplicates. + The intra-subnet exception is installed separately by nexthop_intra_exception_enable (always), + since keeping host<->guest / guest<->guest traffic on main is a connectivity concern, not a + fail-closed one -- an unbound external flow still matches no per-task rule and hits this blackhole.""" + run(settings.ip, "route", "replace", "blackhole", "default", "table", fail_table) + run(settings.ip, "rule", "del", "from", vm_net, "lookup", fail_table, "priority", priority_low) + run(settings.ip, "rule", "add", "from", vm_net, "lookup", fail_table, "priority", priority_low) + + +def nexthop_teardown(gateway_tables, vm_net, fail_table, priority_low, band_lo, band_hi): + """Remove ALL nexthop policy-routing state (cleanup_rooter only sweeps iptables). + gateway_tables: comma-joined table ids. Idempotent; every step is a best-effort run().""" + for rt in [t for t in gateway_tables.split(",") if t]: + # NEVER flush a reserved/system routing table (gemini #14 HIGH). See RESERVED_RT_TABLES. + if rt in RESERVED_RT_TABLES: + log.error("nexthop_teardown refusing to flush reserved routing table %r (fix the [gwX] rt_table)", rt) + continue + run(settings.ip, "route", "flush", "table", rt) + run(settings.ip, "route", "del", "blackhole", "default", "table", fail_table) + run(settings.ip, "rule", "del", "from", vm_net, "lookup", fail_table, "priority", priority_low) + # intra-subnet exception is armed at band_lo-1 (just below the per-task band); mirror-delete it + run(settings.ip, "rule", "del", "from", vm_net, "to", vm_net, "lookup", "main", "priority", str(int(band_lo) - 1)) + lo, hi = int(band_lo), int(band_hi) + try: + vm_network = ipaddress.ip_network(vm_net, strict=False) + except ValueError: + vm_network = None + stdout, _ = run(settings.ip, "rule", "show") + for line in stdout.splitlines(): + head = line.split(":", 1)[0].strip() + if not (head.isdigit() and lo <= int(head) <= hi): + continue + # Only sweep OUR per-task rules -- `from ...` with vm_ip inside vm_net. Every nexthop + # per-task rule matches `from `, so anything in the 10000-10255 band whose source is + # NOT in vm_net is an unrelated host admin rule; leave it alone (codex). Fail-safe: if vm_net + # is unparseable or the source can't be confirmed inside it, skip rather than delete. + toks = line.split() + if vm_network is None or "from" not in toks: + continue + try: + src = toks[toks.index("from") + 1] + if ipaddress.ip_address(src.split("/")[0]) not in vm_network: + continue + except (ValueError, IndexError): + continue + # Delete by the FULL selector (from + priority), not by priority alone: iproute2 allows + # multiple rules at one priority, so `ip rule del priority N` could remove a host admin rule + # that happens to share this priority instead of ours. Matching on our confirmed vm_net source + # too targets exactly our per-task rule (codex). + run(settings.ip, "rule", "del", "from", src, "priority", head) + + +def nexthop_configure(tables_csv, vm_net, fail_table, prio_low, band_lo, band_hi): + """Called by the [gwX] loader to set the module globals that nexthop_teardown sweeps.""" + global GATEWAY_TABLES_CSV, NEXTHOP_VM_NET, NEXTHOP_FAIL_TABLE + global NEXTHOP_PRIORITY_LOW, NEXTHOP_BAND_LO, NEXTHOP_BAND_HI + GATEWAY_TABLES_CSV, NEXTHOP_VM_NET, NEXTHOP_FAIL_TABLE = tables_csv, vm_net, fail_table + NEXTHOP_PRIORITY_LOW, NEXTHOP_BAND_LO, NEXTHOP_BAND_HI = prio_low, band_lo, band_hi + + +def nexthop_teardown_if_configured(): + """SIGTERM helper: remove nexthop policy-routing state ONLY if the [gwX] loader configured it + this session (nexthop_configure sets GATEWAY_TABLES_CSV; it is "" on a node that never enabled + [nexthop]). Skipping otherwise honours the disabled=no-op contract -- an unconfigured node must + not mutate host routing on shutdown, in particular must not sweep the 10000-10255 priority band + (which could delete an unrelated host rule). Module-level + importable so the contract is + CI-guarded (the SIGTERM handler itself lives inside __main__ and is not testable).""" + if not GATEWAY_TABLES_CSV: + return + nexthop_teardown(GATEWAY_TABLES_CSV, NEXTHOP_VM_NET, NEXTHOP_FAIL_TABLE, + NEXTHOP_PRIORITY_LOW, NEXTHOP_BAND_LO, NEXTHOP_BAND_HI) + + def dns_forward(action, vm_ip, dns_ip, dns_port="53"): """Route DNS requests from the VM to a custom DNS on a separate network.""" run_iptables( @@ -1113,6 +1287,13 @@ def sslproxy_disable(interface, client, proxy_port, resultserver_port, rt_table= "libvirt_fwo_disable": libvirt_fwo_disable, "sslproxy_enable": sslproxy_enable, "sslproxy_disable": sslproxy_disable, + "nexthop_init": nexthop_init, + "nexthop_enable": nexthop_enable, + "nexthop_disable": nexthop_disable, + "nexthop_intra_exception_enable": nexthop_intra_exception_enable, + "nexthop_fail_closed_enable": nexthop_fail_closed_enable, + "nexthop_teardown": nexthop_teardown, + "nexthop_configure": nexthop_configure, } if __name__ == "__main__": @@ -1193,6 +1374,10 @@ def handle_sigterm(sig, f): server.shutdown(socket.SHUT_RDWR) server.close() cleanup_rooter() + # Remove nexthop policy-routing state (cleanup_rooter is iptables-only), but only when the + # [gwX] loader actually configured nexthop this session -- see nexthop_teardown_if_configured + # (extracted + CI-guarded so the disabled=no-op contract can't silently regress) (Copilot). + nexthop_teardown_if_configured() signal.signal(signal.SIGTERM, handle_sigterm) diff --git a/web/submission/views.py b/web/submission/views.py index 7888d98d4f4..3de4de7b49d 100644 --- a/web/submission/views.py +++ b/web/submission/views.py @@ -733,6 +733,24 @@ def index(request, task_id=None, resubmit_hash=None): {"name": v["name"], "description": v["description"], "interface": v["interface"], "type": "vpn"} for v in vpns.values() ] + # nexthop gateway pool: expose the configured [gwX] profiles (and the "nexthop" pool + # sentinel) as route options so GUI submissions can select the pool, mirroring vpns_data. + # Defensive: a malformed [nexthop] config must never 500 the submission page. + nexthop_enabled = False + gateways_data = [] + try: + if getattr(routing.nexthop, "enabled", False): + nexthop_enabled = True + for gw_name in str(getattr(routing.nexthop, "gateways", "") or "").split(","): + gw_name = gw_name.strip() + if not gw_name: + continue + gw = routing.get(gw_name) if hasattr(routing, gw_name) else None + desc = getattr(gw, "description", None) if gw is not None else None + gateways_data.append({"name": gw_name, "description": desc or gw_name}) + except Exception: + nexthop_enabled, gateways_data = False, [] + existent_tasks = {} if resubmit_hash: if web_conf.general.get("existent_tasks", False): @@ -752,6 +770,8 @@ def index(request, task_id=None, resubmit_hash=None): "vpns": vpns_data, "random_route": random_route, "socks5s": socks5s_data, + "gateways": gateways_data, + "nexthop_enabled": nexthop_enabled, "route": routing.routing.route, "internet": routing.routing.internet, "inetsim": routing.inetsim.enabled, diff --git a/web/templates/submission/index.html b/web/templates/submission/index.html index a25ee730f45..311f3c0c624 100644 --- a/web/templates/submission/index.html +++ b/web/templates/submission/index.html @@ -259,6 +259,14 @@
Advance {% endfor %} {% endif %} + {% if nexthop_enabled %} + + {% for gw in gateways %} + + {% endfor %} + {% endif %} From b8248919dc92013c2d434bc187de183db7e7f51b Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Wed, 27 May 2026 19:36:26 +0000 Subject: [PATCH 02/53] docs: processor concurrency redesign spec (pluggable pebble/prefork engines) Design for replacing the fork+long-lived-pool+nested-pool model that deadlocked cape-processor. Pluggable engine seam: current pebble path preserved as A/B control; single-threaded prefork supervisor (ephemeral per-task children, process-group timeout kill, os._exit) as the target. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...7-processor-concurrency-redesign-design.md | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md diff --git a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md new file mode 100644 index 00000000000..ec50f43be98 --- /dev/null +++ b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md @@ -0,0 +1,183 @@ +# Processor Concurrency Redesign — Pluggable Engines (Pebble + Prefork) + +**Date:** 2026-05-27 +**Component:** `utils/process.py` (CAPE report processor, systemd `cape-processor.service`) +**Branch:** `feature/processor-prefork-engine` (off `feature/guac-auth-evtx-snapshots`) +**Status:** Design — awaiting review before implementation planning + +--- + +## 1. Background & Problem + +`utils/process.py auto` continuously pulls `TASK_COMPLETED` analyses from the DB and runs +`process()` on each (processing modules → signatures → reporting), `-p20` in parallel, +with a `-pt 900` per-task timeout. It uses a `pebble.ProcessPool` (default **fork** start +method, `maxtasksperchild=7`). Each worker also lazily builds a **nested** +`pebble.ProcessPool` (`_EXTRACTOR_POOL`, 6 procs) for file extractors. + +On 2026-05-27 the processor wedged: all 20 worker slots stuck, scheduler parked in its +"pool full" sleep, log silent for hours, and **zero `Processing timeout` lines ever** +despite `-pt 900`. Diagnosis (py-spy + strace + log forensics) found **two independent +deadlocks**, both rooted in forking a process that owns threads and child processes: + +1. **Startup thread-soup fork hazard.** `lib/cuckoo/common/cleaners_utils.py` created + `resolver_pool = ThreadPool(50)` at *import* time; `process.py` imports that module, so + the parent held 50+ threads when pebble forked workers. Forked children inherited locks + held by now-dead threads. *(Mitigated: `resolver_pool` made lazy — main thread count + 73 → 20.)* + +2. **Worker-recycle exit deadlock (primary).** pebble's `worker_process` *returns normally* + when it hits `max_tasks`, so the worker runs `multiprocessing.util._exit_function`, which + `join()`s the worker's child processes — the nested `_EXTRACTOR_POOL` workers, which never + terminate → hang forever. Every worker deadlocks on its 7th-task recycle (observed: + exactly `140 completed ÷ 20 workers = 7`). *(Stopgap: `-mc 0` disables recycling so the + exit path is never reached; tradeoff is no per-worker memory reclaim.)* + +3. **Timeout enforcement gap.** pebble only times out *acknowledged* tasks + (`TaskManager.timeout()` requires `task.started`). A worker that hangs before/while + starting is immortal, and even when the timeout fires it kills one PID, not the worker's + descendant tree (extractors, external tools orphan to init). + +These are properties of the **fork + long-lived pool + pool-inside-a-worker** model, not +one-off bugs. This redesign replaces that model while keeping the old one runnable for A/B. + +## 2. Goals / Non-Goals + +**Goals** +- A processing model where a single hung/crashy task or extractor can never wedge the pool. +- A per-task timeout that **always** fires and fully cleans up the task's entire process tree. +- No fork-from-multithreaded-process hazard; no orphaned grandchildren; no immortal tasks. +- Keep the existing (pebble) model selectable so the new model can be A/B-tested against it + in production with a flag flip and measurable comparison. +- Per-task init cost stays near zero (preserve the warm-state benefit of long-lived workers). + +**Non-Goals** +- Changing what `process()` does (the processing/signature/reporting pipeline is unchanged). +- Changing the DB task lifecycle/statuses. +- Reworking `cape.service` or other CAPE components. + +## 3. Workload facts (measured, this deployment) +- Task duration: p50 22s, p90 52s, p99 107s, **max 129s** — none near the 900s limit. +- Throughput: a few tasks/min, bursty; ~160 tasks over a working day. +- Host: 16 CPU, 125 GB RAM; worker RSS ~1.8 GB. Memory is not the binding constraint. +- Implication: tasks are short and heavyweight; warm workers exist only to amortize + cold init (YARA compile ~3s + importing all processing/signature/reporting modules). + +## 4. Architecture — pluggable engines + +A clean seam splits "pull & track work" from "isolate & run one task." Both engines share +everything except worker isolation. + +**Shared (unchanged):** +- `process(task, report=True, auto=True, ...)` — the per-task work. +- DB poll for `TASK_COMPLETED`, concurrency accounting (`parallel`), status update helpers, + config (`-pt`, `-p`). + +**Selection:** new CLI flag `--engine {pebble,prefork}` (default `pebble` until prefork wins +the A/B; then flip the default). systemd `ExecStart` / drop-in carries the flag. Each engine +tags log lines with its name and emits per-task timing so A/B is measurable. + +``` + ┌───────────────────────────┐ + │ autoprocess(engine=...) │ DB poll, concurrency, status + └─────────────┬─────────────┘ + ┌────────────────┴────────────────┐ + ┌─────▼──────┐ ┌──────▼───────┐ + │ PebbleEngine│ (control/baseline)│ PreforkEngine│ (target) + └────────────┘ └──────────────┘ +``` + +### 4.1 `PebbleEngine` (preserved baseline) +Today's implementation, kept intact as the A/B control. Default `max_tasks=0` (recycling is +what deadlocks it; with 0 the exit path is never hit). It is explicitly *not* the long-term +target — it remains for comparison and rollback. + +### 4.2 `PreforkEngine` (target — Approach A) + +A single-threaded **supervisor** owning the full lifecycle. No library-internal pool, no +channels, no background manager threads. + +**(a) Warm base (once):** supervisor calls `init_database()` + `init_modules()` + YARA +compile, then **stays single-threaded** — no MongoClient, no thread pools in the supervisor. +Invariant enforced by `assert threading.active_count() == 1` immediately before each fork. + +**(b) Scheduler loop (supervisor main thread):** +- Poll DB for up to `parallel - in_flight` `TASK_COMPLETED` tasks. +- For each: `fork()` a task child (inherits warm modules/YARA copy-on-write ≈ zero init). +- Track `in_flight = {pid: (task_id, start_monotonic, pgid)}`. +- Reap finished children non-blocking (`os.waitpid(-1, WNOHANG)` / `pidfd`); on reap, apply + status (see (e)). +- **Timeout:** for each in-flight child where `now - start > processing_timeout`, + `os.killpg(pgid, SIGTERM)`, then `SIGKILL` after a short grace; mark task + `FAILED_PROCESSING`. Timer starts at *launch* — no "started" bookkeeping to be blind to. +- Emit a periodic heartbeat (`in_flight=N oldest=Xs`) so a stall is never silent. + +**(c) Task child:** +- `os.setsid()` → own session/process group, so the supervisor's `killpg` sweeps the entire + subtree (extractor sub-pool, external tools like sigma/suricata). +- Post-fork re-init: dispose+recreate SQLAlchemy engine connection, reset log handlers + (reuse current `init_worker` logic), establish its own Mongo client lazily. +- Run `process(task, ...)` **exactly once**. +- Terminate with **`os._exit(code)`** — never `return`/`sys.exit`, so the multiprocessing + atexit `join` (the exact thing that deadlocked pebble) never runs. + +**(d) Extractors — per-task process sub-pool (replaces `_EXTRACTOR_POOL`):** +- Created **inside** the task child, used for the file extractors, then **explicitly + `close()`+`join()`d** before the child exits. +- All sub-pool processes live in the task child's process group → swept by the supervisor's + `killpg` on timeout. The child's `os._exit()` is the backstop if teardown is imperfect. +- Each extractor keeps its existing per-tool subprocess timeout. +- **Open implementation detail (resolve with measurement, not guesswork):** how the sub-pool + is spawned. `forkserver` gives clean isolation but re-imports modules per task (cost on a + 22s task); `fork` from the *warm, still-single-threaded* task child inherits modules + cheaply but must be created before the child spawns any thread. Decision criterion: measure + per-task extractor-pool startup cost both ways on a representative task; pick the faster one + that preserves the os._exit + process-group-kill safety properties. + +**(e) Status (child sets, supervisor overrides):** +- The child runs `process()`, which performs the existing status writes (e.g. → reported). +- The supervisor overrides to `FAILED_PROCESSING` only on timeout (killpg) or abnormal child + exit (non-zero / signal). One proven status path; supervisor adds the failure cases. + +### 4.3 Why this removes all three hazards +- Fork-from-multithreaded: base is single-threaded (asserted) → safe forks. +- Immortal tasks: supervisor owns a launch-relative wall-clock timeout → always fires. +- Orphans / exit deadlock: `os._exit()` in the child (no atexit join) + `killpg` of the + process group → complete cleanup, nothing to hang on. + +## 5. A/B testing & metrics +- Run identical workload under each engine (flag flip + restart). Compare: + - throughput (tasks/min), per-task wall-clock (p50/p90/p99), + - wedge incidents (should be 0 for prefork), peak/sustained RSS, + - orphaned-process count after induced timeouts. +- Engine name + per-task timing logged for both so comparison is from the same source. + +## 6. Risks & mitigations +| Risk | Mitigation | +|---|---| +| Supervisor accidentally goes multithreaded before fork (Mongo/thread pool import side effect) | `assert active_count()==1` pre-fork; keep Mongo/cleaners out of supervisor; test it. | +| `init_modules()` import side effects differ under the supervisor vs `main()` | Supervisor performs the same init `main()` did; integration test runs a real task end-to-end. | +| Per-task extractor sub-pool startup cost erodes throughput | Measure both spawn strategies (§4.2d); choose empirically; this is the A/B's job to catch. | +| Post-fork SQLAlchemy/Mongo/logging state | Reuse the proven `init_worker` re-init; child establishes its own connections. | +| `-mc 0` stopgap on PebbleEngine grows memory until prefork ships | Monitor RSS; add a periodic clean systemd restart (`RuntimeMaxSec`) if it climbs. | + +## 7. Testing strategy +- **Timeout:** task that sleeps > `-pt` → asserted `killpg`'d at limit, task `FAILED_PROCESSING`, + no orphan processes remain, pool keeps running. +- **Crash:** task that raises / segfaults → supervisor detects abnormal exit, marks failed, + other tasks unaffected. +- **External-tool cleanup:** task whose extractor spawns a long subprocess → on timeout the + subprocess is killed (process-group sweep), zero orphans (PPID 1 check). +- **Concurrency:** N>parallel queued → max in-flight never exceeds `parallel`. +- **Fork-safety invariant:** assert supervisor single-threaded before each fork. +- **A/B parity:** same corpus through both engines → identical task outputs/statuses. + +## 8. Rollout +1. Land `PreforkEngine` behind `--engine prefork`, default remains `pebble`. +2. A/B in production via flag; collect §5 metrics. +3. Flip default to `prefork` once it wins; keep `pebble` selectable for one release. +4. Remove the `-mc 0` stopgap drop-in once `prefork` is the default. + +## 9. Out of scope / follow-ups +- Committing the already-applied `resolver_pool` lazy fix on the mainline. +- Broader processing-pipeline changes inside `process()`. From 455a035523c7885d682ea6cb895b46f27d1cdad4 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Wed, 27 May 2026 21:05:43 +0000 Subject: [PATCH 03/53] docs: add audited memory model & shared-state section to processor redesign spec Audit verdict: all heavy load-once structures (YARA, geoip, capa, pyattck, DECRYPTORS) are read-only after load; _CLAMAV_CACHE is per-worker and self-clearing. Nothing needs live mutable cross-worker sharing, so Approach A's warm-base COW fork preserves the "load once, share" model identically and reclaims C-lib leaks per task. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...7-processor-concurrency-redesign-design.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md index ec50f43be98..1d58fecdef2 100644 --- a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md +++ b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md @@ -145,6 +145,40 @@ Invariant enforced by `assert threading.active_count() == 1` immediately before - Orphans / exit deadlock: `os._exit()` in the child (no atexit join) + `killpg` of the process group → complete cleanup, nothing to hang on. +### 4.4 Memory model & shared state (audited 2026-05-27) + +The heavy load-once structures were audited for post-load mutation. All are **read-only +after load** or per-worker caches never shared across workers: + +| Structure | Loaded at | Post-load writes | Verdict | +|---|---|---|---| +| `File.yara_rules` (compiled YARA) | `init_yara()` (idempotent) | none — read at scan (`objects.py:593`) | read-only | +| `_MAXMINDDB_CLIENT` (geoip) | module-level, mtime-guarded | reload only if DB file changes | read-only (mmap) | +| capa `RuleSet` | import-time `get_rules()` | none — read at match | read-only | +| `mitre`/pyattck (`BASE_OBJECTS`, `RELATIONSHIP_MAP`) | `Attck()` at `abstracts.py:71` import | none — `techniques` recomputes views; caches nothing | read-only | +| `DECRYPTORS` (vbadeobf) | import-time decorator registry | none — read via `.get` | read-only | +| `_CLAMAV_CACHE` | during processing | per-worker, self-`clear()`s (`clamav.py:113`) | per-worker, not shared | + +**Nothing requires a live mutable instance shared across concurrent workers.** Decisive +proof: today's pebble model already shares these *only* via COW-fork (parent loads → workers +inherit, diverging on write). If anything needed cross-worker mutation it would already be +broken in production. + +Consequence for Approach A: the "load once, share" model is **preserved identically** — the +single-threaded warm base loads them where `init_modules`/import does today; per-task children +inherit by COW. Per-task `fork()` copies page tables, not the GBs of data. Short-lived +children also retain COW sharing better than 7-task workers, which gradually un-share heavy +pages via CPython refcount writes + cyclic GC. + +The leak concern is *better* served by A: a fresh process per task reclaims C-lib leaks every +task (recycle-every-task), strictly stronger than `maxtasksperchild=7`, without the +`_exit_function` deadlock. + +**Known per-task recompute cost:** pyattck `techniques` recomputes on each access (observed +burning CPU in `_get_relationship_objects`) — same in both models, but paid per-task under A +vs amortized over 7. Mitigation if material: pre-compute/cache it in the warm base so children +inherit the result. Treated as an A/B-measured optimization, not a correctness item. + ## 5. A/B testing & metrics - Run identical workload under each engine (flag flip + restart). Compare: - throughput (tasks/min), per-task wall-clock (p50/p90/p99), From 420b5b15a3a61d4073d2dd7080f5b67069845679 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Wed, 27 May 2026 21:35:23 +0000 Subject: [PATCH 04/53] =?UTF-8?q?docs:=20finalize=20processor=20redesign?= =?UTF-8?q?=20=E2=80=94=20fork-per-task=20lifecycle=20+=20measured=20lazy-?= =?UTF-8?q?load=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifecycle decided: fork-per-task, no supervisor->child dispatch channel (child runs one task then os._exit). Avoids rebuilding the inter-process channel whose deadlock motivated the redesign; max leak containment. recycle-after-N left as a documented future knob. Adds the load taxonomy (Category 1 init-time/COW-shared incl. yara-forge; Category 2 lazy first-use) and the measured Category-2 cost: loldrivers ~0.35s/+41MB, security_tools ~0, mmbot disabled, sigma via subprocess. ~1.6% per-task => no pre-warm registry needed. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...7-processor-concurrency-redesign-design.md | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md index 1d58fecdef2..90837a70690 100644 --- a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md +++ b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md @@ -121,6 +121,15 @@ Invariant enforced by `assert threading.active_count() == 1` immediately before - Terminate with **`os._exit(code)`** — never `return`/`sys.exit`, so the multiprocessing atexit `join` (the exact thing that deadlocked pebble) never runs. +**Lifecycle decision — fork-per-task, no dispatch channel.** Each child handles exactly +one task. The supervisor only forks, reaps, and times-out; there is **no supervisor→child +dispatch channel**, which deliberately eliminates the inter-process channel whose deadlock +motivated this redesign. Recycling a child across N>1 tasks would *require* such a channel; +the measured lazy-reload cost (§4.4) is ~0.35 s/task (~1.6%), so amortizing it is not worth +that complexity/failure surface. `--tasks-per-child N` is therefore a **documented future +knob, not built now** (YAGNI until a measured need appears). Fork-per-task also gives maximal +leak containment (C-lib leaks reclaimed every task), which was the primary concern. + **(d) Extractors — per-task process sub-pool (replaces `_EXTRACTOR_POOL`):** - Created **inside** the task child, used for the file extractors, then **explicitly `close()`+`join()`d** before the child exits. @@ -175,9 +184,30 @@ task (recycle-every-task), strictly stronger than `maxtasksperchild=7`, without `_exit_function` deadlock. **Known per-task recompute cost:** pyattck `techniques` recomputes on each access (observed -burning CPU in `_get_relationship_objects`) — same in both models, but paid per-task under A -vs amortized over 7. Mitigation if material: pre-compute/cache it in the warm base so children -inherit the result. Treated as an A/B-measured optimization, not a correctness item. +burning CPU in `_get_relationship_objects`) — same in both models. Mitigation if material: +pre-compute/cache it in the warm base so children inherit the result. Treated as an +A/B-measured optimization, not a correctness item. + +**Two load categories (the audit's real taxonomy):** +- *Category 1 — loaded at init/import:* YARA incl. yara-forge (`custom/yara/binaries/yara-rules-extended.yar`, + 17 MB, compiled by `init_yara()`), MITRE/pyattck (45 MB, `abstracts.py` import), capa (5.6 MB). + The warm base loads these **once** before forking → COW-shared to all children. Strictly + better than today, where `init_worker` compiles YARA **per worker** (1× vs 20×). +- *Category 2 — lazy, loaded on first use:* `_LOLD_CACHE` (loldrivers, **29 MB JSON**), + `_TOOLS_CACHE` (security_tools), etc. Reloaded per task under fork-per-task. + +**Measured Category-2 cost (2026-05-27, isolated fresh process):** +| Loader | time | RSS | +|---|---|---| +| `_load_loldrivers()` (29 MB JSON → 619 entries) | **0.35 s** | +41 MB | +| `_load_tools()` (security_tools) | ~0 s | +0 MB | +| maliciousmacrobot (ML model) | **disabled** | — | +| sigma rules (~28 MB) | n/a — loaded inside a shelled-out subprocess, process-group-swept, own timeout | + +So per-task lazy reload ≈ **0.35 s (~1.6% of a p50 22 s task)**; 41 MB × 20 children ≈ 0.8 GB +of 125 GB. Conclusion: **no pre-warm registry needed** — the cost it would save is negligible +and an exhaustive registry is the fragile thing we want to avoid (this audit already missed +loldrivers once). Fork-per-task absorbs it. ## 5. A/B testing & metrics - Run identical workload under each engine (flag flip + restart). Compare: From eaddf0553aa1b87c6be7ff75b19abe4f176b2d5f Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Wed, 27 May 2026 22:15:41 +0000 Subject: [PATCH 05/53] docs: implementation plan for processor prefork engine 11 TDD tasks across 3 phases: engine seam + PebbleEngine control (behavior- preserving), PreforkEngine supervisor (fork-per-task, os._exit, killpg timeout, single-threaded invariant), extractor sub-pool spike+fix, A/B runbook. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-05-27-processor-prefork-engine.md | 986 ++++++++++++++++++ 1 file changed, 986 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-27-processor-prefork-engine.md diff --git a/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md b/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md new file mode 100644 index 00000000000..ea958bf8e0f --- /dev/null +++ b/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md @@ -0,0 +1,986 @@ +# Processor Prefork Engine Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the deadlock-prone pebble worker pool in `utils/process.py` with a pluggable engine layer: the existing pebble loop preserved as an A/B control, and a new single-threaded **prefork supervisor** (fork-per-task children, `os._exit`, process-group timeout kill) as the reliability target. + +**Architecture:** A thin `autoprocess()` dispatches to a `ProcessingEngine` chosen by `--engine {pebble,prefork}`. Both engines share a `TaskSource` (DB poll + status writes) and a `run_task(task)` adapter. `PebbleEngine` wraps today's loop unchanged. `PreforkEngine` is a single-threaded supervisor that forks one child per task; the child `os.setsid()`s, runs the task, and `os._exit()`s; the supervisor reaps children and enforces a launch-relative wall-clock timeout via `os.killpg`. No supervisor↔child channel exists. + +**Tech Stack:** Python 3.12, SQLAlchemy (CAPE `Database`), pytest 7.2.2 (`poetry run pytest`, sqlite `db` fixture), `os.fork`/`waitpid`/`killpg`/`setsid`. + +**Spec:** `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` + +**Conventions for every task below:** +- Run tests with: `cd /opt/CAPEv2 && poetry run pytest -v` +- Commit only the files listed in the task (the repo working tree has many unrelated dirty files — never `git add -A`). +- Status constants import from `lib.cuckoo.core.data.task` (`TASK_COMPLETED`, `TASK_FAILED_PROCESSING`, `TASK_REPORTED`, `Task`). + +--- + +## Phase 1 — Engine seam (behavior-preserving) + +After Phase 1, behavior is unchanged: `--engine pebble` (the default) runs exactly today's loop, now behind the seam. + +### Task 1: `TaskSource` — DB polling + status writes + +**Files:** +- Create: `lib/cuckoo/core/processing_engine/__init__.py` (empty) +- Create: `lib/cuckoo/core/processing_engine/source.py` +- Test: `tests/test_processing_engine_source.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_processing_engine_source.py +import pytest +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING +from lib.cuckoo.core.processing_engine.source import TaskSource + + +def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): + t1 = db.add_path(temp_pe32) + t2 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) + db.set_status(t2, TASK_COMPLETED) + + src = TaskSource(db) + got = src.fetch(limit=10, exclude_ids={t2}) + ids = [t.id for t in got] + assert t1 in ids and t2 not in ids + + +def test_mark_failed_sets_status(db, temp_pe32): + t1 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) + + src = TaskSource(db) + src.mark_failed(t1) + assert db.view_task(t1).status == TASK_FAILED_PROCESSING +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_processing_engine_source.py -v` +Expected: FAIL with `ModuleNotFoundError: lib.cuckoo.core.processing_engine.source` + +- [ ] **Step 3: Write minimal implementation** + +```python +# lib/cuckoo/core/processing_engine/source.py +"""Shared task source for processing engines: pulls tasks needing processing +from the DB and writes terminal status. Engines differ in how they *run* tasks, +not in how they pull them.""" +import logging + +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, Task + +log = logging.getLogger(__name__) + + +class TaskSource: + def __init__(self, db, failed_processing=False): + self.db = db + self._status = TASK_FAILED_PROCESSING if failed_processing else TASK_COMPLETED + + def fetch(self, limit, exclude_ids): + """Return up to `limit` tasks needing processing, excluding `exclude_ids` + (in-flight). Tasks are expunged so they are safe to use after the txn.""" + if limit <= 0: + return [] + with self.db.session.begin(): + tasks = self.db.list_tasks(status=self._status, limit=limit, order_by=Task.completed_on.asc()) + self.db.session.expunge_all() + return [t for t in tasks if t.id not in exclude_ids] + + def mark_failed(self, task_id): + with self.db.session.begin(): + self.db.set_status(task_id, TASK_FAILED_PROCESSING) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_processing_engine_source.py -v` +Expected: PASS (2 passed) + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/__init__.py lib/cuckoo/core/processing_engine/source.py tests/test_processing_engine_source.py +git commit -m "feat(processor): add TaskSource for engine task polling/status" +``` + +--- + +### Task 2: `ProcessingEngine` base + `get_engine()` registry + +**Files:** +- Create: `lib/cuckoo/core/processing_engine/base.py` +- Modify: `lib/cuckoo/core/processing_engine/__init__.py` +- Test: `tests/test_processing_engine_registry.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_processing_engine_registry.py +import pytest +from lib.cuckoo.core.processing_engine import get_engine +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +def _noop(*a, **k): + pass + + +def test_get_engine_returns_requested_class(): + eng = get_engine("pebble", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) + assert isinstance(eng, ProcessingEngine) + + +def test_get_engine_unknown_raises(): + with pytest.raises(ValueError): + get_engine("nope", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_processing_engine_registry.py -v` +Expected: FAIL with `ImportError`/`cannot import name 'get_engine'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# lib/cuckoo/core/processing_engine/base.py +"""Abstract processing engine: drives the per-task lifecycle for autoprocess. +Concrete engines (pebble, prefork) differ only in worker isolation.""" + + +class ProcessingEngine: + def __init__(self, task_fn, worker_init, source, parallel, timeout): + self.task_fn = task_fn # task_fn(task) -> None: runs ONE task to completion + self.worker_init = worker_init # called in worker context (pool init / post-fork) + self.source = source # TaskSource + self.parallel = parallel + self.timeout = timeout + + def run(self): + raise NotImplementedError +``` + +```python +# lib/cuckoo/core/processing_engine/__init__.py +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +def get_engine(name, **kwargs): + """Construct the named engine. Imports are local so that selecting one engine + never imports the other's dependencies.""" + if name == "pebble": + from lib.cuckoo.core.processing_engine.pebble import PebbleEngine + return PebbleEngine(**kwargs) + if name == "prefork": + from lib.cuckoo.core.processing_engine.prefork import PreforkEngine + return PreforkEngine(**kwargs) + raise ValueError("unknown processing engine: %r" % name) + + +__all__ = ["ProcessingEngine", "get_engine"] +``` + +NOTE: this task references `pebble.PebbleEngine` (Task 4) and `prefork.PreforkEngine` (Task 5). The registry test only exercises the `pebble` branch, which Task 4 makes importable. Run Task 2's test again after Task 4. For now, add a temporary stub so Task 2 passes in isolation: + +```python +# lib/cuckoo/core/processing_engine/pebble.py (TEMPORARY STUB — replaced in Task 4) +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +class PebbleEngine(ProcessingEngine): + def run(self): + raise NotImplementedError +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_processing_engine_registry.py -v` +Expected: PASS (2 passed) + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/base.py lib/cuckoo/core/processing_engine/__init__.py lib/cuckoo/core/processing_engine/pebble.py tests/test_processing_engine_registry.py +git commit -m "feat(processor): add ProcessingEngine base + get_engine registry" +``` + +--- + +### Task 3: Extract `run_task(task)` adapter in `process.py` + +The per-task argument building currently inlined in `autoprocess()` (`utils/process.py` ~lines 463–470: sample-hash lookup + building `args`/`kwargs` for `process()`) becomes a standalone `run_task(task)` so both engines share it. + +**Files:** +- Modify: `utils/process.py` (add `run_task`; reference existing `process()` at line 107 and `db`) +- Test: `tests/test_run_task_adapter.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_run_task_adapter.py +import types +import utils.process as proc + + +def test_run_task_calls_process_with_task_and_auto(monkeypatch, db, temp_pe32): + captured = {} + + def fake_process(target=None, sample_sha256=None, task=None, report=False, auto=False, **kw): + captured["task_id"] = task.id + captured["auto"] = auto + captured["report"] = report + + monkeypatch.setattr(proc, "process", fake_process) + monkeypatch.setattr(proc, "db", db) + + tid = db.add_path(temp_pe32) + task = db.view_task(tid) + proc.run_task(task) + + assert captured["task_id"] == tid + assert captured["auto"] is True and captured["report"] is True +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_run_task_adapter.py -v` +Expected: FAIL with `AttributeError: module 'utils.process' has no attribute 'run_task'` + +- [ ] **Step 3: Write minimal implementation** + +Add to `utils/process.py` (near `process()`), preserving the existing sample-hash logic from `autoprocess`: + +```python +def run_task(task, memory_debugging=False, debug=False): + """Run exactly one completed task to completion (processing -> report). + Extracted from autoprocess so every engine shares identical per-task setup.""" + sample_hash = "" + if task.category != "url": + with db.session.begin(): + sample = db.view_sample(task.sample_id) + if sample: + sample_hash = sample.sha256 + process( + task.target, + sample_hash, + report=True, + auto=True, + task=task, + memory_debugging=memory_debugging, + debug=debug, + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_run_task_adapter.py -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add utils/process.py tests/test_run_task_adapter.py +git commit -m "refactor(processor): extract run_task(task) adapter shared by engines" +``` + +--- + +### Task 4: `PebbleEngine` (relocate today's loop) + wire `autoprocess` + `--engine` flag + +Move the current pebble pool loop (`utils/process.py` `autoprocess` body, ~lines 430–508, plus `processing_finished`, ~lines 376–407) into `PebbleEngine`, calling injected `task_fn`/`worker_init`/`source` instead of the inline DB/process code. Behavior must be identical to today (including `-mc 0` default). + +**Files:** +- Modify: `lib/cuckoo/core/processing_engine/pebble.py` (replace the Task 2 stub) +- Modify: `utils/process.py` (`autoprocess()` → build `TaskSource` + `get_engine`; add `--engine` arg; keep `init_worker`, `process`, `run_task`) +- Test: `tests/test_pebble_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_pebble_engine.py +import threading +import time +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_REPORTED +from lib.cuckoo.core.processing_engine.pebble import PebbleEngine +from lib.cuckoo.core.processing_engine.source import TaskSource + + +def test_pebble_engine_processes_one_task(db, temp_pe32, monkeypatch): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + ran = threading.Event() + + def fake_task_fn(task): + with db.session.begin(): + db.set_status(task.id, TASK_REPORTED) + ran.set() + + eng = PebbleEngine(task_fn=fake_task_fn, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30, max_count=1) + eng.run() + assert ran.wait(timeout=20) + assert db.view_task(tid).status == TASK_REPORTED +``` + +NOTE: `max_count` lets the loop exit after N scheduled tasks for tests (mirror the existing `cfg.cuckoo.max_analysis_count` exit; default 0 = run forever in production). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_pebble_engine.py -v` +Expected: FAIL with `NotImplementedError` (Task 2 stub) or `TypeError` (no `max_count`). + +- [ ] **Step 3: Write minimal implementation** + +```python +# lib/cuckoo/core/processing_engine/pebble.py +"""Pebble-pool engine — preserved as the A/B control. Mirrors the historical +autoprocess loop, parameterized behind the engine seam. max_tasks defaults to 0 +(no recycle): worker recycling deadlocks in multiprocessing _exit_function while +joining the nested extractor pool — see the redesign spec.""" +import logging +import time + +import pebble + +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + +log = logging.getLogger(__name__) + + +class PebbleEngine(ProcessingEngine): + def __init__(self, task_fn, worker_init, source, parallel, timeout, + max_tasks=0, max_count=0): + super().__init__(task_fn, worker_init, source, parallel, timeout) + self.max_tasks = max_tasks + self.max_count = max_count + self._pending = {} # future -> task_id + + def _done(self, future): + task_id = self._pending.pop(future, None) + try: + future.result() + except Exception: + log.exception("[%s] pebble: task failed", task_id) + if task_id is not None: + self.source.mark_failed(task_id) + + def run(self): + count = 0 + with pebble.ProcessPool(max_workers=self.parallel, max_tasks=self.max_tasks, + initializer=self.worker_init) as pool: + while not self.max_count or count < self.max_count: + if len(self._pending) >= self.parallel: + time.sleep(1) + continue + tasks = self.source.fetch(limit=self.parallel, + exclude_ids=set(self._pending.values())) + added = False + for task in tasks: + future = pool.schedule(self.task_fn, args=(task,), timeout=self.timeout) + self._pending[future] = task.id + future.add_done_callback(self._done) + count += 1 + added = True + break + if not added and not self.max_count: + time.sleep(1) + if not added and self.max_count: + break + # drain + while self._pending: + time.sleep(0.2) +``` + +In `utils/process.py`, replace the body of `autoprocess()` with engine dispatch (keep `memory_limit()`/`free_space_monitor` setup as today), and add the CLI flag: + +```python +# autoprocess() body (replace the pebble loop): +from lib.cuckoo.core.processing_engine import get_engine +from lib.cuckoo.core.processing_engine.source import TaskSource + +def autoprocess(parallel=1, failed_processing=False, maxtasksperchild=0, + memory_debugging=False, processing_timeout=300, debug=False, + disable_memory_limit=False, engine="pebble"): + if not disable_memory_limit: + memory_limit() + log.info("Processing analysis data (engine=%s)", engine) + source = TaskSource(db, failed_processing=failed_processing) + eng = get_engine( + engine, + task_fn=lambda task: run_task(task, memory_debugging=memory_debugging, debug=debug), + worker_init=init_worker, + source=source, + parallel=parallel, + timeout=processing_timeout, + ) + if engine == "pebble": + eng.max_tasks = maxtasksperchild + eng.run() +``` + +```python +# argparse (near the other autoprocess args): +parser.add_argument("--engine", choices=["pebble", "prefork"], default="pebble", + help="Processing engine: pebble (default, A/B control) or prefork.") +# main() autoprocess(...) call: add engine=args.engine +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_pebble_engine.py tests/test_processing_engine_registry.py -v` +Expected: PASS (3 passed) + +- [ ] **Step 5: Smoke-test the real CLI (no behavior change)** + +Run: `poetry run python utils/process.py --help` — Expected: shows `--engine {pebble,prefork}`. +Run (brief, then Ctrl-C): `poetry run python utils/process.py -p2 auto -pt 900 --engine pebble` — Expected: logs `Processing analysis data (engine=pebble)` and processes as before. + +- [ ] **Step 6: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/pebble.py utils/process.py tests/test_pebble_engine.py +git commit -m "feat(processor): PebbleEngine behind engine seam + --engine flag (default pebble)" +``` + +--- + +## Phase 2 — PreforkEngine + +A single-threaded supervisor. All tests inject a fake `task_fn` (sleep/crash/normal) so the real processing pipeline is not needed. + +### Task 5: Supervisor scaffolding + concurrency cap + single-threaded invariant + +**Files:** +- Create: `lib/cuckoo/core/processing_engine/prefork.py` (replace any stub) +- Test: `tests/test_prefork_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_prefork_engine.py +import os +import threading +import time + +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, TASK_REPORTED +from lib.cuckoo.core.processing_engine.prefork import PreforkEngine +from lib.cuckoo.core.processing_engine.source import TaskSource + + +def test_single_threaded_invariant_raises_when_extra_thread(db): + eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30) + stop = threading.Event() + t = threading.Thread(target=stop.wait) + t.start() + try: + import pytest + with pytest.raises(RuntimeError, match="single-threaded"): + eng._assert_single_threaded() + finally: + stop.set() + t.join() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_prefork_engine.py -v` +Expected: FAIL with `ModuleNotFoundError`/`AttributeError`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# lib/cuckoo/core/processing_engine/prefork.py +"""Single-threaded prefork supervisor: forks one child per task, each child runs +exactly one task then os._exit(); the supervisor reaps children and enforces a +launch-relative wall-clock timeout via process-group kill. No supervisor<->child +channel exists. See docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md.""" +import logging +import os +import signal +import threading +import time + +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + +log = logging.getLogger(__name__) + + +class _Child: + __slots__ = ("task_id", "pid", "start", "pgid", "timed_out", "kill_deadline") + + def __init__(self, task_id, pid, start, pgid): + self.task_id = task_id + self.pid = pid + self.start = start + self.pgid = pgid + self.timed_out = False + self.kill_deadline = None + + +class PreforkEngine(ProcessingEngine): + def __init__(self, task_fn, worker_init, source, parallel, timeout, + heartbeat_interval=30, term_grace=5, max_count=0, poll_interval=0.2): + super().__init__(task_fn, worker_init, source, parallel, timeout) + self.heartbeat_interval = heartbeat_interval + self.term_grace = term_grace + self.max_count = max_count + self.poll_interval = poll_interval + self._inflight = {} # pid -> _Child + + def _assert_single_threaded(self): + n = threading.active_count() + if n != 1: + raise RuntimeError( + "prefork supervisor must be single-threaded before fork; " + "active_count=%d (a thread/Mongo client/pool leaked into the supervisor)" % n) + + def _inflight_task_ids(self): + return {c.task_id for c in self._inflight.values()} + + def _heartbeat(self): + if self._inflight: + oldest = max(time.monotonic() - c.start for c in self._inflight.values()) + else: + oldest = 0.0 + log.info("prefork heartbeat: in_flight=%d oldest=%.0fs", len(self._inflight), oldest) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_prefork_engine.py -v` +Expected: PASS (1 passed) + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py +git commit -m "feat(processor): PreforkEngine scaffolding + single-threaded invariant" +``` + +--- + +### Task 6: Fork-per-task execution + reap + status (child sets / supervisor overrides) + +**Files:** +- Modify: `lib/cuckoo/core/processing_engine/prefork.py` +- Test: `tests/test_prefork_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/test_prefork_engine.py + +def test_normal_task_runs_and_status_set_by_child(db, temp_pe32): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + # child runs this; it must set status itself (child sets, supervisor overrides only on failure) + def task_fn(task): + # fresh DB handle in child after fork + from lib.cuckoo.core.database import Database + Database().set_status(task.id, TASK_REPORTED) + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30, max_count=1) + eng.run() + assert db.view_task(tid).status == TASK_REPORTED + + +def test_crashing_task_marked_failed_by_supervisor(db, temp_pe32): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + def task_fn(task): + os._exit(3) # simulate abnormal exit / crash + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30, max_count=1) + eng.run() + assert db.view_task(tid).status == TASK_FAILED_PROCESSING +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_prefork_engine.py -v` +Expected: FAIL (`run`/`_launch`/`_reap` not implemented). + +- [ ] **Step 3: Write minimal implementation** + +```python +# add methods to PreforkEngine + + def _child_main(self, task): + os.setsid() # own session/process group; supervisor killpg sweeps the subtree + try: + self.worker_init() + self.task_fn(task) + return 0 + except BaseException: + log.exception("prefork child: task %s crashed", getattr(task, "id", "?")) + return 1 + + def _launch(self, task): + self._assert_single_threaded() + pid = os.fork() + if pid == 0: + code = 1 + try: + code = self._child_main(task) + finally: + os._exit(code) # NEVER return: skip multiprocessing atexit join + self._inflight[pid] = _Child(task_id=task.id, pid=pid, start=time.monotonic(), pgid=pid) + log.debug("prefork: launched task %d as pid %d", task.id, pid) + + def _reap(self): + while True: + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return + if pid == 0: + return + child = self._inflight.pop(pid, None) + if child is None: + continue + if child.timed_out: + continue # already marked failed during timeout enforcement + ok = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + if not ok: + log.warning("prefork: task %d (pid %d) abnormal exit status=%d -> FAILED_PROCESSING", + child.task_id, pid, status) + self.source.mark_failed(child.task_id) + + def run(self): + count = 0 + last_hb = 0.0 + while True: + self._reap() + if self.max_count and count >= self.max_count and not self._inflight: + return + free = self.parallel - len(self._inflight) + launchable = free if not self.max_count else min(free, self.max_count - count) + if launchable > 0: + tasks = self.source.fetch(limit=launchable, exclude_ids=self._inflight_task_ids()) + for task in tasks[:launchable]: + self._launch(task) + count += 1 + now = time.monotonic() + if now - last_hb >= self.heartbeat_interval: + self._heartbeat() + last_hb = now + time.sleep(self.poll_interval) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_prefork_engine.py -v` +Expected: PASS (3 passed) + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py +git commit -m "feat(processor): prefork fork-per-task execution + reap + failure status" +``` + +--- + +### Task 7: Wall-clock timeout via `killpg` + no-orphan cleanup + +**Files:** +- Modify: `lib/cuckoo/core/processing_engine/prefork.py` +- Test: `tests/test_prefork_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/test_prefork_engine.py + +def test_timeout_kills_process_group_no_orphans(db, temp_pe32): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + marker = "/tmp/prefork_orphan_%d" % os.getpid() + + def task_fn(task): + # spawn a grandchild that would outlive the worker, then hang + import subprocess, sys + subprocess.Popen([sys.executable, "-c", + "import time,os;open(%r,'w').close();time.sleep(120)" % marker]) + time.sleep(120) + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, source=TaskSource(db), + parallel=1, timeout=1, term_grace=1, max_count=1, poll_interval=0.1) + if os.path.exists(marker): + os.unlink(marker) + eng.run() + + assert db.view_task(tid).status == TASK_FAILED_PROCESSING + # grandchild must have been swept by killpg (give it a moment) + time.sleep(2) + # nothing holding the marker's process group alive: assert no python sleeping on it + # (best-effort: the grandchild process should be gone) + import subprocess + out = subprocess.run(["pgrep", "-f", marker], capture_output=True, text=True) + assert out.stdout.strip() == "", "orphaned grandchild survived killpg" + if os.path.exists(marker): + os.unlink(marker) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_prefork_engine.py::test_timeout_kills_process_group_no_orphans -v` +Expected: FAIL (timeout enforcement not implemented → test hangs to its own safety or task not failed). If it hangs, that confirms missing enforcement; add enforcement in Step 3. + +- [ ] **Step 3: Write minimal implementation** + +```python +# add to PreforkEngine + + def _enforce_timeouts(self): + now = time.monotonic() + for child in list(self._inflight.values()): + if child.timed_out or now - child.start <= self.timeout: + continue + log.error("prefork: task %d (pid %d) exceeded %ds -> killpg", + child.task_id, child.pid, self.timeout) + child.timed_out = True + self.source.mark_failed(child.task_id) + try: + os.killpg(child.pgid, signal.SIGTERM) + except ProcessLookupError: + continue + child.kill_deadline = now + self.term_grace + + def _escalate_kills(self): + now = time.monotonic() + for child in list(self._inflight.values()): + if child.timed_out and child.kill_deadline and now > child.kill_deadline: + try: + os.killpg(child.pgid, signal.SIGKILL) + except ProcessLookupError: + pass + child.kill_deadline = None +``` + +Wire both into `run()`'s loop, immediately after `self._reap()`: + +```python + self._reap() + self._enforce_timeouts() + self._escalate_kills() +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_prefork_engine.py -v` +Expected: PASS (4 passed) + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py +git commit -m "feat(processor): prefork wall-clock timeout via killpg + grace escalation" +``` + +--- + +### Task 8: Post-fork child reinit + wire `--engine prefork` + +`init_worker()` today is the pebble pool initializer (disposes the SQLAlchemy engine post-fork, resets log handlers, pre-compiles YARA). For prefork it runs in `_child_main` via the injected `worker_init`. Confirm it is fork-safe to call in the child (it already does `db.engine.dispose(close=False)` and resets handlers without lock-acquiring calls). + +**Files:** +- Modify: `utils/process.py` (`autoprocess` already passes `worker_init=init_worker` and `engine=args.engine` from Task 4 — verify prefork path constructs cleanly) +- Test: `tests/test_prefork_engine.py` + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/test_prefork_engine.py + +def test_worker_init_called_in_child(db, temp_pe32, tmp_path): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + flag = str(tmp_path / "winit_ran") + + def worker_init(): + open(flag, "w").close() + + def task_fn(task): + assert os.path.exists(flag), "worker_init must run before task_fn in child" + from lib.cuckoo.core.database import Database + Database().set_status(task.id, TASK_REPORTED) + + eng = PreforkEngine(task_fn=task_fn, worker_init=worker_init, source=TaskSource(db), + parallel=1, timeout=30, max_count=1) + eng.run() + assert db.view_task(tid).status == TASK_REPORTED +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_prefork_engine.py::test_worker_init_called_in_child -v` +Expected: PASS already if Task 6 calls `self.worker_init()` before `self.task_fn(task)` in `_child_main`. If it FAILS, fix ordering in `_child_main`. + +- [ ] **Step 3: Verify ordering (no code change expected)** + +Confirm `_child_main` calls `self.worker_init()` then `self.task_fn(task)`. If not, reorder. + +- [ ] **Step 4: Smoke-test the real prefork CLI against the live DB (read-only-ish)** + +Run (brief, then Ctrl-C): `poetry run python utils/process.py -p2 auto -pt 900 --engine prefork` +Expected: logs `Processing analysis data (engine=prefork)`, a `prefork heartbeat: in_flight=...` line, and tasks transition to reported. Verify no `_exit_function`/`join` stacks via `py-spy dump` on a child (should show the task running, not multiprocessing atexit). + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py +git commit -m "feat(processor): verify post-fork worker_init ordering for prefork children" +``` + +--- + +## Phase 3 — Extractor sub-pool + finalize + +### Task 9: SPIKE — resolve §4.2(d) extractor sub-pool spawn strategy by measurement + +The nested extractor pool in `lib/cuckoo/common/integrations/file_extra_info.py` (`_EXTRACTOR_POOL`) must, under prefork, be: created inside the task child, explicitly `close()`+`join()`d before the child `os._exit()`s, and entirely within the child's process group (so `killpg` sweeps it). The open question is the spawn context: `forkserver` (clean, re-imports per task) vs `fork` from the warm single-threaded child (inherits modules cheaply, must be created before the child spawns threads). + +**Files:** +- Create: `docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md` (findings only; no production code) + +- [ ] **Step 1: Measure both strategies** + +Write a throwaway script that, inside a forked child, builds (a) `pebble.ProcessPool(context=multiprocessing.get_context("forkserver"))` and (b) `...get_context("fork")`, schedules a trivial extractor-like function across `max_workers=6`, and records: pool-create+first-result latency, and whether `killpg` of the child's group leaves any survivors (`pgrep`). Run on a representative extracted-file workload. + +- [ ] **Step 2: Record the decision** + +Write findings + the chosen context to the notes file, with the latency numbers and the orphan check. Decision criterion: choose the faster strategy that leaves zero survivors after `killpg` and runs cleanly with `os._exit()`. + +- [ ] **Step 3: Commit** + +```bash +git add docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md +git commit -m "docs(processor): spike results for extractor sub-pool spawn strategy" +``` + +--- + +### Task 10: Make `_EXTRACTOR_POOL` prefork-safe (explicit teardown + process-group membership) + +**Files:** +- Modify: `lib/cuckoo/common/integrations/file_extra_info.py` (`_get_extractor_pool`/`generic_file_extractors`, ~lines 436–520) +- Test: `tests/test_extractor_pool_teardown.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_extractor_pool_teardown.py +import os +import time +import multiprocessing as mp + + +def _busy(_): + time.sleep(60) + + +def test_extractor_pool_is_swept_by_killpg(tmp_path): + """A child that creates the extractor pool and is killpg'd must leave no survivors.""" + marker = str(tmp_path / "epool_child") + + def child(): + os.setsid() + open(marker, "w").close() + import pebble + ctx = mp.get_context("forkserver") # or the strategy chosen in Task 9 + pool = pebble.ProcessPool(max_workers=2, context=ctx) + for _ in range(2): + pool.schedule(_busy, args=(1,), timeout=60) + time.sleep(60) + + pid = os.fork() + if pid == 0: + try: + child() + finally: + os._exit(0) + while not os.path.exists(marker): + time.sleep(0.05) + time.sleep(1) + os.killpg(pid, 15) + time.sleep(2) + import subprocess + out = subprocess.run(["pgrep", "-g", str(pid)], capture_output=True, text=True) + assert out.stdout.strip() == "", "extractor sub-pool survived killpg" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `poetry run pytest tests/test_extractor_pool_teardown.py -v` +Expected: PASS or FAIL depending on context choice; if FAIL (survivors), the chosen context/teardown is wrong — adjust per Task 9 findings. + +- [ ] **Step 3: Implement teardown + per-call lifecycle** + +In `generic_file_extractors`, replace the process-wide cached `_EXTRACTOR_POOL` with a pool created and torn down per call (under prefork the child is ephemeral, so per-task lifetime is correct), using the Task-9 context, and `pool.close(); pool.join()` in a `finally`. Keep the per-future timeouts. + +```python +def generic_file_extractors(file, destination_folder, data_dictionary, options, results, duplicated, tests=False): + ... + import multiprocessing + ctx = multiprocessing.get_context("forkserver") # per Task 9 decision + pool = pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) + try: + ... # schedule extractors, collect futures, gather results (unchanged logic) + finally: + pool.close() + pool.join() +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `poetry run pytest tests/test_extractor_pool_teardown.py -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add lib/cuckoo/common/integrations/file_extra_info.py tests/test_extractor_pool_teardown.py +git commit -m "fix(extractors): per-call pool with explicit teardown, swept by process-group kill" +``` + +--- + +### Task 11: Docs + A/B systemd drop-in + stopgap retirement note + +**Files:** +- Create: `docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md` +- Modify: `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` (mark Phase status) + +- [ ] **Step 1: Write the A/B runbook** + +Document: how to flip engines (systemd drop-in `ExecStart ... --engine prefork`), the metrics to compare (throughput tasks/min, wedge incidents, peak RSS, orphan count after induced timeouts via `pgrep`), how to roll back (remove drop-in line), and the condition to retire the `-mc 0` stopgap (once `prefork` is the default). + +- [ ] **Step 2: Update spec status + commit** + +```bash +git add docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md +git commit -m "docs(processor): A/B runbook + mark redesign implementation status" +``` + +--- + +## Self-review notes (author) + +- **Spec coverage:** pluggable seam (T1–T4), PebbleEngine control (T4), Prefork supervisor incl. fork-per-task/os._exit/killpg/single-threaded invariant/status (T5–T8), warm-base inheritance (relies on `main()` init before `run()`; T8 smoke-test), extractor sub-pool §4.2(d) (T9–T10), A/B + metrics + heartbeat (T6 heartbeat, T11 runbook). Memory model is design-level (no code). +- **Deferred-by-design:** §4.2(d) spawn context is resolved by the T9 spike before T10 codes it — intentional, not a placeholder. +- **Type consistency:** `ProcessingEngine(task_fn, worker_init, source, parallel, timeout)` used uniformly; `task_fn(task)`, `source.fetch(limit, exclude_ids)`, `source.mark_failed(task_id)` consistent across T1/T4/T5–T8. +- **Risk:** the live `/opt/CAPEv2` tree has many dirty files — every commit step lists explicit paths; never `git add -A`. From 91e32f756a23048eff097c892a4187f30590b81d Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 13:45:46 +0000 Subject: [PATCH 06/53] docs(plan): clarify A/B runs on production PostgreSQL; sqlite is test-only Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/superpowers/plans/2026-05-27-processor-prefork-engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md b/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md index ea958bf8e0f..8d63c652769 100644 --- a/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md +++ b/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md @@ -967,7 +967,7 @@ git commit -m "fix(extractors): per-call pool with explicit teardown, swept by p - [ ] **Step 1: Write the A/B runbook** -Document: how to flip engines (systemd drop-in `ExecStart ... --engine prefork`), the metrics to compare (throughput tasks/min, wedge incidents, peak RSS, orphan count after induced timeouts via `pgrep`), how to roll back (remove drop-in line), and the condition to retire the `-mc 0` stopgap (once `prefork` is the default). +Document: how to flip engines (systemd drop-in `ExecStart ... --engine prefork`), the metrics to compare (throughput tasks/min, wedge incidents, peak RSS, orphan count after induced timeouts via `pgrep`), how to roll back (remove drop-in line), and the condition to retire the `-mc 0` stopgap (once `prefork` is the default). **State explicitly that the A/B runs against the production PostgreSQL database — both engines share the same module-level `Database()` (configured `postgresql://...`); the `sqlite://` `db` fixture in `tests/conftest.py` is a unit-test harness only and has no production path.** - [ ] **Step 2: Update spec status + commit** From 4afafc8f11bfb466645524f5bff32147e17dd838 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 14:53:24 +0000 Subject: [PATCH 07/53] feat(processor): add TaskSource for engine task polling/status Co-Authored-By: Claude Sonnet 4.6 --- lib/cuckoo/core/processing_engine/__init__.py | 0 lib/cuckoo/core/processing_engine/source.py | 28 +++++++++++++++++++ tests/test_processing_engine_source.py | 24 ++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 lib/cuckoo/core/processing_engine/__init__.py create mode 100644 lib/cuckoo/core/processing_engine/source.py create mode 100644 tests/test_processing_engine_source.py diff --git a/lib/cuckoo/core/processing_engine/__init__.py b/lib/cuckoo/core/processing_engine/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/cuckoo/core/processing_engine/source.py b/lib/cuckoo/core/processing_engine/source.py new file mode 100644 index 00000000000..f49c9871701 --- /dev/null +++ b/lib/cuckoo/core/processing_engine/source.py @@ -0,0 +1,28 @@ +"""Shared task source for processing engines: pulls tasks needing processing +from the DB and writes terminal status. Engines differ in how they *run* tasks, +not in how they pull them.""" +import logging + +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, Task + +log = logging.getLogger(__name__) + + +class TaskSource: + def __init__(self, db, failed_processing=False): + self.db = db + self._status = TASK_FAILED_PROCESSING if failed_processing else TASK_COMPLETED + + def fetch(self, limit, exclude_ids): + """Return up to `limit` tasks needing processing, excluding `exclude_ids` + (in-flight). Tasks are expunged so they are safe to use after the txn.""" + if limit <= 0: + return [] + with self.db.session.begin_nested(): + tasks = self.db.list_tasks(status=self._status, limit=limit, order_by=Task.completed_on.asc()) + self.db.session.expunge_all() + return [t for t in tasks if t.id not in exclude_ids] + + def mark_failed(self, task_id): + with self.db.session.begin_nested(): + self.db.set_status(task_id, TASK_FAILED_PROCESSING) diff --git a/tests/test_processing_engine_source.py b/tests/test_processing_engine_source.py new file mode 100644 index 00000000000..89d9e9082d2 --- /dev/null +++ b/tests/test_processing_engine_source.py @@ -0,0 +1,24 @@ +import pytest +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING +from lib.cuckoo.core.processing_engine.source import TaskSource + + +def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): + t1 = db.add_path(temp_pe32) + t2 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) + db.set_status(t2, TASK_COMPLETED) + + src = TaskSource(db) + got = src.fetch(limit=10, exclude_ids={t2}) + ids = [t.id for t in got] + assert t1 in ids and t2 not in ids + + +def test_mark_failed_sets_status(db, temp_pe32): + t1 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) + + src = TaskSource(db) + src.mark_failed(t1) + assert db.view_task(t1).status == TASK_FAILED_PROCESSING From a14a5d4883ff976037b0dc2b15e17ab2c5e841d2 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 14:57:04 +0000 Subject: [PATCH 08/53] fix(processor): revert TaskSource to session.begin(); fix test setup to match production txn discipline --- lib/cuckoo/core/processing_engine/source.py | 4 ++-- tests/test_processing_engine_source.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/cuckoo/core/processing_engine/source.py b/lib/cuckoo/core/processing_engine/source.py index f49c9871701..18ba489960f 100644 --- a/lib/cuckoo/core/processing_engine/source.py +++ b/lib/cuckoo/core/processing_engine/source.py @@ -18,11 +18,11 @@ def fetch(self, limit, exclude_ids): (in-flight). Tasks are expunged so they are safe to use after the txn.""" if limit <= 0: return [] - with self.db.session.begin_nested(): + with self.db.session.begin(): tasks = self.db.list_tasks(status=self._status, limit=limit, order_by=Task.completed_on.asc()) self.db.session.expunge_all() return [t for t in tasks if t.id not in exclude_ids] def mark_failed(self, task_id): - with self.db.session.begin_nested(): + with self.db.session.begin(): self.db.set_status(task_id, TASK_FAILED_PROCESSING) diff --git a/tests/test_processing_engine_source.py b/tests/test_processing_engine_source.py index 89d9e9082d2..0f055396182 100644 --- a/tests/test_processing_engine_source.py +++ b/tests/test_processing_engine_source.py @@ -4,10 +4,11 @@ def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): - t1 = db.add_path(temp_pe32) - t2 = db.add_path(temp_pe32) - db.set_status(t1, TASK_COMPLETED) - db.set_status(t2, TASK_COMPLETED) + with db.session.begin(): + t1 = db.add_path(temp_pe32) + t2 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) + db.set_status(t2, TASK_COMPLETED) src = TaskSource(db) got = src.fetch(limit=10, exclude_ids={t2}) @@ -16,9 +17,12 @@ def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): def test_mark_failed_sets_status(db, temp_pe32): - t1 = db.add_path(temp_pe32) - db.set_status(t1, TASK_COMPLETED) + with db.session.begin(): + t1 = db.add_path(temp_pe32) + db.set_status(t1, TASK_COMPLETED) src = TaskSource(db) src.mark_failed(t1) - assert db.view_task(t1).status == TASK_FAILED_PROCESSING + + with db.session.begin(): + assert db.view_task(t1).status == TASK_FAILED_PROCESSING From c6c382ee5f49b779a705ffd9789480a0637ae8ad Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:02:54 +0000 Subject: [PATCH 09/53] refactor(processor): document TaskSource.fetch limit semantics; cover failed_processing path --- lib/cuckoo/core/processing_engine/source.py | 9 ++++++++- tests/test_processing_engine_source.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/cuckoo/core/processing_engine/source.py b/lib/cuckoo/core/processing_engine/source.py index 18ba489960f..9210a60c890 100644 --- a/lib/cuckoo/core/processing_engine/source.py +++ b/lib/cuckoo/core/processing_engine/source.py @@ -15,7 +15,12 @@ def __init__(self, db, failed_processing=False): def fetch(self, limit, exclude_ids): """Return up to `limit` tasks needing processing, excluding `exclude_ids` - (in-flight). Tasks are expunged so they are safe to use after the txn.""" + (in-flight). Tasks are expunged so they are safe to use after the txn. + + Note: `limit` is applied at the DB level before `exclude_ids` filtering, + so the returned list may contain fewer than `limit` tasks even when more + eligible tasks exist in the DB (those in `exclude_ids` are still counted + against `limit`).""" if limit <= 0: return [] with self.db.session.begin(): @@ -24,5 +29,7 @@ def fetch(self, limit, exclude_ids): return [t for t in tasks if t.id not in exclude_ids] def mark_failed(self, task_id): + # Always writes TASK_FAILED_PROCESSING regardless of which status this + # source polls (the `failed_processing` constructor flag controls reads). with self.db.session.begin(): self.db.set_status(task_id, TASK_FAILED_PROCESSING) diff --git a/tests/test_processing_engine_source.py b/tests/test_processing_engine_source.py index 0f055396182..bca34f41eea 100644 --- a/tests/test_processing_engine_source.py +++ b/tests/test_processing_engine_source.py @@ -1,4 +1,3 @@ -import pytest from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING from lib.cuckoo.core.processing_engine.source import TaskSource @@ -26,3 +25,16 @@ def test_mark_failed_sets_status(db, temp_pe32): with db.session.begin(): assert db.view_task(t1).status == TASK_FAILED_PROCESSING + + +def test_fetch_with_failed_processing_returns_failed_tasks(db, temp_pe32): + with db.session.begin(): + t1 = db.add_path(temp_pe32) + t2 = db.add_path(temp_pe32) + db.set_status(t1, TASK_FAILED_PROCESSING) + db.set_status(t2, TASK_COMPLETED) + + src = TaskSource(db, failed_processing=True) + got = src.fetch(limit=10, exclude_ids=set()) + ids = [t.id for t in got] + assert t1 in ids and t2 not in ids From c54e40087d82edf18506f227a661040d20ee3a27 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:04:46 +0000 Subject: [PATCH 10/53] feat(processor): add ProcessingEngine base + get_engine registry --- lib/cuckoo/core/processing_engine/__init__.py | 16 ++++++++++++++++ lib/cuckoo/core/processing_engine/base.py | 14 ++++++++++++++ lib/cuckoo/core/processing_engine/pebble.py | 6 ++++++ tests/test_processing_engine_registry.py | 17 +++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 lib/cuckoo/core/processing_engine/base.py create mode 100644 lib/cuckoo/core/processing_engine/pebble.py create mode 100644 tests/test_processing_engine_registry.py diff --git a/lib/cuckoo/core/processing_engine/__init__.py b/lib/cuckoo/core/processing_engine/__init__.py index e69de29bb2d..9f4a5024206 100644 --- a/lib/cuckoo/core/processing_engine/__init__.py +++ b/lib/cuckoo/core/processing_engine/__init__.py @@ -0,0 +1,16 @@ +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +def get_engine(name, **kwargs): + """Construct the named engine. Imports are local so that selecting one engine + never imports the other's dependencies.""" + if name == "pebble": + from lib.cuckoo.core.processing_engine.pebble import PebbleEngine + return PebbleEngine(**kwargs) + if name == "prefork": + from lib.cuckoo.core.processing_engine.prefork import PreforkEngine + return PreforkEngine(**kwargs) + raise ValueError("unknown processing engine: %r" % name) + + +__all__ = ["ProcessingEngine", "get_engine"] diff --git a/lib/cuckoo/core/processing_engine/base.py b/lib/cuckoo/core/processing_engine/base.py new file mode 100644 index 00000000000..bcfd5d5afdf --- /dev/null +++ b/lib/cuckoo/core/processing_engine/base.py @@ -0,0 +1,14 @@ +"""Abstract processing engine: drives the per-task lifecycle for autoprocess. +Concrete engines (pebble, prefork) differ only in worker isolation.""" + + +class ProcessingEngine: + def __init__(self, task_fn, worker_init, source, parallel, timeout): + self.task_fn = task_fn # task_fn(task) -> None: runs ONE task to completion + self.worker_init = worker_init # called in worker context (pool init / post-fork) + self.source = source # TaskSource + self.parallel = parallel + self.timeout = timeout + + def run(self): + raise NotImplementedError diff --git a/lib/cuckoo/core/processing_engine/pebble.py b/lib/cuckoo/core/processing_engine/pebble.py new file mode 100644 index 00000000000..09cf86974c7 --- /dev/null +++ b/lib/cuckoo/core/processing_engine/pebble.py @@ -0,0 +1,6 @@ +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +class PebbleEngine(ProcessingEngine): + def run(self): + raise NotImplementedError diff --git a/tests/test_processing_engine_registry.py b/tests/test_processing_engine_registry.py new file mode 100644 index 00000000000..2b9ac8d0fa4 --- /dev/null +++ b/tests/test_processing_engine_registry.py @@ -0,0 +1,17 @@ +import pytest +from lib.cuckoo.core.processing_engine import get_engine +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + + +def _noop(*a, **k): + pass + + +def test_get_engine_returns_requested_class(): + eng = get_engine("pebble", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) + assert isinstance(eng, ProcessingEngine) + + +def test_get_engine_unknown_raises(): + with pytest.raises(ValueError): + get_engine("nope", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) From ed7e17ccb65e66f35efc7a206292125fbed40a9f Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:11:29 +0000 Subject: [PATCH 11/53] refactor(processor): extract run_task(task) adapter shared by engines Co-Authored-By: Claude Sonnet 4.6 --- tests/test_run_task_adapter.py | 21 +++++++++++++++++++++ utils/process.py | 19 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/test_run_task_adapter.py diff --git a/tests/test_run_task_adapter.py b/tests/test_run_task_adapter.py new file mode 100644 index 00000000000..898d3910b66 --- /dev/null +++ b/tests/test_run_task_adapter.py @@ -0,0 +1,21 @@ +import types +import utils.process as proc + + +def test_run_task_calls_process_with_task_and_auto(monkeypatch, db, temp_pe32): + captured = {} + + def fake_process(target=None, sample_sha256=None, task=None, report=False, auto=False, **kw): + captured["task_id"] = task.id + captured["auto"] = auto + captured["report"] = report + + monkeypatch.setattr(proc, "process", fake_process) + monkeypatch.setattr(proc, "db", db) + + tid = db.add_path(temp_pe32) + task = db.view_task(tid) + proc.run_task(task) + + assert captured["task_id"] == tid + assert captured["auto"] is True and captured["report"] is True diff --git a/utils/process.py b/utils/process.py index 79470761edd..f4c066c529f 100644 --- a/utils/process.py +++ b/utils/process.py @@ -194,6 +194,25 @@ def process( db.session.remove() +def run_task(task, memory_debugging=False, debug=False): + """Run exactly one completed task to completion (processing -> report). + Extracted from autoprocess so every engine shares identical per-task setup.""" + sample_hash = "" + if task.category != "url": + sample = db.view_sample(task.sample_id) + if sample: + sample_hash = sample.sha256 + process( + task.target, + sample_hash, + report=True, + auto=True, + task=task, + memory_debugging=memory_debugging, + debug=debug, + ) + + def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) # See https://docs.sqlalchemy.org/en/14/core/pooling.html#using-connection-pools-with-multiprocessing-or-os-fork From ca6d82167aab2acb19ccdda85826206c407ed335 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:13:16 +0000 Subject: [PATCH 12/53] fix(processor): restore session.begin() wrapper in run_task; fix test setup to match production txn discipline --- tests/test_run_task_adapter.py | 7 ++++--- utils/process.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_run_task_adapter.py b/tests/test_run_task_adapter.py index 898d3910b66..4e27b6a2091 100644 --- a/tests/test_run_task_adapter.py +++ b/tests/test_run_task_adapter.py @@ -1,4 +1,3 @@ -import types import utils.process as proc @@ -13,8 +12,10 @@ def fake_process(target=None, sample_sha256=None, task=None, report=False, auto= monkeypatch.setattr(proc, "process", fake_process) monkeypatch.setattr(proc, "db", db) - tid = db.add_path(temp_pe32) - task = db.view_task(tid) + with db.session.begin(): + tid = db.add_path(temp_pe32) + task = db.view_task(tid) + proc.run_task(task) assert captured["task_id"] == tid diff --git a/utils/process.py b/utils/process.py index f4c066c529f..c89515ceef9 100644 --- a/utils/process.py +++ b/utils/process.py @@ -199,9 +199,10 @@ def run_task(task, memory_debugging=False, debug=False): Extracted from autoprocess so every engine shares identical per-task setup.""" sample_hash = "" if task.category != "url": - sample = db.view_sample(task.sample_id) - if sample: - sample_hash = sample.sha256 + with db.session.begin(): + sample = db.view_sample(task.sample_id) + if sample: + sample_hash = sample.sha256 process( task.target, sample_hash, From 7f042775ee50618bca0d7271ca4ab88975eafd3d Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:30:25 +0000 Subject: [PATCH 13/53] feat(processor): PebbleEngine behind engine seam + --engine flag (default pebble) Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/cuckoo/core/processing_engine/pebble.py | 108 ++++++++++++- tests/test_pebble_engine.py | 43 +++++ utils/process.py | 167 +++++--------------- 3 files changed, 190 insertions(+), 128 deletions(-) create mode 100644 tests/test_pebble_engine.py diff --git a/lib/cuckoo/core/processing_engine/pebble.py b/lib/cuckoo/core/processing_engine/pebble.py index 09cf86974c7..025a511a39e 100644 --- a/lib/cuckoo/core/processing_engine/pebble.py +++ b/lib/cuckoo/core/processing_engine/pebble.py @@ -1,6 +1,112 @@ +"""Pebble-pool engine — preserved as the A/B control. Mirrors the historical +autoprocess loop, parameterized behind the engine seam. max_tasks defaults to 0 +(no recycle): worker recycling deadlocks in multiprocessing _exit_function while +joining the nested extractor pool — see the redesign spec.""" +import logging +import os +import time + +import pebble +from concurrent.futures import TimeoutError + from lib.cuckoo.core.processing_engine.base import ProcessingEngine +log = logging.getLogger(__name__) + class PebbleEngine(ProcessingEngine): + """Pebble-pool processing engine. + + Parameters + ---------- + task_fn : callable + Called in a worker process for each task: ``task_fn(task) -> None``. + worker_init : callable + Called once per worker process at pool initialisation. + source : TaskSource + Supplies tasks to run and records failure status. + parallel : int + Maximum number of concurrent worker processes. + timeout : int + Per-task timeout in seconds (passed to pebble). + max_tasks : int + Max tasks per child process (pebble ``max_tasks``). Default 0 = no + recycling — prevents the multiprocessing ``_exit_function`` / pool-join + deadlock seen with worker recycling in production. + max_count : int + Exit after scheduling this many tasks. 0 (default) = run forever, + matching ``cfg.cuckoo.max_analysis_count == 0`` production default. + """ + + def __init__(self, task_fn, worker_init, source, parallel, timeout, + max_tasks=0, max_count=0): + super().__init__(task_fn, worker_init, source, parallel, timeout) + self.max_tasks = max_tasks + self.max_count = max_count + self._pending = {} # future -> task_id + + def _done(self, future): + """Pebble done-callback: fires in the pool's internal thread.""" + task_id = self._pending.pop(future, None) + try: + future.result() + log.info("Reports generation completed for Task #%s", task_id) + except TimeoutError as error: + log.error("[%s] Processing timeout: %s. Function: %s", task_id, error, error.args[1] if len(error.args) > 1 else "") + if task_id is not None: + self.source.mark_failed(task_id) + except (pebble.ProcessExpired, Exception) as error: + log.exception("[%s] Exception when processing task: %s", task_id, error) + if task_id is not None: + self.source.mark_failed(task_id) + def run(self): - raise NotImplementedError + """Drive the pebble pool loop, mirroring the historical autoprocess body.""" + from lib.cuckoo.common.config import Config + from lib.cuckoo.common.constants import CUCKOO_ROOT + + cfg = Config() + count = 0 + + with pebble.ProcessPool(max_workers=self.parallel, max_tasks=self.max_tasks, + initializer=self.worker_init) as pool: + while not self.max_count or count < self.max_count: + # If not enough free disk space is available, block until space + # is reclaimed. Mirrors the original autoprocess freespace check + # (only when cfg.cuckoo.freespace_processing is non-zero). + if cfg.cuckoo.freespace_processing: + from lib.cuckoo.common.cleaners_utils import free_space_monitor + dir_path = os.path.join(CUCKOO_ROOT, "storage", "analyses") + free_space_monitor(dir_path, processing=True) + + # If the pool is saturated, wait before polling again. + if len(self._pending) >= self.parallel: + time.sleep(1) + continue + + tasks = self.source.fetch(limit=self.parallel, + exclude_ids=set(self._pending.values())) + added = False + # Schedule at most one task per iteration to avoid overshooting + # max_count (same rationale as the original "For loop to add + # only one, nice." comment). + for task in tasks: + log.info("Processing analysis data for Task #%d", task.id) + future = pool.schedule(self.task_fn, args=(task,), timeout=self.timeout) + self._pending[future] = task.id + future.add_done_callback(self._done) + count += 1 + added = True + break + + if not added and not self.max_count: + # Nothing ready; avoid busy-wait in production. + time.sleep(5) + if not added and self.max_count: + # We've exhausted available tasks and we have a fixed + # max_count limit — break out so the drain below runs. + break + + # Drain: wait for all in-flight tasks to finish before returning. + while self._pending: + time.sleep(0.2) diff --git a/tests/test_pebble_engine.py b/tests/test_pebble_engine.py new file mode 100644 index 00000000000..7f60ce1597f --- /dev/null +++ b/tests/test_pebble_engine.py @@ -0,0 +1,43 @@ +import os + +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_REPORTED +from lib.cuckoo.core.processing_engine.pebble import PebbleEngine +from lib.cuckoo.core.processing_engine.source import TaskSource + + +# Must be module-level (not a closure) so pickle can serialise it. +def _task_fn_write_sentinel(task): + """Writes task.id to an env-var-specified sentinel file. + + Runs inside a pebble subprocess. Uses os.environ to communicate the + sentinel path because closures are not picklable with stdlib pickle. + """ + sentinel = os.environ.get("_PEBBLE_TEST_SENTINEL", "") + if sentinel: + with open(sentinel, "w") as fh: + fh.write(str(task.id)) + + +def test_pebble_engine_processes_one_task(db, temp_pe32, tmp_path, monkeypatch): + """PebbleEngine schedules a TASK_COMPLETED task, calls task_fn in a + worker subprocess, drains completely, and returns. + + We use a filesystem sentinel to confirm task_fn executed. + threading.Event cannot cross process boundaries; closures are not + picklable with stdlib pickle so we pass the path via os.environ.""" + with db.session.begin(): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + sentinel = str(tmp_path / "ran.txt") + monkeypatch.setenv("_PEBBLE_TEST_SENTINEL", sentinel) + + eng = PebbleEngine(task_fn=_task_fn_write_sentinel, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30, max_count=1) + eng.run() + + # run() only returns after all in-flight futures complete (drain guarantee). + assert eng._pending == {}, "drain loop should leave _pending empty" + # Confirm task_fn actually executed in the worker subprocess. + assert os.path.exists(sentinel), "worker did not write sentinel — task_fn never ran" + assert open(sentinel).read().strip() == str(tid) diff --git a/utils/process.py b/utils/process.py index c89515ceef9..90b3033a666 100644 --- a/utils/process.py +++ b/utils/process.py @@ -70,8 +70,6 @@ check_linux_dist() -pending_future_map = {} -pending_task_id_map = {} original_proctitle = getproctitle() @@ -397,148 +395,56 @@ def init_per_analysis_logging(tid=0, debug=False): return fhpa -def processing_finished(future): - """ - Callback function to handle the completion of a processing task. - - This function is called when a future task is completed. It retrieves the task ID from the - pending_future_map, logs the result, and updates the task status in the database. If an - exception occurs during processing, it logs the error and sets the task status to failed. - - Args: - future (concurrent.futures.Future): The future object representing the asynchronous task. - - Raises: - TimeoutError: If the processing task times out. - pebble.ProcessExpired: If the processing task expires. - Exception: For any other exceptions that occur during processing. - """ - task_id = pending_future_map.get(future) - with db.session.begin(): - try: - _ = future.result() - log.info("Reports generation completed for Task #%d", task_id) - except TimeoutError as error: - log.error("[%d] Processing timeout: %s. Function: %s", task_id, error, error.args[1]) - db.set_status(task_id, TASK_FAILED_PROCESSING) - except (pebble.ProcessExpired, Exception) as error: - log.exception("[%d] Exception when processing task: %s", task_id, error) - db.set_status(task_id, TASK_FAILED_PROCESSING) - - pending_future_map.pop(future) - pending_task_id_map.pop(task_id) - set_formatter_fmt() - setproctitle(original_proctitle) - def autoprocess( - parallel=1, failed_processing=False, maxtasksperchild=7, memory_debugging=False, processing_timeout=300, debug: bool = False, disable_memory_limit: bool = False + parallel=1, + failed_processing=False, + maxtasksperchild=7, + memory_debugging=False, + processing_timeout=300, + debug: bool = False, + disable_memory_limit: bool = False, + engine: str = "pebble", ): """ - Automatically processes analysis data using a process pool. + Automatically processes analysis data using a process pool engine. Args: parallel (int): Number of parallel processes to use. Default is 1. failed_processing (bool): Whether to process failed tasks. Default is False. - maxtasksperchild (int): Maximum number of tasks per child process. Default is 7. + maxtasksperchild (int): Maximum number of tasks per child process (pebble engine only). + Default is 7. Pass 0 to disable worker recycling (avoids deadlock on some systems). memory_debugging (bool): Whether to enable memory debugging. Default is False. processing_timeout (int): Timeout for processing each task in seconds. Default is 300. debug (bool): Whether to enable debug mode. Default is False. + disable_memory_limit (bool): Skip setting process memory limits. Default is False. + engine (str): Processing engine to use: "pebble" (default) or "prefork". Raises: KeyboardInterrupt: If the process is interrupted by the user. MemoryError: If there is not enough free RAM to run processing. OSError: If an OS-related error occurs. Exception: If any other exception occurs during processing. - """ - maxcount = cfg.cuckoo.max_analysis_count - count = 0 - # pool = multiprocessing.Pool(parallel, init_worker) - pool = False - try: - if not disable_memory_limit: - memory_limit() - log.info("Processing analysis data") - with pebble.ProcessPool(max_workers=parallel, max_tasks=maxtasksperchild, initializer=init_worker) as pool: - # CAUTION - big ugly loop ahead. - while count < maxcount or not maxcount: - # If not enough free disk space is available, then we print an - # error message and wait another round (this check is ignored - # when the freespace configuration variable is set to zero). - if cfg.cuckoo.freespace_processing: - # Resolve the full base path to the analysis folder, just in - # case somebody decides to make a symbolic link out of it. - dir_path = os.path.join(CUCKOO_ROOT, "storage", "analyses") - free_space_monitor(dir_path, processing=True) - - # If still full, don't add more (necessary despite pool). - if len(pending_task_id_map) >= parallel: - time.sleep(5) - continue - with db.session.begin(): - if failed_processing: - tasks = db.list_tasks(status=TASK_FAILED_PROCESSING, limit=parallel, order_by=Task.completed_on.asc()) - else: - tasks = db.list_tasks(status=TASK_COMPLETED, limit=parallel, order_by=Task.completed_on.asc()) - # Make sure the tasks are available as normal objects after the transaction ends, so that - # sqlalchemy doesn't auto-initiate a new transaction the next time they are accessed. - db.session.expunge_all() - added = False - # For loop to add only one, nice. (reason is that we shouldn't overshoot maxcount) - for task in tasks: - # Not-so-efficient lock. - if pending_task_id_map.get(task.id): - continue - - log.info("Processing analysis data for Task #%d", task.id) - sample_hash = "" - if task.category != "url": - with db.session.begin(): - sample = db.view_sample(task.sample_id) - if sample: - sample_hash = sample.sha256 - - args = task.target, sample_hash - kwargs = dict(report=True, auto=True, task=task, memory_debugging=memory_debugging, debug=debug) - if memory_debugging: - gc.collect() - log.info("(before) GC object counts: %d, %d", len(gc.get_objects()), len(gc.garbage)) - # result = pool.apply_async(process, args, kwargs) - future = pool.schedule(process, args, kwargs, timeout=processing_timeout) - pending_future_map[future] = task.id - pending_task_id_map[task.id] = future - future.add_done_callback(processing_finished) - if memory_debugging: - gc.collect() - log.info("(after) GC object counts: %d, %d", len(gc.get_objects()), len(gc.garbage)) - count += 1 - added = True - break - - if not added: - # don't hog cpu - time.sleep(5) - except KeyboardInterrupt: - # ToDo verify in finally - # pool.terminate() - raise - except (MemoryError, OSError) as e: - mem = get_memory() / 1024 / 1024 - log.critical( - "Memory Exception: Remain: %.2f GB. Your system doesn't have enough FREE RAM to run processing! Error: %s", - mem, - e, - ) - sys.exit(1) - except Exception: - import traceback - - traceback.print_exc() - finally: - if pool: - pool.close() - pool.join() + from lib.cuckoo.core.processing_engine import get_engine + from lib.cuckoo.core.processing_engine.source import TaskSource + + if not disable_memory_limit: + memory_limit() + log.info("Processing analysis data (engine=%s)", engine) + source = TaskSource(db, failed_processing=failed_processing) + eng = get_engine( + engine, + task_fn=lambda task: run_task(task, memory_debugging=memory_debugging, debug=debug), + worker_init=init_worker, + source=source, + parallel=parallel, + timeout=processing_timeout, + ) + if engine == "pebble": + eng.max_tasks = maxtasksperchild + eng.max_count = cfg.cuckoo.max_analysis_count + eng.run() def _load_report(task_id: int): @@ -712,6 +618,12 @@ def main(): required=False, default=str_to_bool(os.getenv("CAPE_DISABLE_MEMORY_LIMIT", "false")), ) + parser.add_argument( + "--engine", + choices=["pebble", "prefork"], + default="pebble", + help="Processing engine: pebble (default, A/B control) or prefork.", + ) args = parser.parse_args() init_database() @@ -725,7 +637,8 @@ def main(): memory_debugging=args.memory_debugging, processing_timeout=args.processing_timeout, debug=args.debug, - disable_memory_limit= args.disable_memory_limit, + disable_memory_limit=args.disable_memory_limit, + engine=args.engine, ) else: for start, end in args.id: From bdb2b009ac07950ca9c18b0717ff131e92761cf4 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:45:08 +0000 Subject: [PATCH 14/53] fix(processor): make pebble task_fn picklable (functools.partial); restore per-task proctitle/formatter cleanup Co-Authored-By: Claude Sonnet 4.6 --- tests/test_pebble_engine.py | 13 ++++++++++++- utils/process.py | 25 +++++++++++++++---------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/test_pebble_engine.py b/tests/test_pebble_engine.py index 7f60ce1597f..b6a339b5048 100644 --- a/tests/test_pebble_engine.py +++ b/tests/test_pebble_engine.py @@ -1,6 +1,8 @@ +import functools import os +import pickle -from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_REPORTED +from lib.cuckoo.core.data.task import TASK_COMPLETED from lib.cuckoo.core.processing_engine.pebble import PebbleEngine from lib.cuckoo.core.processing_engine.source import TaskSource @@ -41,3 +43,12 @@ def test_pebble_engine_processes_one_task(db, temp_pe32, tmp_path, monkeypatch): # Confirm task_fn actually executed in the worker subprocess. assert os.path.exists(sentinel), "worker did not write sentinel — task_fn never ran" assert open(sentinel).read().strip() == str(tid) + + +def test_autoprocess_task_fn_is_picklable(): + """Regression: pebble dispatches task_fn over multiprocessing pipes; a + function-scope lambda would crash with 'Can't pickle local object' on the + first task. functools.partial is picklable; lambdas are not.""" + from utils.process import run_task + task_fn = functools.partial(run_task, memory_debugging=False, debug=False) + pickle.dumps(task_fn) # would raise PicklingError for a lambda diff --git a/utils/process.py b/utils/process.py index 90b3033a666..e9ff0b730a3 100644 --- a/utils/process.py +++ b/utils/process.py @@ -3,6 +3,7 @@ # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. import argparse +import functools import gc import json import logging @@ -201,15 +202,19 @@ def run_task(task, memory_debugging=False, debug=False): sample = db.view_sample(task.sample_id) if sample: sample_hash = sample.sha256 - process( - task.target, - sample_hash, - report=True, - auto=True, - task=task, - memory_debugging=memory_debugging, - debug=debug, - ) + try: + process( + task.target, + sample_hash, + report=True, + auto=True, + task=task, + memory_debugging=memory_debugging, + debug=debug, + ) + finally: + set_formatter_fmt() + setproctitle(original_proctitle) def init_worker(): @@ -435,7 +440,7 @@ def autoprocess( source = TaskSource(db, failed_processing=failed_processing) eng = get_engine( engine, - task_fn=lambda task: run_task(task, memory_debugging=memory_debugging, debug=debug), + task_fn=functools.partial(run_task, memory_debugging=memory_debugging, debug=debug), worker_init=init_worker, source=source, parallel=parallel, From e9bd9b6c43f0f211cc2cc4272fffef1dc92fe68b Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:48:13 +0000 Subject: [PATCH 15/53] feat(processor): PreforkEngine scaffolding + single-threaded invariant --- lib/cuckoo/core/processing_engine/prefork.py | 53 ++++++++++++++++++++ tests/test_prefork_engine.py | 22 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 lib/cuckoo/core/processing_engine/prefork.py create mode 100644 tests/test_prefork_engine.py diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py new file mode 100644 index 00000000000..3a1060b6194 --- /dev/null +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -0,0 +1,53 @@ +"""Single-threaded prefork supervisor: forks one child per task, each child runs +exactly one task then os._exit(); the supervisor reaps children and enforces a +launch-relative wall-clock timeout via process-group kill. No supervisor<->child +channel exists. See docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md.""" +import logging +import os +import signal +import threading +import time + +from lib.cuckoo.core.processing_engine.base import ProcessingEngine + +log = logging.getLogger(__name__) + + +class _Child: + __slots__ = ("task_id", "pid", "start", "pgid", "timed_out", "kill_deadline") + + def __init__(self, task_id, pid, start, pgid): + self.task_id = task_id + self.pid = pid + self.start = start + self.pgid = pgid + self.timed_out = False + self.kill_deadline = None + + +class PreforkEngine(ProcessingEngine): + def __init__(self, task_fn, worker_init, source, parallel, timeout, + heartbeat_interval=30, term_grace=5, max_count=0, poll_interval=0.2): + super().__init__(task_fn, worker_init, source, parallel, timeout) + self.heartbeat_interval = heartbeat_interval + self.term_grace = term_grace + self.max_count = max_count + self.poll_interval = poll_interval + self._inflight = {} # pid -> _Child + + def _assert_single_threaded(self): + n = threading.active_count() + if n != 1: + raise RuntimeError( + "prefork supervisor must be single-threaded before fork; " + "active_count=%d (a thread/Mongo client/pool leaked into the supervisor)" % n) + + def _inflight_task_ids(self): + return {c.task_id for c in self._inflight.values()} + + def _heartbeat(self): + if self._inflight: + oldest = max(time.monotonic() - c.start for c in self._inflight.values()) + else: + oldest = 0.0 + log.info("prefork heartbeat: in_flight=%d oldest=%.0fs", len(self._inflight), oldest) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py new file mode 100644 index 00000000000..ab06e358f42 --- /dev/null +++ b/tests/test_prefork_engine.py @@ -0,0 +1,22 @@ +import os +import threading +import time + +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, TASK_REPORTED +from lib.cuckoo.core.processing_engine.prefork import PreforkEngine +from lib.cuckoo.core.processing_engine.source import TaskSource + + +def test_single_threaded_invariant_raises_when_extra_thread(db): + eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30) + stop = threading.Event() + t = threading.Thread(target=stop.wait) + t.start() + try: + import pytest + with pytest.raises(RuntimeError, match="single-threaded"): + eng._assert_single_threaded() + finally: + stop.set() + t.join() From 0ad421c5ce9b7b40bb7d706c43cd6eff1542b809 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 15:56:39 +0000 Subject: [PATCH 16/53] feat(processor): prefork fork-per-task execution + reap + failure status Co-Authored-By: Claude Sonnet 4.6 --- lib/cuckoo/core/processing_engine/prefork.py | 61 ++++++++++++++++++++ tests/test_prefork_engine.py | 53 +++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 3a1060b6194..86ee1734eaf 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -51,3 +51,64 @@ def _heartbeat(self): else: oldest = 0.0 log.info("prefork heartbeat: in_flight=%d oldest=%.0fs", len(self._inflight), oldest) + + def _child_main(self, task): + os.setsid() # own session/process group; supervisor killpg sweeps the subtree + try: + self.worker_init() + self.task_fn(task) + return 0 + except BaseException: + log.exception("prefork child: task %s crashed", getattr(task, "id", "?")) + return 1 + + def _launch(self, task): + self._assert_single_threaded() + pid = os.fork() + if pid == 0: + code = 1 + try: + code = self._child_main(task) + finally: + os._exit(code) # NEVER return: skip multiprocessing atexit join + self._inflight[pid] = _Child(task_id=task.id, pid=pid, start=time.monotonic(), pgid=pid) + log.debug("prefork: launched task %d as pid %d", task.id, pid) + + def _reap(self): + while True: + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return + if pid == 0: + return + child = self._inflight.pop(pid, None) + if child is None: + continue + if child.timed_out: + continue # already marked failed during timeout enforcement + ok = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + if not ok: + log.warning("prefork: task %d (pid %d) abnormal exit status=%d -> FAILED_PROCESSING", + child.task_id, pid, status) + self.source.mark_failed(child.task_id) + + def run(self): + count = 0 + last_hb = 0.0 + while True: + self._reap() + if self.max_count and count >= self.max_count and not self._inflight: + return + free = self.parallel - len(self._inflight) + launchable = free if not self.max_count else min(free, self.max_count - count) + if launchable > 0: + tasks = self.source.fetch(limit=launchable, exclude_ids=self._inflight_task_ids()) + for task in tasks[:launchable]: + self._launch(task) + count += 1 + now = time.monotonic() + if now - last_hb >= self.heartbeat_interval: + self._heartbeat() + last_hb = now + time.sleep(self.poll_interval) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index ab06e358f42..94abe588f08 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -2,11 +2,28 @@ import threading import time +import pytest + from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, TASK_REPORTED +from lib.cuckoo.core.database import Database, init_database, reset_database_FOR_TESTING_ONLY from lib.cuckoo.core.processing_engine.prefork import PreforkEngine from lib.cuckoo.core.processing_engine.source import TaskSource +@pytest.fixture +def db_file(tmp_path): + """File-backed SQLite fixture needed for fork tests: in-memory sqlite:// + is per-connection so cross-process writes are invisible to the parent.""" + dsn = f"sqlite:///{tmp_path}/test.db" + reset_database_FOR_TESTING_ONLY() + try: + init_database(dsn=dsn) + retval = Database() + yield retval + finally: + reset_database_FOR_TESTING_ONLY() + + def test_single_threaded_invariant_raises_when_extra_thread(db): eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, source=TaskSource(db), parallel=2, timeout=30) @@ -20,3 +37,39 @@ def test_single_threaded_invariant_raises_when_extra_thread(db): finally: stop.set() t.join() + + +def test_normal_task_runs_and_status_set_by_child(db_file, temp_pe32): + with db_file.session.begin(): + tid = db_file.add_path(temp_pe32) + db_file.set_status(tid, TASK_COMPLETED) + + # child runs this; it must set status itself (child sets, supervisor overrides only on failure) + def task_fn(task): + # fresh DB connection in child (file-backed SQLite so the write is + # visible to the parent); must be wrapped in a transaction to commit + from lib.cuckoo.core.database import Database + db = Database() + with db.session.begin(): + db.set_status(task.id, TASK_REPORTED) + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, + source=TaskSource(db_file), parallel=2, timeout=30, max_count=1) + eng.run() + with db_file.session.begin(): + assert db_file.view_task(tid).status == TASK_REPORTED + + +def test_crashing_task_marked_failed_by_supervisor(db, temp_pe32): + with db.session.begin(): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + def task_fn(task): + os._exit(3) # simulate abnormal exit / crash + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30, max_count=1) + eng.run() + with db.session.begin(): + assert db.view_task(tid).status == TASK_FAILED_PROCESSING From 1e65cd64f516b336419d9919b15086d55a9d6dde Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:06:17 +0000 Subject: [PATCH 17/53] feat(processor): prefork wall-clock timeout via killpg + grace escalation Co-Authored-By: Claude Sonnet 4.6 --- lib/cuckoo/core/processing_engine/prefork.py | 27 +++++++++++++++++ tests/test_prefork_engine.py | 32 ++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 86ee1734eaf..c5d5bbe1dce 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -93,11 +93,38 @@ def _reap(self): child.task_id, pid, status) self.source.mark_failed(child.task_id) + def _enforce_timeouts(self): + now = time.monotonic() + for child in list(self._inflight.values()): + if child.timed_out or now - child.start <= self.timeout: + continue + log.error("prefork: task %d (pid %d) exceeded %ds -> killpg", + child.task_id, child.pid, self.timeout) + child.timed_out = True + self.source.mark_failed(child.task_id) + try: + os.killpg(child.pgid, signal.SIGTERM) + except ProcessLookupError: + continue + child.kill_deadline = now + self.term_grace + + def _escalate_kills(self): + now = time.monotonic() + for child in list(self._inflight.values()): + if child.timed_out and child.kill_deadline and now > child.kill_deadline: + try: + os.killpg(child.pgid, signal.SIGKILL) + except ProcessLookupError: + pass + child.kill_deadline = None + def run(self): count = 0 last_hb = 0.0 while True: self._reap() + self._enforce_timeouts() + self._escalate_kills() if self.max_count and count >= self.max_count and not self._inflight: return free = self.parallel - len(self._inflight) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index 94abe588f08..c42d4ae0ffe 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -73,3 +73,35 @@ def task_fn(task): eng.run() with db.session.begin(): assert db.view_task(tid).status == TASK_FAILED_PROCESSING + + +def test_timeout_kills_process_group_no_orphans(db, temp_pe32): + with db.session.begin(): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) + + marker = "/tmp/prefork_orphan_%d" % os.getpid() + + def task_fn(task): + # spawn a grandchild that would outlive the worker, then hang + import subprocess, sys + subprocess.Popen([sys.executable, "-c", + "import time,os;open(%r,'w').close();time.sleep(120)" % marker]) + time.sleep(120) + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, source=TaskSource(db), + parallel=1, timeout=1, term_grace=1, max_count=1, poll_interval=0.1) + if os.path.exists(marker): + os.unlink(marker) + eng.run() + + with db.session.begin(): + assert db.view_task(tid).status == TASK_FAILED_PROCESSING + + # grandchild must have been swept by killpg (give it a moment) + time.sleep(2) + import subprocess + out = subprocess.run(["pgrep", "-f", marker], capture_output=True, text=True) + assert out.stdout.strip() == "", "orphaned grandchild survived killpg" + if os.path.exists(marker): + os.unlink(marker) From f5c1a88d1fc9d49f4b7eb627d375a8b67d51a17a Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:12:22 +0000 Subject: [PATCH 18/53] fix(processor): log SIGKILL escalation; broaden killpg exception handling; strengthen no-orphan test --- lib/cuckoo/core/processing_engine/prefork.py | 9 +++++++++ tests/test_prefork_engine.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index c5d5bbe1dce..722eeee25e2 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -106,16 +106,25 @@ def _enforce_timeouts(self): os.killpg(child.pgid, signal.SIGTERM) except ProcessLookupError: continue + except OSError as e: + log.warning("prefork: killpg(SIGTERM) failed for task %d (pid %d): %s", + child.task_id, child.pid, e) + continue child.kill_deadline = now + self.term_grace def _escalate_kills(self): now = time.monotonic() for child in list(self._inflight.values()): if child.timed_out and child.kill_deadline and now > child.kill_deadline: + log.warning("prefork: task %d (pid %d) did not exit after SIGTERM, escalating to SIGKILL", + child.task_id, child.pid) try: os.killpg(child.pgid, signal.SIGKILL) except ProcessLookupError: pass + except OSError as e: + log.warning("prefork: killpg(SIGKILL) failed for task %d (pid %d): %s", + child.task_id, child.pid, e) child.kill_deadline = None def run(self): diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index c42d4ae0ffe..4bbe76669d0 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -100,6 +100,9 @@ def task_fn(task): # grandchild must have been swept by killpg (give it a moment) time.sleep(2) + # Sanity: the grandchild must have started and written the marker + # before being killed; otherwise the pgrep check below would pass vacuously. + assert os.path.exists(marker), "grandchild never started — test is vacuous" import subprocess out = subprocess.run(["pgrep", "-f", marker], capture_output=True, text=True) assert out.stdout.strip() == "", "orphaned grandchild survived killpg" From 4cdd9c6abc9e19520e2134037bf5fdbd8da9a535 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:15:45 +0000 Subject: [PATCH 19/53] test(processor): regression test locking worker_init-before-task_fn ordering Add test_worker_init_called_in_child to tests/test_prefork_engine.py using the db_file fixture (file-backed SQLite visible across fork boundary). The test asserts that a filesystem flag written by worker_init is present when task_fn runs in the child, locking the _child_main contract from Task 6. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_prefork_engine.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index 4bbe76669d0..6eb057dd963 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -108,3 +108,32 @@ def task_fn(task): assert out.stdout.strip() == "", "orphaned grandchild survived killpg" if os.path.exists(marker): os.unlink(marker) + + +def test_worker_init_called_in_child(db_file, temp_pe32): + with db_file.session.begin(): + tid = db_file.add_path(temp_pe32) + db_file.set_status(tid, TASK_COMPLETED) + flag = "/tmp/prefork_winit_ran_%d" % os.getpid() + + def worker_init(): + open(flag, "w").close() + + def task_fn(task): + assert os.path.exists(flag), "worker_init must run before task_fn in child" + from lib.cuckoo.core.database import Database + db = Database() + with db.session.begin(): + db.set_status(task.id, TASK_REPORTED) + + if os.path.exists(flag): + os.unlink(flag) + try: + eng = PreforkEngine(task_fn=task_fn, worker_init=worker_init, source=TaskSource(db_file), + parallel=1, timeout=30, max_count=1) + eng.run() + with db_file.session.begin(): + assert db_file.view_task(tid).status == TASK_REPORTED + finally: + if os.path.exists(flag): + os.unlink(flag) From 4659b668df4143f6423da65160713284a7c60687 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:27:13 +0000 Subject: [PATCH 20/53] docs(processor): spike results for extractor sub-pool spawn strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves design-spec §4.2(d): measured pebble.ProcessPool latency and killpg orphan safety for both 'fork' and 'forkserver' contexts. Decision: use fork (115.7 ms mean vs 125.5 ms / 219 ms p95 for forkserver; both pass orphan check). Task 10 will implement _EXTRACTOR_POOL accordingly. Co-Authored-By: Claude Sonnet 4.6 --- .../2026-05-27-extractor-subpool-spike.md | 506 ++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100644 docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md diff --git a/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md b/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md new file mode 100644 index 00000000000..a1b2894f012 --- /dev/null +++ b/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md @@ -0,0 +1,506 @@ +# Extractor Sub-Pool Spawn Strategy: Spike Results + +**Date:** 2026-05-27 +**Scope:** Resolve design-spec §4.2(d) — which `multiprocessing` start method +(`forkserver` vs `fork`) the per-task `_EXTRACTOR_POOL` should use under the +prefork supervisor (Tasks 5–8). +**Status:** DECIDED — use **`fork`** +**Implements:** Task 9 of the processor-prefork-engine plan. +**Next:** Task 10 rewrites `_EXTRACTOR_POOL` in +`lib/cuckoo/common/integrations/file_extra_info.py` using +`multiprocessing.get_context("fork")`. + +--- + +## Decision Criterion + +Choose the faster strategy that leaves **zero survivors after `killpg`** and +runs cleanly with `os._exit()`. If latency is a tie, prefer `forkserver` for +stronger isolation; otherwise take the winner on latency. + +--- + +## Environment + +| Item | Value | +|---|---| +| Host Python | 3.12.3 (GCC 13.3.0) | +| pebble | 5.1.0 | +| Available start methods | `fork`, `spawn`, `forkserver` | +| `MAX_WORKERS` tested | 6 (matches planned `_EXTRACTOR_POOL`) | +| Iterations per context | 7 clean samples (8th occasionally lost to fork-pipe flush; stats below are over the captured samples) | + +--- + +## Results + +### Latency (pool-create + first-result, ms) + +Each iteration: create a fresh `pebble.ProcessPool(max_workers=6, context=ctx)`, +schedule one `trivial_extractor` call (200× SHA-256 rounds on 256 random bytes), +collect result, `pool.close(); pool.join()`. Measured inside a forked child +that is single-threaded at the time of pool creation (mirrors production: the +task child is forked from the supervisor before doing any threaded work). + +| context | mean | median | p95 | min | max | errors | +|---|---|---|---|---|---|---| +| `fork` | **115.7 ms** | 117.0 ms | 118.3 ms | 112.9 ms | 118.3 ms | 0 | +| `forkserver` | **125.5 ms** | 112.3 ms | **219.0 ms** | 111.3 ms | 219.0 ms | 0 | + +Key observations: +- `fork` is stable: all 7 samples land within a 5.5 ms band (112.9–118.3 ms). +- `forkserver` shows a **~107 ms cold-start penalty on iteration 1** (219 ms vs + the steady-state ~112 ms). This is the forkserver helper process booting. + Because each task child is ephemeral (it exits via `os._exit()` after the task + completes), **every task pays the cold-start cost** — the forkserver process + does not survive across tasks. The steady-state median (~112 ms) is therefore + misleading; the operative number for one-shot task children is the p95 + (219 ms). +- `fork` mean (115.7 ms) vs `forkserver` mean (125.5 ms): **9.8 ms faster** on + mean; the true per-task advantage is larger due to the cold-start effect. + +### Orphan Check (killpg safety) + +For each context: fork a child that calls `os.setsid()`, creates the sub-pool +with `MAX_WORKERS=6` workers all running, then sleeps. Parent records the +child's `pgid`, waits 3 s, sends `SIGKILL` to the process group +(`os.killpg(pgid, SIGKILL)`), waits 0.5 s, then runs `pgrep -g ` to +check for survivors. + +| context | child pgid | survivors after killpg | result | +|---|---|---|---| +| `fork` | 3435871 | (none) | **PASS** | +| `forkserver` | 3435882 | (none) | **PASS** | + +Both contexts leave zero survivors. The forkserver helper process and all pool +workers are members of the child's process group (they inherit the session from +the `setsid()` call) and are swept cleanly by `SIGKILL`. + +### Side Note: forkserver Requires Importable Task Functions + +During direct (`-c`) testing, `forkserver` workers failed with: + +``` +AttributeError: Can't get attribute 'trivial_extractor' on + )> +``` + +This is expected behaviour: the forkserver re-imports `__main__` to find the +callable, and inline `-c` code has no importable module. In production, +`_EXTRACTOR_POOL` schedules functions defined in +`lib/cuckoo/common/integrations/file_extra_info.py`, which IS a real importable +module, so this is not a blocker. It is noted here because it would surface +immediately in tests that define callables in `__main__` without writing a real +module file. + +--- + +## Decision + +**Chosen context: `fork`** + +Rationale: + +1. **Latency wins on every metric that matters for ephemeral task children.** + `fork` mean (115.7 ms) is 9.8 ms faster than `forkserver` mean (125.5 ms). + More importantly, `forkserver` p95 is 219 ms vs `fork` p95 of 118 ms — a + 100 ms difference — because the forkserver helper process must be created + fresh for every task child. + +2. **Both contexts pass the orphan/killpg safety check** (zero survivors after + `SIGKILL` to the process group). Safety is not a differentiator here. + +3. **Fork-from-warm-child is safe in this context.** The design mandates that + `_EXTRACTOR_POOL` is created immediately after the task child is forked from + the supervisor, before the child spawns any threads. At that point the child + is single-threaded, so `fork` is free of the fork-after-thread deadlock risk. + +4. **Forkserver's isolation benefit is not needed here.** The task child itself + is already isolated (it exits via `os._exit()` and is killed via `killpg`). + The additional process-level isolation that forkserver provides over fork + adds latency cost for no measurable safety gain. + +--- + +## Spike Script + +Paste below for reproducibility. Run with the CAPEv2 venv Python: + +``` +/home/cape/.cache/pypoetry/virtualenvs/capev2-Cxvx6Y4B-py3.12/bin/python3.12 /tmp/extractor_spike.py +``` + +```python +#!/usr/bin/env python3 +""" +Throwaway spike script: measure pebble.ProcessPool creation+first-result latency +for 'forkserver' vs 'fork' start methods, and verify orphan behaviour under killpg. + +Usage: run with the CAPEv2 venv Python, e.g. + /home/cape/.cache/pypoetry/virtualenvs/capev2-Cxvx6Y4B-py3.12/bin/python3.12 /tmp/extractor_spike.py + +Results are printed to stdout. +""" + +import hashlib +import multiprocessing +import os +import signal +import subprocess +import sys +import time + +import pebble + +ITERATIONS = 8 # latency iterations per context inside the child +MAX_WORKERS = 6 # mirrors the planned _EXTRACTOR_POOL size +ORPHAN_SLEEP = 3.0 # seconds the orphan child sleeps while we killpg it + + +# --------------------------------------------------------------------------- +# Trivial extractor-like work function (must be importable at module top level +# for forkserver, which re-imports the module to bootstrap workers). +# --------------------------------------------------------------------------- +def trivial_extractor(payload: bytes) -> str: + """Simulate cheap extractor work: hash a small buffer, spin briefly.""" + # ~200 ms of CPU-ish work without importing CAPE modules + digest = payload + for _ in range(200): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +# --------------------------------------------------------------------------- +# Latency measurement +# --------------------------------------------------------------------------- +def measure_latency(context_name: str, n: int = ITERATIONS) -> dict: + """ + Create a fresh ProcessPool, schedule one task, collect the result, + then close+join. Repeat n times. Returns stats dict. + Must be called from a single-threaded process so 'fork' is safe. + """ + ctx = multiprocessing.get_context(context_name) + payload = os.urandom(256) + samples = [] + errors = 0 + + for i in range(n): + t0 = time.perf_counter() + try: + pool = pebble.ProcessPool(max_workers=MAX_WORKERS, context=ctx) + future = pool.schedule(trivial_extractor, args=(payload,)) + result = future.result(timeout=30) + pool.close() + pool.join() + except Exception as exc: + errors += 1 + print(f" [{context_name}] iter {i}: ERROR {exc}", file=sys.stderr) + continue + elapsed = time.perf_counter() - t0 + samples.append(elapsed) + print(f" [{context_name}] iter {i+1}/{n}: {elapsed*1000:.1f} ms result={result[:8]}...") + + if not samples: + return {"context": context_name, "n": n, "errors": errors} + + samples_sorted = sorted(samples) + n_s = len(samples_sorted) + mean_ms = sum(samples) / n_s * 1000 + median_ms = samples_sorted[n_s // 2] * 1000 + p95_ms = samples_sorted[int(n_s * 0.95)] * 1000 + min_ms = samples_sorted[0] * 1000 + max_ms = samples_sorted[-1] * 1000 + + return { + "context": context_name, + "n": n_s, + "errors": errors, + "mean_ms": round(mean_ms, 2), + "median_ms": round(median_ms, 2), + "p95_ms": round(p95_ms, 2), + "min_ms": round(min_ms, 2), + "max_ms": round(max_ms, 2), + } + + +# --------------------------------------------------------------------------- +# Orphan check +# --------------------------------------------------------------------------- +def orphan_child_main(context_name: str, ready_w: int, done_r: int) -> None: + """ + Child process body for the orphan test: + 1. setsid() to become its own process group leader. + 2. Tell parent our pgid via the write end of the pipe. + 3. Create a sub-pool, schedule a long-sleeping task. + 4. Sleep ORPHAN_SLEEP seconds while parent sends SIGKILL to the group. + 5. os._exit(0) -- should never reach here if killpg works. + """ + os.setsid() + pgid = os.getpgid(0) + + # Signal parent that we're ready and tell it our pgid + os.write(ready_w, f"{pgid}\n".encode()) + os.close(ready_w) + os.close(done_r) + + ctx = multiprocessing.get_context(context_name) + try: + pool = pebble.ProcessPool(max_workers=MAX_WORKERS, context=ctx) + # Schedule tasks that will outlive the killpg + futures = [pool.schedule(trivial_extractor, args=(os.urandom(256),)) + for _ in range(MAX_WORKERS)] + # Sleep long enough for parent to killpg us + time.sleep(ORPHAN_SLEEP * 2) + pool.close() + pool.join() + except Exception: + pass + os._exit(0) + + +def run_orphan_check(context_name: str) -> dict: + """ + Fork a child that creates the sub-pool; the child setsid()s. + Parent waits for ready signal, then killpg(SIGKILL), waits, then + checks pgrep for survivors in the child's pgroup. + """ + ready_r, ready_w = os.pipe() + done_r, done_w = os.pipe() # placeholder + + pid = os.fork() + if pid == 0: + # child + os.close(ready_r) + os.close(done_w) + orphan_child_main(context_name, ready_w, done_r) + os._exit(0) # never reached + + # parent + os.close(ready_w) + os.close(done_r) + os.close(done_w) + + # Read pgid from child + data = b"" + while True: + chunk = os.read(ready_r, 64) + if not chunk: + break + data += chunk + if b"\n" in data: + break + os.close(ready_r) + + child_pgid = int(data.strip()) + print(f" [{context_name}] child pgid={child_pgid}; sleeping {ORPHAN_SLEEP}s to let sub-pool spin up...") + time.sleep(ORPHAN_SLEEP) + + print(f" [{context_name}] sending SIGKILL to pgid {child_pgid}") + try: + os.killpg(child_pgid, signal.SIGKILL) + except ProcessLookupError: + pass # already dead + + # Wait for the direct child to be reaped + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + # Give OS a moment to clean up + time.sleep(0.5) + + # Check for survivors in the process group + pgrep_result = subprocess.run( + ["pgrep", "-g", str(child_pgid)], + capture_output=True, text=True + ) + survivors = pgrep_result.stdout.strip() + survivor_count = len(survivors.splitlines()) if survivors else 0 + + return { + "context": context_name, + "child_pgid": child_pgid, + "survivors": survivors or "(none)", + "survivor_count": survivor_count, + "pass": survivor_count == 0, + } + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +def main(): + print("=" * 60) + print(f"pebble version : {pebble.__version__}") + print(f"Python : {sys.version}") + print(f"PID : {os.getpid()}") + print(f"MAX_WORKERS : {MAX_WORKERS}") + print(f"ITERATIONS : {ITERATIONS}") + print("=" * 60) + + # ----------------------------------------------------------------------- + # Phase 1: Latency measurement + # We fork a clean child for each context so that: + # - 'fork' measurements start from a single-threaded process (as it will + # in production: the task child is forked from the supervisor before + # doing any threaded work). + # - Both measurements run from equivalent baseline states. + # ----------------------------------------------------------------------- + latency_results = {} + for ctx_name in ("fork", "forkserver"): + print(f"\n--- Latency: {ctx_name} ---") + r_pipe, w_pipe = os.pipe() + pid = os.fork() + if pid == 0: + # child + os.close(r_pipe) + import json + result = measure_latency(ctx_name, n=ITERATIONS) + payload = json.dumps(result).encode() + b"\n" + os.write(w_pipe, payload) + os.close(w_pipe) + os._exit(0) + else: + os.close(w_pipe) + import json + data = b"" + while True: + chunk = os.read(r_pipe, 4096) + if not chunk: + break + data += chunk + os.close(r_pipe) + os.waitpid(pid, 0) + result = json.loads(data.strip()) + latency_results[ctx_name] = result + + # ----------------------------------------------------------------------- + # Phase 2: Orphan check + # ----------------------------------------------------------------------- + orphan_results = {} + for ctx_name in ("fork", "forkserver"): + print(f"\n--- Orphan check: {ctx_name} ---") + orphan_results[ctx_name] = run_orphan_check(ctx_name) + + # ----------------------------------------------------------------------- + # Summary + # ----------------------------------------------------------------------- + print("\n" + "=" * 60) + print("LATENCY SUMMARY") + print("=" * 60) + fmt = "{:<12} {:>10} {:>10} {:>10} {:>10} {:>10} {:>8}" + print(fmt.format("context", "mean_ms", "median_ms", "p95_ms", "min_ms", "max_ms", "errors")) + print("-" * 70) + for ctx_name in ("fork", "forkserver"): + r = latency_results[ctx_name] + print(fmt.format( + ctx_name, + r.get("mean_ms", "N/A"), + r.get("median_ms", "N/A"), + r.get("p95_ms", "N/A"), + r.get("min_ms", "N/A"), + r.get("max_ms", "N/A"), + r.get("errors", 0), + )) + + print("\n" + "=" * 60) + print("ORPHAN CHECK SUMMARY") + print("=" * 60) + for ctx_name in ("fork", "forkserver"): + r = orphan_results[ctx_name] + status = "PASS (zero survivors)" if r["pass"] else f"FAIL ({r['survivor_count']} survivors)" + print(f" {ctx_name:<12}: pgid={r['child_pgid']} survivors={r['survivors']} -> {status}") + + print("\n" + "=" * 60) + print("DECISION") + print("=" * 60) + fork_pass = orphan_results["fork"]["pass"] + forksvr_pass = orphan_results["forkserver"]["pass"] + fork_mean = latency_results["fork"].get("mean_ms", float("inf")) + forksvr_mean = latency_results["forkserver"].get("mean_ms", float("inf")) + + if not fork_pass and not forksvr_pass: + decision = "NEITHER context passes orphan check -- escalate." + elif not fork_pass: + decision = f"forkserver (fork failed orphan check)" + elif not forksvr_pass: + decision = f"fork (forkserver failed orphan check)" + elif fork_mean <= forksvr_mean: + delta = forksvr_mean - fork_mean + decision = f"fork (faster by {delta:.1f} ms mean; both pass orphan check)" + else: + delta = fork_mean - forksvr_mean + decision = f"forkserver (faster by {delta:.1f} ms mean; both pass orphan check)" + + print(f" Chosen context: {decision}") + + +if __name__ == "__main__": + main() +``` + +--- + +## Raw Output + +``` +============================================================ +pebble version : 5.1.0 +Python : 3.12.3 (main, Mar 23 2026, 19:04:32) [GCC 13.3.0] +PID : 3435721 +MAX_WORKERS : 6 +ITERATIONS : 8 +============================================================ + +--- Latency: fork --- + [fork] iter 1/8: 112.8 ms result=7ee30c1c... + [fork] iter 2/8: 117.2 ms result=7ee30c1c... + [fork] iter 3/8: 117.0 ms result=7ee30c1c... + [fork] iter 4/8: 118.3 ms result=7ee30c1c... + [fork] iter 5/8: 113.9 ms result=7ee30c1c... + [fork] iter 6/8: 114.2 ms result=7ee30c1c... + [fork] iter 7/8: 118.0 ms result=7ee30c1c... + +--- Latency: forkserver --- + [forkserver] iter 1/8: 219.0 ms result=2596f03c... + [forkserver] iter 2/8: 111.3 ms result=2596f03c... + [forkserver] iter 3/8: 111.9 ms result=2596f03c... + [forkserver] iter 4/8: 112.0 ms result=2596f03c... + [forkserver] iter 5/8: 112.3 ms result=2596f03c... + [forkserver] iter 6/8: 112.2 ms result=2596f03c... + [forkserver] iter 7/8: 112.3 ms result=2596f03c... + +--- Orphan check: fork --- + [fork] child pgid=3435871; sleeping 3.0s to let sub-pool spin up... + [fork] sending SIGKILL to pgid 3435871 + +--- Orphan check: forkserver --- + [forkserver] child pgid=3435882; sleeping 3.0s to let sub-pool spin up... + [forkserver] sending SIGKILL to pgid 3435882 + +============================================================ +LATENCY SUMMARY +============================================================ +context mean_ms median_ms p95_ms min_ms max_ms errors +---------------------------------------------------------------------- +fork 115.66 116.95 118.3 112.85 118.3 0 +forkserver 125.52 112.29 218.97 111.27 218.97 0 + +============================================================ +ORPHAN CHECK SUMMARY +============================================================ + fork : pgid=3435871 survivors=(none) -> PASS (zero survivors) + forkserver : pgid=3435882 survivors=(none) -> PASS (zero survivors) + +============================================================ +DECISION +============================================================ + Chosen context: fork (faster by 9.9 ms mean; both pass orphan check) +``` + +Note: the script uses `os.fork()` to run each context measurement in a clean child +process. As a side effect, forked children inherit the parent's buffered stdout +and may re-print the banner header; the actual latency data is transmitted via a +pipe and aggregated correctly in the parent. The raw output above shows only the +relevant lines per context. From fbb7a20733049245c22b7dc5bb76137d4e25a9c6 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:34:59 +0000 Subject: [PATCH 21/53] fix(extractors): per-call pool with explicit teardown, swept by process-group kill Replace the process-wide _EXTRACTOR_POOL cache and _get_extractor_pool() lazy init with a per-call pebble.ProcessPool using multiprocessing.get_context("fork") (Task 9 SPIKE decision). The pool is created inside generic_file_extractors, used for all extractor scheduling and result collection, then closed and joined in a try/finally block. This ensures worker processes are deterministically torn down before the ephemeral task child exits, and that killpg(child_pgid) from the prefork supervisor sweeps any survivors if per-call teardown fails. Deleted: _EXTRACTOR_POOL, _EXTRACTOR_POOL_LOCK, _get_extractor_pool(), threading import. Added: test_extractor_pool_is_swept_by_killpg confirming fork+setsid+killpg leaves zero survivors. --- .../common/integrations/file_extra_info.py | 183 ++++++++---------- tests/test_extractor_pool_teardown.py | 42 ++++ 2 files changed, 120 insertions(+), 105 deletions(-) create mode 100644 tests/test_extractor_pool_teardown.py diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 8ee073cf2e3..d023461eb2d 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -2,6 +2,7 @@ import hashlib import json import logging +import multiprocessing import os import re import shlex @@ -418,39 +419,6 @@ def _extracted_files_metadata( from lib.cuckoo.common.integrations.utils import run_tool -# Process-wide pebble.ProcessPool reused across every call to -# `generic_file_extractors`. The previous code created and tore down a -# fresh ProcessPool per file (one `with pebble.ProcessPool(...) as pool:` -# block per call). On heavy tasks with 50+ extracted files that's -# 50+ × (subprocess fork + Python interpreter startup + module imports -# + pool teardown) of pure overhead — measured at ~1.2s per file on -# our sandbox (~85s on a 70-file task before any extractor work). -# -# Pebble's ProcessPool is explicitly designed for long-lived reuse: -# its workers respawn after task completion (max_tasks=1 by default -# would tear them down per call, but the default unlimited keeps them -# warm) and a crashed worker is replaced automatically. We lazily -# instantiate a single pool on first use and keep it for the rest of -# the worker process's lifetime — handing each call a slice of the -# already-warm worker pool instead of paying startup costs every time. -_EXTRACTOR_POOL = None -_EXTRACTOR_POOL_LOCK = threading.Lock() - - -def _get_extractor_pool(): - """Return a process-wide shared pebble.ProcessPool, creating it - on first use. Thread-safe.""" - global _EXTRACTOR_POOL - if _EXTRACTOR_POOL is not None: - return _EXTRACTOR_POOL - with _EXTRACTOR_POOL_LOCK: - if _EXTRACTOR_POOL is None: - _EXTRACTOR_POOL = pebble.ProcessPool( - max_workers=int(integration_conf.general.max_workers), - ) - return _EXTRACTOR_POOL - - def generic_file_extractors( file: str, destination_folder: str, @@ -486,85 +454,90 @@ def generic_file_extractors( futures = {} executed_tools = data_dictionary.setdefault("executed_tools", []) - pool = _get_extractor_pool() - # Prefer custom modules over the built-in ones, since only 1 is allowed - # to be the extracted_files_tool. - if extra_info_modules: - for module in extra_info_modules: - func_timeout = int(getattr(module, "timeout", 60)) - funcname = module.__name__.split(".")[-1] + # Per-call pool lifecycle: create, use, then close+join in finally so the + # worker processes are torn down before the task child exits. Using the + # "fork" context (Task 9 SPIKE decision: see + # docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md). + ctx = multiprocessing.get_context("fork") + pool = pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) + try: + # Prefer custom modules over the built-in ones, since only 1 is allowed + # to be the extracted_files_tool. + if extra_info_modules: + for module in extra_info_modules: + func_timeout = int(getattr(module, "timeout", 60)) + funcname = module.__name__.split(".")[-1] + if funcname in executed_tools: + continue + executed_tools.append(funcname) + futures[funcname] = pool.schedule(module.extract_details, args=args, kwargs=kwargs, timeout=func_timeout) + + for extraction_func in file_info_funcs: + funcname = extraction_func.__name__.split(".")[-1] + if ( + not getattr(integration_conf, funcname, {}).get("enabled", False) + and getattr(extraction_func, "enabled", False) is False + ): + continue + if funcname in executed_tools: continue executed_tools.append(funcname) - futures[funcname] = pool.schedule(module.extract_details, args=args, kwargs=kwargs, timeout=func_timeout) - - for extraction_func in file_info_funcs: - funcname = extraction_func.__name__.split(".")[-1] - if ( - not getattr(integration_conf, funcname, {}).get("enabled", False) - and getattr(extraction_func, "enabled", False) is False - ): - continue - if funcname in executed_tools: - continue - executed_tools.append(funcname) + func_timeout = int(getattr(integration_conf, funcname, {}).get("timeout", 60)) + futures[funcname] = pool.schedule(extraction_func, args=args, kwargs=kwargs, timeout=func_timeout) - func_timeout = int(getattr(integration_conf, funcname, {}).get("timeout", 60)) - futures[funcname] = pool.schedule(extraction_func, args=args, kwargs=kwargs, timeout=func_timeout) - # The shared pool stays alive across calls; we no longer call pool.join() - # here because that would shut it down. Each future's per-task timeout - # (set above via pool.schedule timeout=) bounds wall-clock per extractor, - # and the iteration below collects results per-future via .result(). - - for funcname, future in futures.items(): - func_result = None - try: - func_result = future.result() - except concurrent.futures.TimeoutError as err: - timeout = err.args[0] - log.debug("Function: %s took longer than %d seconds", funcname, timeout) - continue - except TypeError as err: - log.debug("TypeError on getting results: %s", str(err)) - except Exception as err: - log.exception("file_extra_info: %s", err) - continue - if not func_result: - continue - extraction_result = func_result.get("result") - if extraction_result is None: - continue - tempdir = extraction_result.get("tempdir") - if extraction_result.get("data_dictionary"): - data_dictionary.update(extraction_result["data_dictionary"]) - extraction_result.pop("data_dictionary") - """ - if extraction_result.get("parent_sample"): - results.setdefault("info", {}).setdefault("parent_sample", {}) - results["info"]["parent_sample"] = extraction_result["parent_sample"] - extraction_result.pop("parent_sample") - """ - try: - extracted_files = extraction_result.get("extracted_files", []) - if not extracted_files: + for funcname, future in futures.items(): + func_result = None + try: + func_result = future.result() + except concurrent.futures.TimeoutError as err: + timeout = err.args[0] + log.debug("Function: %s took longer than %d seconds", funcname, timeout) continue - old_tool_name = data_dictionary.get("extracted_files_tool") - new_tool_name = extraction_result["tool_name"] - if old_tool_name: - log.debug("Files already extracted from %s by %s. Also extracted with %s", file, old_tool_name, new_tool_name) + except TypeError as err: + log.debug("TypeError on getting results: %s", str(err)) + except Exception as err: + log.exception("file_extra_info: %s", err) continue - metadata = _extracted_files_metadata(tempdir, destination_folder, files=extracted_files, results=results) - data_dictionary.setdefault("selfextract", {}) - data_dictionary["selfextract"][new_tool_name] = { - "extracted_files": metadata, - "extracted_files_time": func_result["took_seconds"], - "password": extraction_result.get("password", ""), - } - finally: - if tempdir: - # ToDo doesn't work - shutil.rmtree(tempdir, ignore_errors=True) + if not func_result: + continue + extraction_result = func_result.get("result") + if extraction_result is None: + continue + tempdir = extraction_result.get("tempdir") + if extraction_result.get("data_dictionary"): + data_dictionary.update(extraction_result["data_dictionary"]) + extraction_result.pop("data_dictionary") + """ + if extraction_result.get("parent_sample"): + results.setdefault("info", {}).setdefault("parent_sample", {}) + results["info"]["parent_sample"] = extraction_result["parent_sample"] + extraction_result.pop("parent_sample") + """ + try: + extracted_files = extraction_result.get("extracted_files", []) + if not extracted_files: + continue + old_tool_name = data_dictionary.get("extracted_files_tool") + new_tool_name = extraction_result["tool_name"] + if old_tool_name: + log.debug("Files already extracted from %s by %s. Also extracted with %s", file, old_tool_name, new_tool_name) + continue + metadata = _extracted_files_metadata(tempdir, destination_folder, files=extracted_files, results=results) + data_dictionary.setdefault("selfextract", {}) + data_dictionary["selfextract"][new_tool_name] = { + "extracted_files": metadata, + "extracted_files_time": func_result["took_seconds"], + "password": extraction_result.get("password", ""), + } + finally: + if tempdir: + # ToDo doesn't work + shutil.rmtree(tempdir, ignore_errors=True) + finally: + pool.close() + pool.join() def _generic_post_extraction_process(file: str, tool_name: str, decoded: str) -> SuccessfulExtractionReturnType: diff --git a/tests/test_extractor_pool_teardown.py b/tests/test_extractor_pool_teardown.py new file mode 100644 index 00000000000..2465573ec8b --- /dev/null +++ b/tests/test_extractor_pool_teardown.py @@ -0,0 +1,42 @@ +import os +import time +import multiprocessing as mp + + +def _busy(_): + time.sleep(60) + + +def test_extractor_pool_is_swept_by_killpg(tmp_path): + """A child that creates the extractor pool and is killpg'd must leave no survivors.""" + marker = str(tmp_path / "epool_child") + + def child(): + os.setsid() + open(marker, "w").close() + import pebble + ctx = mp.get_context("fork") # spike decision: fork from warm child + pool = pebble.ProcessPool(max_workers=2, context=ctx) + for _ in range(2): + pool.schedule(_busy, args=(1,), timeout=60) + time.sleep(60) + + pid = os.fork() + if pid == 0: + try: + child() + finally: + os._exit(0) + while not os.path.exists(marker): + time.sleep(0.05) + time.sleep(1) + os.killpg(pid, 15) + time.sleep(2) + # Reap the direct child's zombie so pgrep only sees actual survivors. + try: + os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + pass + import subprocess + out = subprocess.run(["pgrep", "-g", str(pid)], capture_output=True, text=True) + assert out.stdout.strip() == "", "extractor sub-pool survived killpg" From db07ede9e6f52f0781f88452cbfc902b8756ae41 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:44:42 +0000 Subject: [PATCH 22/53] revert: drop out-of-scope static_file_info elif accidentally bundled in Task 10 Co-Authored-By: Claude Sonnet 4.6 --- lib/cuckoo/common/integrations/file_extra_info.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index d023461eb2d..0447710443b 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -185,14 +185,6 @@ def static_file_info( if "pe" not in data_dictionary: with PortableExecutable(file_path) as pe: data_dictionary["pe"] = pe.run(task_id) - elif not data_dictionary["pe"].get("digital_signers") and data_dictionary["pe"].get("guest_signers", {}).get("aux_signers"): - # Only re-run cert extraction when DigiSig.json confirms the file is signed - # (aux_signers non-empty) but digital_signers is empty — avoids re-parsing - # every unsigned PE on reprocess. - with PortableExecutable(file_path) as pe: - data_dictionary["pe"]["digital_signers"] = pe.get_digital_signers(pe.pe) - if not data_dictionary["pe"]["guest_signers"].get("aux_sha1"): - data_dictionary["pe"]["guest_signers"] = pe.get_guest_digital_signers(task_id) if HAVE_FLARE_CAPA and "flare_capa" not in data_dictionary: # https://github.com/mandiant/capa/issues/2620 From f7b6daf5f08b40c191d1bad200b2e61e8f309993 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:49:37 +0000 Subject: [PATCH 23/53] docs(processor): A/B runbook + mark redesign implementation status Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-05-27-processor-engine-ab-runbook.md | 276 ++++++++++++++++++ ...7-processor-concurrency-redesign-design.md | 2 + 2 files changed, 278 insertions(+) create mode 100644 docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md diff --git a/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md b/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md new file mode 100644 index 00000000000..cf66abf5a54 --- /dev/null +++ b/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md @@ -0,0 +1,276 @@ +# Processor Engine A/B Runbook + +**Date:** 2026-05-27 +**Scope:** How to flip between `pebble` and `prefork` processing engines, what to measure, how to declare a winner, when to retire the `-mc 0` stopgap. +**Status:** Ready for A/B — branch `feature/processor-prefork-engine` implementation complete. +**Audience:** Engineer running the A/B in production. + +--- + +## Preconditions + +Before running the A/B, confirm: + +1. Branch `feature/processor-prefork-engine` is deployed. Implementation spans commits `47576a166` through `beb829e67` (14 commits: `TaskSource`, engine registry/base, `run_task` adapter, `PebbleEngine`, `PreforkEngine` scaffolding, fork-per-task execution, wall-clock timeout + `killpg`, extractor sub-pool rewrite, and a docs revert). + +2. The existing stopgap drop-in is in place at `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf`: + ``` + # STOPGAP (2026-05-27): maxtasksperchild=0 disables worker recycling. + # Worker recycle at max_tasks>0 deadlocks in multiprocessing _exit_function + # joining the nested _EXTRACTOR_POOL children. See cape-processor redesign. + # Tradeoff: no per-worker memory reclaim; monitor RSS / pair with periodic restart. + [Service] + ExecStart= + ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 + ``` + This drop-in does **not** pass `--engine`, so the pebble engine (argparse default) is active. + +3. `cape-processor.service` is running and healthy (check for recent `Reports generation completed for Task #N` lines in `/opt/CAPEv2/log/process.log`). + +4. You have `sudo` on this host. + +--- + +## Database Note (important) + +The A/B runs against the **production PostgreSQL database**. Both engines share the module-level `db = Database()` instance configured from `conf/cuckoo.conf` (`postgresql://...`). There is no separate staging database path. + +The `sqlite://` database in `tests/conftest.py` is the unit-test fixture only. It has no production code path and is completely irrelevant to the A/B. + +--- + +## Flipping Engines + +### To enable the prefork engine + +Edit the drop-in at `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf`. Add `--engine prefork` to the `ExecStart` line: + +``` +# STOPGAP (2026-05-27): maxtasksperchild=0 disables worker recycling. +# Worker recycle at max_tasks>0 deadlocks in multiprocessing _exit_function +# joining the nested _EXTRACTOR_POOL children. See cape-processor redesign. +# Tradeoff: no per-worker memory reclaim; monitor RSS / pair with periodic restart. +# NOTE: -mc 0 is a pebble-specific workaround; it is harmless but meaningless +# under prefork (different lifecycle, no worker-recycle exit path). +[Service] +ExecStart= +ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 --engine prefork +``` + +Then reload and restart: + +```bash +sudo systemctl daemon-reload +sudo systemctl restart cape-processor.service +``` + +Verify it started with the right engine — look for this log line within a few seconds of restart: + +```bash +sudo journalctl -u cape-processor.service -f --since "1 minute ago" +grep -m1 "engine" /opt/CAPEv2/log/process.log | tail -5 +``` + +Expected: a log line referencing `PreforkEngine` or `engine=prefork` at startup (exact text from `utils/process.py`). + +### To switch back to pebble + +Remove `--engine prefork` from the `ExecStart` line (or explicitly add `--engine pebble`), then: + +```bash +sudo systemctl daemon-reload +sudo systemctl restart cape-processor.service +``` + +--- + +## Rollback + +If at any point you need to revert to the exact pre-A/B pebble configuration, restore the drop-in to its original content (no `--engine` flag): + +``` +[Service] +ExecStart= +ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 +``` + +Then `sudo systemctl daemon-reload && sudo systemctl restart cape-processor.service`. This fully restores the pebble+`-mc 0` baseline. + +--- + +## Metrics to Compare + +Run each engine for at least **48 hours** (or 500 completed tasks, whichever comes first) under the same workload before drawing conclusions. Collect all four metrics for both engines. + +### 1. Throughput (tasks/min) + +Parse `Reports generation completed` completions from the log, compute rate per minute: + +```bash +grep "Reports generation completed" /opt/CAPEv2/log/process.log \ + | awk '{print $1, $2}' \ + | awk -F'[: ]' '{print $1"T"$2":"$3":00"}' \ + | sort | uniq -c \ + | awk '{printf "%s completions=%d\n", $2, $1}' +``` + +Or a simpler rolling summary over the last hour: + +```bash +SINCE=$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S') +awk -v since="$SINCE" '$0 >= since && /Reports generation completed/' \ + /opt/CAPEv2/log/process.log | wc -l +``` + +Target: prefork throughput within 5% of pebble. Any significant regression needs investigation before declaring prefork the winner. + +### 2. Wedge incidents + +A "wedge" is a period of >5 minutes with no `Reports generation completed` lines while tasks are still scheduled. The `-mc 0` stopgap eliminated wedges for pebble; prefork should also have zero. + +Check for silent gaps in the last 24 hours: + +```bash +grep "Reports generation completed" /opt/CAPEv2/log/process.log \ + | awk '{print $1" "$2}' \ + | awk 'NR>1 { + cmd="date -d \""$1"\" +%s"; cmd | getline ts; close(cmd) + cmd2="date -d \""prev"\" +%s"; cmd2 | getline pts; close(cmd2) + gap = ts - pts + if (gap > 300) printf "GAP %d s ending at %s\n", gap, $1 + prev=$1" "$2 + } + NR==1 {prev=$1" "$2}' +``` + +Alternatively, watch the heartbeat log line that `PreforkEngine` emits periodically (`in_flight=N oldest=Xs`). A heartbeat without completions for >5 minutes while `in_flight > 0` is an early wedge indicator. + +**Decision criterion:** any wedge under either engine is a failure for that engine. Both should show zero over the observation window. + +### 3. Peak and sustained RSS + +Sum the RSS of all python workers under the processor's main PID at 5-minute intervals. First find the service's MainPID: + +```bash +MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) +echo "MainPID: $MAIN_PID" +``` + +Then poll (run in a loop, or via cron, or as a one-shot snapshot): + +```bash +# One snapshot (GB): +MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) +ps --no-headers -o rss --ppid "$MAIN_PID" \ + | awk '{sum+=$1} END {printf "RSS %.2f GB (%d workers)\n", sum/1048576, NR}' +``` + +For a continuous 5-minute polling log: + +```bash +while true; do + MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) + RSS=$(ps --no-headers -o rss --ppid "$MAIN_PID" \ + | awk '{sum+=$1} END {printf "%.2f", sum/1048576}') + echo "$(date '+%Y-%m-%dT%H:%M:%S') RSS=${RSS}GB" + sleep 300 +done | tee /tmp/rss-log.txt +``` + +**Expected behavior difference:** +- Pebble with `-mc 0`: RSS grows gradually over time (workers never recycle, C-lib leaks accumulate). This is the known trade-off of the `-mc 0` stopgap. +- Prefork: RSS should be roughly steady (each task child exits via `os._exit()`, reclaiming its memory per-task). A monotonically growing RSS trend under prefork is a regression. + +**Decision criterion:** prefork RSS should not trend upward continuously over 48 hours. Pebble RSS growing is expected and acceptable (it is a known stopgap trade-off). A periodic `RuntimeMaxSec` systemd restart can bound pebble RSS if needed. + +### 4. Orphan count after induced timeouts + +This metric requires inducing a task that exceeds `-pt 900` (or temporarily reducing `-pt` for a test). It validates that `killpg` in prefork actually cleans up the entire process tree and leaves no grandchildren reparented to init. + +**Procedure:** + +1. Before the test, note the current `in_flight` from logs or the heartbeat line. +2. Submit (or identify) a task known to run longer than the current `-pt` value. Alternatively: temporarily set `-pt 60` in the drop-in to trigger timeouts faster, then restore. +3. When a `Processing timeout` line appears in `process.log`, immediately check for orphans: + +```bash +# Check for python processes reparented to init (PPID=1): +pgrep -P 1 | xargs -I{} ps -o pid,ppid,cmd -p {} 2>/dev/null \ + | grep -i python + +# Or check for any surviving members of the task's process group. +# Get the task child PID from the timeout log line first, then: +# (The supervisor logs the child PID and PGID in the SIGTERM/SIGKILL log lines.) +TASK_PGID= +pgrep -g "$TASK_PGID" +``` + +**Expected results:** +- Prefork: `pgrep` returns nothing. The `killpg` swept the entire process group (task child + extractor sub-pool workers). Zero orphans. +- Pebble: `pgrep` may return orphaned grandchildren. Pebble's `stop_process` sends a signal to one worker PID; it does not sweep the extractor sub-pool children, which may reparent to init. + +**Decision criterion:** prefork must show zero orphans after any induced timeout. Non-zero is a regression in the primary correctness guarantee of the new engine. + +--- + +## Decision Criterion + +Declare prefork the **winner** if, over the observation window (48 hours or 500 tasks): + +- Throughput within 5% of pebble (tasks/min). +- Zero wedge incidents (same as pebble with `-mc 0`). +- RSS steady-state, not trending upward. +- Zero orphaned processes after any induced or naturally-occurring timeout. + +Declare pebble the **winner** (and treat prefork as needing more work) if any of: + +- Prefork throughput is >5% lower than pebble. +- Prefork wedges (even once). +- Prefork RSS trends upward monotonically over 48 hours. +- Prefork leaves orphans after a timeout. + +If there is a prefork regression, file an issue, roll back to pebble (one `systemctl` command — see Rollback above), and investigate before re-attempting the A/B. + +--- + +## Stopgap Retirement + +The `-mc 0` flag in the drop-in exists solely to work around the pebble worker-recycle deadlock (`multiprocessing._exit_function` joining the nested `_EXTRACTOR_POOL` children). It is a **pebble-specific stopgap**. + +Under `--engine prefork`, the `-mc 0` flag is meaningless (prefork has no worker-recycle exit path) but harmless. It does not need to be removed immediately. + +**When to remove `-mc 0`:** once `--engine prefork` is made the argparse default in `utils/process.py` (a one-line code change flipping the `--engine` default from `pebble` to `prefork`, done after A/B declares prefork the winner), the pebble engine will no longer be the active engine on startup and the `-mc 0` stopgap becomes dead weight. At that point: + +1. Remove the `-mc 0` argument from the drop-in's `ExecStart` line. +2. The drop-in itself can eventually be deleted entirely once `--engine prefork` is also baked into the unit file's `ExecStart` (or once the old pebble path is removed from the codebase). + +Do not remove `-mc 0` earlier than this. It is harmless under prefork and removing it prematurely would leave the pebble engine unprotected if someone ever toggles back. + +--- + +## Open Follow-ups + +These items are not part of the A/B itself but are outstanding work to track: + +1. **Uncommitted `resolver_pool` lazy fix.** `lib/cuckoo/common/cleaners_utils.py` contains the `_LazyThreadPool` fix that defers `ThreadPool(50)` creation to the first call (reduces supervisor thread count 73 → 20, making prefork forks clean). This fix is in the working tree on `feature/processor-prefork-engine` but has not been committed as a standalone commit. It should be committed on a separate branch or PR — see §9 of the spec (`docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md`). + +2. **Proctitle/formatter cleanup deferral.** Task 4 (`d944dcc4d`) restored per-task `proctitle` and logging formatter cleanup in `run_task`. The prefork engine's per-task children already have isolated address spaces (no shared log handlers), so the cleanup is technically redundant under prefork — but it was kept for parity and correctness under the pebble path. A future cleanup could remove it from `run_task` and make it engine-specific if it shows measurable cost. + +3. **`--tasks-per-child` knob.** The design spec (§4.2) documents this as a YAGNI future knob: recycling a prefork child across N>1 tasks requires a supervisor→child dispatch channel, which was deliberately not built. If the measured per-task fork cost ever becomes material, this is the documented path to amortize it. Current measured cost: ~0.35 s/task (~1.6% of p50 22 s task) — not material. + +4. **pyattck `techniques` recompute.** Observed burning CPU in `_get_relationship_objects` on each access. The fix (pre-compute in the warm base so children inherit the cached result via COW) was deferred as an A/B-measured optimization. If the A/B reveals CPU spikes on prefork attributable to pyattck, this is the first place to look. + +--- + +## Related Artifacts + +| Artifact | Path | +|---|---| +| Concurrency redesign spec | `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` | +| Implementation plan (11 tasks) | `docs/superpowers/plans/2026-05-27-processor-prefork-engine.md` | +| Extractor sub-pool spike (fork vs forkserver) | `docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md` | +| Stopgap drop-in (live) | `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf` | +| Processor entry point | `utils/process.py` | +| Engine implementations | `utils/engines/pebble_engine.py`, `utils/engines/prefork_engine.py` | +| Engine registry | `utils/engines/__init__.py` | +| Extractor sub-pool (rewritten in Task 10) | `lib/cuckoo/common/integrations/file_extra_info.py` | diff --git a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md index 90837a70690..f49700f544a 100644 --- a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md +++ b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md @@ -5,6 +5,8 @@ **Branch:** `feature/processor-prefork-engine` (off `feature/guac-auth-evtx-snapshots`) **Status:** Design — awaiting review before implementation planning +**Status (2026-05-28):** implementation complete on branch `feature/processor-prefork-engine` (commits `47576a166` through `beb829e67`); A/B runbook at `docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md`. + --- ## 1. Background & Problem From 2dd8627762c51f6f0e131e1c61f91fe395dd0d8a Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 16:58:53 +0000 Subject: [PATCH 24/53] =?UTF-8?q?fix(processor):=20final=20polish=20?= =?UTF-8?q?=E2=80=94=20freespace=20backpressure=20for=20prefork;=20complet?= =?UTF-8?q?ion=20log;=20runbook=20paths;=20decoded=20waitpid=20status;=20w?= =?UTF-8?q?ire=20max=5Fanalysis=5Fcount=20to=20both=20engines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-05-27-processor-engine-ab-runbook.md | 4 ++-- lib/cuckoo/core/processing_engine/prefork.py | 21 ++++++++++++++++--- utils/process.py | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md b/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md index cf66abf5a54..75612a183e5 100644 --- a/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md +++ b/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md @@ -271,6 +271,6 @@ These items are not part of the A/B itself but are outstanding work to track: | Extractor sub-pool spike (fork vs forkserver) | `docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md` | | Stopgap drop-in (live) | `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf` | | Processor entry point | `utils/process.py` | -| Engine implementations | `utils/engines/pebble_engine.py`, `utils/engines/prefork_engine.py` | -| Engine registry | `utils/engines/__init__.py` | +| Engine implementations | `lib/cuckoo/core/processing_engine/pebble.py`, `lib/cuckoo/core/processing_engine/prefork.py` | +| Engine registry | `lib/cuckoo/core/processing_engine/__init__.py` | | Extractor sub-pool (rewritten in Task 10) | `lib/cuckoo/common/integrations/file_extra_info.py` | diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 722eeee25e2..d1964d445be 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -8,6 +8,8 @@ import threading import time +from lib.cuckoo.common.config import Config +from lib.cuckoo.common.constants import CUCKOO_ROOT from lib.cuckoo.core.processing_engine.base import ProcessingEngine log = logging.getLogger(__name__) @@ -88,9 +90,17 @@ def _reap(self): if child.timed_out: continue # already marked failed during timeout enforcement ok = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 - if not ok: - log.warning("prefork: task %d (pid %d) abnormal exit status=%d -> FAILED_PROCESSING", - child.task_id, pid, status) + if ok: + log.info("Reports generation completed for Task #%d", child.task_id) + else: + if os.WIFEXITED(status): + detail = "exit=%d" % os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + detail = "signal=%d" % os.WTERMSIG(status) + else: + detail = "status=%d" % status + log.warning("prefork: task %d (pid %d) abnormal %s -> FAILED_PROCESSING", + child.task_id, pid, detail) self.source.mark_failed(child.task_id) def _enforce_timeouts(self): @@ -128,12 +138,17 @@ def _escalate_kills(self): child.kill_deadline = None def run(self): + cfg = Config() count = 0 last_hb = 0.0 while True: self._reap() self._enforce_timeouts() self._escalate_kills() + if cfg.cuckoo.freespace_processing: + from lib.cuckoo.common.cleaners_utils import free_space_monitor + dir_path = os.path.join(CUCKOO_ROOT, "storage", "analyses") + free_space_monitor(dir_path, processing=True) if self.max_count and count >= self.max_count and not self._inflight: return free = self.parallel - len(self._inflight) diff --git a/utils/process.py b/utils/process.py index e9ff0b730a3..c2d6a7c49d0 100644 --- a/utils/process.py +++ b/utils/process.py @@ -446,9 +446,9 @@ def autoprocess( parallel=parallel, timeout=processing_timeout, ) + eng.max_count = cfg.cuckoo.max_analysis_count if engine == "pebble": eng.max_tasks = maxtasksperchild - eng.max_count = cfg.cuckoo.max_analysis_count eng.run() From 039fd72ea2de7968743cd8e51d112aca56c54f3c Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 17:45:05 +0000 Subject: [PATCH 25/53] feat(processor): prefork invariant also checks /proc/self/task (catches native C-extension threads like STPyV8 V8 workers) --- lib/cuckoo/core/processing_engine/prefork.py | 13 ++++++++--- tests/test_prefork_engine.py | 23 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index d1964d445be..661c8271ff3 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -38,11 +38,18 @@ def __init__(self, task_fn, worker_init, source, parallel, timeout, self._inflight = {} # pid -> _Child def _assert_single_threaded(self): - n = threading.active_count() - if n != 1: + py_count = threading.active_count() + try: + kernel_count = len(os.listdir("/proc/self/task")) + except OSError: + kernel_count = None # /proc unavailable; fall back to Python check only + + if py_count != 1 or (kernel_count is not None and kernel_count != 1): raise RuntimeError( "prefork supervisor must be single-threaded before fork; " - "active_count=%d (a thread/Mongo client/pool leaked into the supervisor)" % n) + "python active_count=%d kernel /proc/self/task=%s " + "(a thread / Mongo client / pool / native C extension like STPyV8 " + "leaked into the supervisor)" % (py_count, kernel_count)) def _inflight_task_ids(self): return {c.task_id for c in self._inflight.values()} diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index 6eb057dd963..88ebbcf33b2 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -39,6 +39,29 @@ def test_single_threaded_invariant_raises_when_extra_thread(db): t.join() +def test_single_threaded_invariant_raises_on_kernel_thread_count(db, monkeypatch): + """Native C-extension threads (e.g. STPyV8's V8 worker pool) show up in + /proc/self/task but NOT in threading.active_count(). The supervisor must + catch them — otherwise it forks from a multi-kernel-thread state.""" + eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, + source=TaskSource(db), parallel=2, timeout=30) + + # Simulate a process with 1 Python thread but several kernel threads + # (e.g., STPyV8 spawned 16 V8 workers at import; Python sees only main). + import os as _os + real_listdir = _os.listdir + + def fake_listdir(path): + if path == "/proc/self/task": + return ["1", "2", "3", "4"] # 4 fake kernel threads + return real_listdir(path) + + monkeypatch.setattr(os, "listdir", fake_listdir) + import pytest + with pytest.raises(RuntimeError, match="single-threaded"): + eng._assert_single_threaded() + + def test_normal_task_runs_and_status_set_by_child(db_file, temp_pe32): with db_file.session.begin(): tid = db_file.add_path(temp_pe32) From d271993525b8f71d428c0546473cb8c3b912b22b Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 17:47:32 +0000 Subject: [PATCH 26/53] fix(peepdf): lazy-import STPyV8/V8 bindings (was spawning 16 native threads at module load, breaking prefork single-threaded invariant) Co-Authored-By: Claude Sonnet 4.6 --- lib/cuckoo/common/integrations/peepdf.py | 34 +++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/cuckoo/common/integrations/peepdf.py b/lib/cuckoo/common/integrations/peepdf.py index 8c6770d3989..3d4d082e23a 100644 --- a/lib/cuckoo/common/integrations/peepdf.py +++ b/lib/cuckoo/common/integrations/peepdf.py @@ -8,14 +8,29 @@ from lib.cuckoo.common.utils import convert_to_printable -try: - from peepdf.JSAnalysis import analyseJS, isJavascript - from peepdf.PDFCore import PDFParser +# Lazy-loaded peepdf bindings. The peepdf library imports STPyV8 (V8 bindings) +# at module load time, which spawns 16 native V8 worker threads in the +# importing process. We do NOT want those threads in the supervisor — see +# PreforkEngine._assert_single_threaded. _load_peepdf() defers the import +# until the first peepdf_parse() call (which runs in a worker / forked child, +# not the supervisor), so the supervisor stays single-threaded at the kernel level. - HAVE_PEEPDF = True -except ImportError: - HAVE_PEEPDF = False - print("OPTIONAL! Missed dependency: poetry run pip install peepdf-3") +_PEEPDF_BINDINGS = None # None = not probed; tuple = loaded; () = unavailable + + +def _load_peepdf(): + """Import peepdf on demand, cache the bindings, return (PDFParser, isJavascript, analyseJS) or () if peepdf is unavailable.""" + global _PEEPDF_BINDINGS + if _PEEPDF_BINDINGS is not None: + return _PEEPDF_BINDINGS + try: + from peepdf.JSAnalysis import analyseJS, isJavascript + from peepdf.PDFCore import PDFParser + _PEEPDF_BINDINGS = (PDFParser, isJavascript, analyseJS) + except ImportError: + print("OPTIONAL! Missed dependency: poetry run pip install peepdf-3") + _PEEPDF_BINDINGS = () + return _PEEPDF_BINDINGS log = logging.getLogger(__name__) @@ -65,9 +80,10 @@ def _set_base_uri(pdf): def peepdf_parse(filepath: str, pdfresult: Dict[str, Any]) -> Dict[str, Any]: """Extract JavaScript from PDF objects.""" - - if not HAVE_PEEPDF: + bindings = _load_peepdf() + if not bindings: return pdfresult + PDFParser, isJavascript, analyseJS = bindings log.debug("About to parse with PDFParser") parser = PDFParser() From 310f5a6bfd58dcd99b40889aee7cfed971300051 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 28 May 2026 17:51:18 +0000 Subject: [PATCH 27/53] fix(processing): lazy-import imagehash to keep prefork supervisor kernel-single-threaded imagehash (via PIL/Pillow C extensions) spawns +15 native kernel threads at import time, breaking PreforkEngine._assert_single_threaded which now checks both Python threads and /proc/self/task. Move the import inside _load_imagehash() with the same deferred-binding pattern used for peepdf/STPyV8. Also includes mtime-ordered reindex_screenshots and zxing-cpp QR support already in working tree. Co-Authored-By: Claude Sonnet 4.6 --- .../common/integrations/file_extra_info.py | 2 - modules/processing/deduplication.py | 98 +++++++++++++++---- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 0447710443b..c30354b3967 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -12,8 +12,6 @@ # from contextlib import suppress from typing import Any, DefaultDict, List, Optional, Set -import threading - import pebble from lib.cuckoo.common.config import Config diff --git a/modules/processing/deduplication.py b/modules/processing/deduplication.py index 52c0358076c..c75daf606b3 100644 --- a/modules/processing/deduplication.py +++ b/modules/processing/deduplication.py @@ -9,13 +9,28 @@ log = logging.getLogger() -HAVE_IMAGEHASH = False -try: - import imagehash +# Lazy-loaded imagehash bindings — defer C-extension import that spawns native +# threads at module load time. See PreforkEngine._assert_single_threaded. +# imagehash transitively loads PIL/Pillow C extensions which spawn ~15 kernel +# threads on import; deferring to first use keeps the prefork supervisor +# single-threaded so it can safely fork workers. +_IMAGEHASH_BINDINGS = None # None = not probed; module = loaded; False = unavailable + + +def _load_imagehash(): + """Import imagehash on demand, cache the result. Returns the imagehash module or False.""" + global _IMAGEHASH_BINDINGS + if _IMAGEHASH_BINDINGS is not None: + return _IMAGEHASH_BINDINGS + try: + import imagehash as _imagehash + + _IMAGEHASH_BINDINGS = _imagehash + except ImportError: + log.error("Missed dependency: poetry run pip install ImageHash") + _IMAGEHASH_BINDINGS = False + return _IMAGEHASH_BINDINGS - HAVE_IMAGEHASH = True -except ImportError: - log.error("Missed dependency: poetry run pip install ImageHash") HAVE_CV2 = False try: @@ -25,6 +40,14 @@ except ImportError: print("Missed dependency: poetry run pip install opencv-python") +HAVE_ZXING = False +try: + import zxingcpp + + HAVE_ZXING = True +except ImportError: + pass + try: from PIL import Image @@ -37,7 +60,12 @@ def reindex_screenshots(shots_path): - for i, cur_basename in enumerate(sorted(os.listdir(shots_path))): + # Sort by modification time so interleaved capture sources (guest JPGs + # with 4-digit padded names and host PNGs with unpadded integer names) + # end up in true chronological order after renumbering. + entries = os.listdir(shots_path) + entries.sort(key=lambda f: os.path.getmtime(os.path.join(shots_path, f))) + for i, cur_basename in enumerate(entries): extension = os.path.splitext(cur_basename)[-1] new_basename = "%s%s" % (str(i).rjust(4, "0"), extension) log.debug("renaming %s to %s", cur_basename, new_basename) @@ -46,23 +74,59 @@ def reindex_screenshots(shots_path): os.rename(old_path, new_path) -def handle_qr_codes(image_path): - if not HAVE_CV2: - return None +def _qr_decode_zxing(image_path): + """Decode QR codes using zxing-cpp. Supports multiple barcodes per image.""" + try: + img = Image.open(image_path) + results = zxingcpp.read_barcodes(img) + urls = [] + for result in results: + text = result.text + if text and "://" in text[:10]: + urls.append(text) + return urls + except Exception as e: + log.error("zxing-cpp error on %s: %s", image_path, e) + return [] + + +def _qr_decode_cv2(image_path): + """Decode QR codes using OpenCV. Single barcode per image.""" try: - # cv2.imread handles file path directly img = cv2.imread(image_path) if img is None: - return None + return [] detector = cv2.QRCodeDetector() extracted, points, straight_qrcode = detector.detectAndDecode(img) if extracted and "://" in extracted[:10]: - return extracted + return [extracted] except Exception as e: log.error("Error detecting QR in %s: %s", image_path, e) + return [] + + +def handle_qr_codes(image_path): + if HAVE_ZXING: + urls = _qr_decode_zxing(image_path) + if urls: + return urls[0] + return None + if HAVE_CV2: + urls = _qr_decode_cv2(image_path) + if urls: + return urls[0] return None +def handle_qr_codes_all(image_path): + """Return all QR code URLs found in an image.""" + if HAVE_ZXING: + return _qr_decode_zxing(image_path) + if HAVE_CV2: + return _qr_decode_cv2(image_path) + return [] + + class Deduplicate(Processing): """Deduplicate screenshots.""" @@ -115,7 +179,8 @@ def run(self): shots = [] shots_path = os.path.join(self.analysis_path, "shots") - if not path_exists(shots_path) or not HAVE_IMAGEHASH: + imagehash = _load_imagehash() + if not path_exists(shots_path) or not imagehash: return shots hashmethod = self.options.get("hashmethod", "ahash") @@ -141,13 +206,12 @@ def hashfunc(img): screenshots = sorted(self.deduplicate_images(userpath=shots_path, hashfunc=hashfunc)) shots = [re.sub(r"\.(png|jpg)$", "", screenshot) for screenshot in screenshots] - if HAVE_CV2: + if HAVE_ZXING or HAVE_CV2: qr_urls = set() for img_name in os.listdir(shots_path): if not img_name.lower().endswith((".jpg", ".png")): continue - url = handle_qr_codes(os.path.join(shots_path, img_name)) - if url: + for url in handle_qr_codes_all(os.path.join(shots_path, img_name)): qr_urls.add(url) if qr_urls: From e7b6cc72d004127f5893496979505d6a9390f25d Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 20:50:20 +0000 Subject: [PATCH 28/53] perf(extractors): shared per-child extractor pool under prefork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prefork child forks once per task, runs every extracted file, then os._exit()s. generic_file_extractors runs once per file, so a per-call pool re-forks worker processes N times per task (~1.2s/file on heavy tasks) — the exact cost upstream's shared _EXTRACTOR_POOL eliminated. Reclaim it safely: the prefork child opts into a single shared pool (enable_shared_extractor_pool) reused across all files and torn down once (shutdown_shared_extractor_pool) before exit. Safe here — unlike pebble worker recycling, which deadlocks joining a persistent nested pool in _exit_function — because the child never cleanly joins mid-life; it exits and any straggler is swept by the supervisor's process-group kill. Pebble keeps the per-call pool. --- .../common/integrations/file_extra_info.py | 80 +++++++++++-- lib/cuckoo/core/processing_engine/prefork.py | 11 ++ tests/test_shared_extractor_pool.py | 113 ++++++++++++++++++ 3 files changed, 196 insertions(+), 8 deletions(-) create mode 100644 tests/test_shared_extractor_pool.py diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index c30354b3967..3c83f8014b0 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -408,6 +408,68 @@ def _extracted_files_metadata( from lib.cuckoo.common.integrations.utils import run_tool +# Extractor sub-pool lifecycle. +# +# `generic_file_extractors` runs once per extracted file, so a task with N +# payloads calls it N times. Two strategies: +# +# * Per-call (pebble default): create a fork-context pool, use it, then +# close+join in a finally. Safe under the pebble A/B control because the +# long-lived worker never has to join a persistent nested pool (which is +# what deadlocks in multiprocessing `_exit_function` on worker recycle). +# +# * Shared-per-child (prefork): the prefork supervisor forks one child per +# task; the child sets its own process group, runs every file, then +# `os._exit()`. There it is safe — and much faster — to build ONE pool and +# reuse it across all N files, tearing it down a single time when the task +# finishes. This reclaims the per-file fork+interpreter-startup cost that a +# per-call pool pays N times (~1.2s/file on heavy tasks). The prefork child +# opts in via `enable_shared_extractor_pool()` and cleans up via +# `shutdown_shared_extractor_pool()`; any stragglers are swept by the +# supervisor's process-group kill. +_SHARED_EXTRACTOR_POOL = None +_USE_SHARED_POOL = False + + +def _new_extractor_pool(): + """Create a fresh fork-context pebble pool (spike decision: fork from the + warm child — see docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md).""" + ctx = multiprocessing.get_context("fork") + return pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) + + +def enable_shared_extractor_pool(): + """Opt this process into a shared extractor pool reused across every call to + `generic_file_extractors`. Called by the prefork child after fork.""" + global _USE_SHARED_POOL + _USE_SHARED_POOL = True + + +def _acquire_extractor_pool(): + """Return ``(pool, is_shared)``. In shared mode a single process-wide pool is + created on first use and reused; otherwise a fresh per-call pool is returned.""" + global _SHARED_EXTRACTOR_POOL + if _USE_SHARED_POOL: + if _SHARED_EXTRACTOR_POOL is None: + _SHARED_EXTRACTOR_POOL = _new_extractor_pool() + return _SHARED_EXTRACTOR_POOL, True + return _new_extractor_pool(), False + + +def shutdown_shared_extractor_pool(): + """Close+join the shared extractor pool once, if one was created. Idempotent + and exception-safe; called by the prefork child before it exits.""" + global _SHARED_EXTRACTOR_POOL + pool = _SHARED_EXTRACTOR_POOL + _SHARED_EXTRACTOR_POOL = None + if pool is None: + return + try: + pool.close() + pool.join() + except Exception: + log.debug("shutdown_shared_extractor_pool: teardown raised", exc_info=True) + def generic_file_extractors( file: str, @@ -444,12 +506,10 @@ def generic_file_extractors( futures = {} executed_tools = data_dictionary.setdefault("executed_tools", []) - # Per-call pool lifecycle: create, use, then close+join in finally so the - # worker processes are torn down before the task child exits. Using the - # "fork" context (Task 9 SPIKE decision: see - # docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md). - ctx = multiprocessing.get_context("fork") - pool = pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) + # Per-call pool (pebble) or shared-per-child pool (prefork) — see the + # module-level lifecycle notes. A shared pool is reused across files and torn + # down once by the prefork child; a per-call pool is closed+joined here. + pool, shared_pool = _acquire_extractor_pool() try: # Prefer custom modules over the built-in ones, since only 1 is allowed # to be the extracted_files_tool. @@ -526,8 +586,12 @@ def generic_file_extractors( # ToDo doesn't work shutil.rmtree(tempdir, ignore_errors=True) finally: - pool.close() - pool.join() + # A shared pool (prefork) is kept warm for the next file and torn down + # once by the child via shutdown_shared_extractor_pool(); a per-call + # pool (pebble) is closed+joined here so no nested pool outlives the call. + if not shared_pool: + pool.close() + pool.join() def _generic_post_extraction_process(file: str, tool_name: str, decoded: str) -> SuccessfulExtractionReturnType: diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 661c8271ff3..91eb803033e 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -63,6 +63,15 @@ def _heartbeat(self): def _child_main(self, task): os.setsid() # own session/process group; supervisor killpg sweeps the subtree + # This child runs exactly one task then os._exit()s, so a single extractor + # pool can be shared across every file in the task and torn down once — + # reclaiming the per-file fork cost a per-call pool would pay N times. Safe + # here (unlike pebble worker recycling) because the child never has to + # cleanly join a persistent nested pool mid-life: it just exits, and any + # straggler is swept by the supervisor's process-group kill. + from lib.cuckoo.common.integrations import file_extra_info + + file_extra_info.enable_shared_extractor_pool() try: self.worker_init() self.task_fn(task) @@ -70,6 +79,8 @@ def _child_main(self, task): except BaseException: log.exception("prefork child: task %s crashed", getattr(task, "id", "?")) return 1 + finally: + file_extra_info.shutdown_shared_extractor_pool() def _launch(self, task): self._assert_single_threaded() diff --git a/tests/test_shared_extractor_pool.py b/tests/test_shared_extractor_pool.py new file mode 100644 index 00000000000..b74367a6dfd --- /dev/null +++ b/tests/test_shared_extractor_pool.py @@ -0,0 +1,113 @@ +"""Extractor sub-pool lifecycle: per-call (pebble default) vs shared-per-child +(prefork). Under prefork a single pool is reused across every extracted file in +the task child and torn down once; under pebble each call gets a fresh pool that +is closed+joined immediately (avoids the recycle-join deadlock).""" +import os + +import lib.cuckoo.common.integrations.file_extra_info as fx +from lib.cuckoo.core.processing_engine.prefork import PreforkEngine + + +class _FakePool: + """Records close/join without forking real worker processes.""" + + _counter = 0 + + def __init__(self): + _FakePool._counter += 1 + self.id = _FakePool._counter + self.closed = False + self.joined = False + + def close(self): + self.closed = True + + def join(self, timeout=None): + self.joined = True + + +def _reset(monkeypatch): + """Point the pool factory at _FakePool and reset shared state.""" + monkeypatch.setattr(fx, "_new_extractor_pool", lambda: _FakePool()) + monkeypatch.setattr(fx, "_SHARED_EXTRACTOR_POOL", None, raising=False) + monkeypatch.setattr(fx, "_USE_SHARED_POOL", False, raising=False) + + +def test_per_call_mode_returns_fresh_pool_each_time(monkeypatch): + _reset(monkeypatch) + pool_a, shared_a = fx._acquire_extractor_pool() + pool_b, shared_b = fx._acquire_extractor_pool() + assert shared_a is False and shared_b is False + assert pool_a is not pool_b + + +def test_shared_mode_reuses_same_pool(monkeypatch): + _reset(monkeypatch) + fx.enable_shared_extractor_pool() + pool_a, shared_a = fx._acquire_extractor_pool() + pool_b, shared_b = fx._acquire_extractor_pool() + assert shared_a is True and shared_b is True + assert pool_a is pool_b + + +def test_shutdown_closes_and_clears_shared_pool(monkeypatch): + _reset(monkeypatch) + fx.enable_shared_extractor_pool() + pool, _ = fx._acquire_extractor_pool() + fx.shutdown_shared_extractor_pool() + assert pool.closed is True + assert pool.joined is True + # A subsequent acquire must build a brand-new pool, not reuse the dead one. + pool_next, _ = fx._acquire_extractor_pool() + assert pool_next is not pool + + +def test_shutdown_is_idempotent_when_no_pool(monkeypatch): + _reset(monkeypatch) + # No pool created yet, shared mode off: must not raise. + fx.shutdown_shared_extractor_pool() + fx.enable_shared_extractor_pool() + # Enabled but never acquired: still a no-op, no raise. + fx.shutdown_shared_extractor_pool() + + +class _Task: + id = 1 + category = "file" + + +def test_prefork_child_enables_and_tears_down_shared_pool(tmp_path): + """The prefork child must (1) opt into the shared extractor pool before + running the task and (2) tear it down once when the task finishes. Verified + across a real fork: the child records both observations to marker files.""" + enabled_marker = tmp_path / "enabled" + closed_marker = tmp_path / "closed" + + def task_fn(task): + # Runs inside the forked child. Observe that shared mode is already on, + # then install a fake pool so teardown has something to close. + enabled_marker.write_text("yes" if fx._USE_SHARED_POOL else "no") + + class _FakeChildPool: + def close(self): + closed_marker.write_text("closed") + + def join(self, timeout=None): + pass + + fx._new_extractor_pool = lambda: _FakeChildPool() + fx._acquire_extractor_pool() # store the shared fake pool + + eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, + source=None, parallel=1, timeout=30) + + pid = os.fork() + if pid == 0: + try: + eng._child_main(_Task()) + finally: + os._exit(0) + os.waitpid(pid, 0) + + assert enabled_marker.read_text() == "yes" + assert closed_marker.read_text() == "closed" From 8f5a348bc53f2550b4e939b6455e3056c83f4e83 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 16:12:28 -0500 Subject: [PATCH 29/53] rooter: gateway liveness requires IFF_UP, not just interface existence nic_available only proves the link exists (true for administratively DOWN interfaces), so _gw_live treated a down gateway NIC as live and round-robin/random selection could bind a task to a dead exit. Add a nic_up rooter verb that checks the IFF_UP flag and use it for gateway liveness. --- lib/cuckoo/core/rooter.py | 8 +++--- tests/test_nexthop_selection.py | 14 ++++++++++ tests/test_rooter_nexthop.py | 48 +++++++++++++++++++++++++++++++++ utils/rooter.py | 17 ++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) diff --git a/lib/cuckoo/core/rooter.py b/lib/cuckoo/core/rooter.py index 4c3e93b3e70..f3a5f3b69e0 100644 --- a/lib/cuckoo/core/rooter.py +++ b/lib/cuckoo/core/rooter.py @@ -67,9 +67,11 @@ def rooter(command, *args, **kwargs): def _gw_live(profile): - """True if the profile's egress interface exists and is up. Delegates to the rooter - nic_available check; overridable in tests.""" - resp = rooter("nic_available", str(profile.interface)) + """True if the profile's egress interface exists AND is administratively up. A down gateway NIC + must not be selected -- round-robin/random would otherwise bind a task to a dead exit. Delegates + to the rooter nic_up check (nic_available only proves existence, incl. DOWN links); overridable + in tests.""" + resp = rooter("nic_up", str(profile.interface)) return bool(resp and resp.get("output")) diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py index 7da2a9697ab..dbae8baaf7e 100644 --- a/tests/test_nexthop_selection.py +++ b/tests/test_nexthop_selection.py @@ -461,3 +461,17 @@ def __init__(self): monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) with pytest.raises(CuckooStartupError): startup.load_nexthop_profiles(_NexthopEmptyPool()) + + +def test_gw_live_queries_nic_up(monkeypatch): + # _gw_live must consult nic_up (admin-up), NOT nic_available (which is true for DOWN links). + calls = [] + + def _fake_rooter(cmd, *a): + calls.append(cmd) + return {"output": (cmd == "nic_up" and a and a[0] == "ens-up")} + + monkeypatch.setattr(core_rooter, "rooter", _fake_rooter) + assert core_rooter._gw_live(_Profile("gw1", "ens-up", "201", 0)) is True + assert core_rooter._gw_live(_Profile("gw2", "ens-down", "202", 0)) is False + assert "nic_up" in calls and "nic_available" not in calls diff --git a/tests/test_rooter_nexthop.py b/tests/test_rooter_nexthop.py index b0de9b333e6..0cc6a507220 100644 --- a/tests/test_rooter_nexthop.py +++ b/tests/test_rooter_nexthop.py @@ -231,3 +231,51 @@ def test_sigterm_teardown_runs_when_configured(rec, monkeypatch): rooter.nexthop_teardown_if_configured() assert ("ip", "route", "flush", "table", "201") in rec["run"] assert ("ip", "route", "flush", "table", "202") in rec["run"] + + +# --- nic_up: gateway liveness must require IFF_UP, not just existence (codex #14 P2) --- + +def _fake_ip_link(line): + def _co(*a, **k): + return line + return _co + + +def test_nic_up_true_when_admin_up(monkeypatch): + class _S: + ip = "ip" + monkeypatch.setattr(rooter, "settings", _S, raising=False) + monkeypatch.setattr(rooter.subprocess, "check_output", + _fake_ip_link("2: ens6: mtu 1500 state UP\n")) + assert rooter.nic_up("ens6") is True + + +def test_nic_up_false_when_admin_down(monkeypatch): + class _S: + ip = "ip" + monkeypatch.setattr(rooter, "settings", _S, raising=False) + monkeypatch.setattr(rooter.subprocess, "check_output", + _fake_ip_link("3: ens7: mtu 1500 state DOWN\n")) + assert rooter.nic_up("ens7") is False + + +def test_nic_up_no_false_match_on_lower_up(monkeypatch): + # LOWER_UP (carrier) present but IFF_UP absent -> not admin-up; "UP" must be an exact token + class _S: + ip = "ip" + monkeypatch.setattr(rooter, "settings", _S, raising=False) + monkeypatch.setattr(rooter.subprocess, "check_output", + _fake_ip_link("4: ens8: mtu 1500 state DOWN\n")) + assert rooter.nic_up("ens8") is False + + +def test_nic_up_false_when_missing(monkeypatch): + class _S: + ip = "ip" + monkeypatch.setattr(rooter, "settings", _S, raising=False) + + def _raise(*a, **k): + raise rooter.subprocess.CalledProcessError(1, "ip") + + monkeypatch.setattr(rooter.subprocess, "check_output", _raise) + assert rooter.nic_up("nope0") is False diff --git a/utils/rooter.py b/utils/rooter.py index 39e1ed346f9..96dc0a2fd24 100644 --- a/utils/rooter.py +++ b/utils/rooter.py @@ -202,6 +202,22 @@ def nic_available(interface): return False +def nic_up(interface): + """Check that the interface exists AND is administratively up (IFF_UP). Unlike nic_available, an + existing-but-DOWN link returns False: the nexthop selector must not treat a down gateway NIC as a + live egress (round-robin/random would otherwise bind a task to a dead exit).""" + try: + out = subprocess.check_output( + [settings.ip, "-o", "link", "show", interface], stderr=subprocess.PIPE, universal_newlines=True + ) + except subprocess.CalledProcessError: + return False + # Flags live in the angle-bracket group, e.g. ""; IFF_UP == "UP" + # (an exact token, so "LOWER_UP" carrier does not false-match). + flags = out.split("<", 1)[-1].split(">", 1)[0] if "<" in out else "" + return "UP" in flags.split(",") + + def rt_available(rt_table): """Check if specified routing table is defined.""" try: @@ -1249,6 +1265,7 @@ def sslproxy_disable(interface, client, proxy_port, resultserver_port, rt_table= handlers = { "nic_available": nic_available, + "nic_up": nic_up, "rt_available": rt_available, "vpn_status": vpn_status, "forward_drop": forward_drop, From 9d5fa65e924a63184e17616106cc6a3c3c9151d4 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 21:15:38 +0000 Subject: [PATCH 30/53] fix(extractors): bound extractor-pool teardown so a hung grandchild can't wedge the worker pool.join() was unbounded: a wedged extractor grandchild (D-state IO, a hung Java decompiler) could block teardown forever, hanging the worker with no external timeout to break it (the pebble task timeout only covers execution, not teardown). Both teardown paths (per-call pebble, shared prefork) now go through _teardown_extractor_pool: graceful join up to EXTRACTOR_POOL_JOIN_TIMEOUT, then stop()+join() to force-kill stragglers. Addresses the last unbounded-hang in the pebble A/B path without touching the -mc 7 recycling default. --- .../common/integrations/file_extra_info.py | 40 ++++++++++---- tests/test_shared_extractor_pool.py | 55 +++++++++++++++++++ 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 3c83f8014b0..5e1d7b75f42 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -430,6 +430,13 @@ def _extracted_files_metadata( _SHARED_EXTRACTOR_POOL = None _USE_SHARED_POOL = False +# Bound extractor-pool teardown. A wedged grandchild (D-state IO, a hung Java +# decompiler — see the stray hs_err_pid dumps) would otherwise make pool.join() +# block forever, hanging the whole worker with no external timeout to break it +# (the pebble task timeout only covers task execution, not teardown). Wait this +# long for a graceful drain, then force-stop. +EXTRACTOR_POOL_JOIN_TIMEOUT = 60 + def _new_extractor_pool(): """Create a fresh fork-context pebble pool (spike decision: fork from the @@ -438,6 +445,26 @@ def _new_extractor_pool(): return pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) +def _teardown_extractor_pool(pool): + """Close+join a pool with a bounded wait, force-stopping any straggler so a + hung extractor can never wedge the worker indefinitely. Exception-safe.""" + if pool is None: + return + try: + pool.close() + try: + pool.join(timeout=EXTRACTOR_POOL_JOIN_TIMEOUT) + except Exception: + log.warning( + "extractor pool did not drain within %ss; force-stopping stragglers", + EXTRACTOR_POOL_JOIN_TIMEOUT, + ) + pool.stop() + pool.join() + except Exception: + log.debug("_teardown_extractor_pool: teardown raised", exc_info=True) + + def enable_shared_extractor_pool(): """Opt this process into a shared extractor pool reused across every call to `generic_file_extractors`. Called by the prefork child after fork.""" @@ -462,13 +489,7 @@ def shutdown_shared_extractor_pool(): global _SHARED_EXTRACTOR_POOL pool = _SHARED_EXTRACTOR_POOL _SHARED_EXTRACTOR_POOL = None - if pool is None: - return - try: - pool.close() - pool.join() - except Exception: - log.debug("shutdown_shared_extractor_pool: teardown raised", exc_info=True) + _teardown_extractor_pool(pool) def generic_file_extractors( @@ -588,10 +609,9 @@ def generic_file_extractors( finally: # A shared pool (prefork) is kept warm for the next file and torn down # once by the child via shutdown_shared_extractor_pool(); a per-call - # pool (pebble) is closed+joined here so no nested pool outlives the call. + # pool (pebble) is torn down here so no nested pool outlives the call. if not shared_pool: - pool.close() - pool.join() + _teardown_extractor_pool(pool) def _generic_post_extraction_process(file: str, tool_name: str, decoded: str) -> SuccessfulExtractionReturnType: diff --git a/tests/test_shared_extractor_pool.py b/tests/test_shared_extractor_pool.py index b74367a6dfd..60880bdcc2a 100644 --- a/tests/test_shared_extractor_pool.py +++ b/tests/test_shared_extractor_pool.py @@ -71,6 +71,61 @@ def test_shutdown_is_idempotent_when_no_pool(monkeypatch): fx.shutdown_shared_extractor_pool() +class _HangingPool: + """join(timeout) times out (wedged grandchild); stop()+unbounded join reaps.""" + + def __init__(self): + self.closed = False + self.stopped = False + self.join_timeouts = [] + + def close(self): + self.closed = True + + def join(self, timeout=None): + self.join_timeouts.append(timeout) + if timeout is not None: + raise TimeoutError("pool still draining") + + def stop(self): + self.stopped = True + + +class _CleanPool: + def __init__(self): + self.stopped = False + self.joined = False + + def close(self): + pass + + def join(self, timeout=None): + self.joined = True + + def stop(self): + self.stopped = True + + +def test_teardown_force_stops_when_join_times_out(): + pool = _HangingPool() + fx._teardown_extractor_pool(pool) + assert pool.closed is True + assert pool.stopped is True, "a wedged pool must be force-stopped, not left to hang" + assert pool.join_timeouts[0] is not None, "first join must be bounded by a timeout" + assert pool.join_timeouts[-1] is None, "after stop() the final reap join is unbounded" + + +def test_teardown_does_not_stop_when_join_succeeds(): + pool = _CleanPool() + fx._teardown_extractor_pool(pool) + assert pool.joined is True + assert pool.stopped is False, "a cleanly-drained pool must not be force-stopped" + + +def test_teardown_none_is_noop(): + fx._teardown_extractor_pool(None) # must not raise + + class _Task: id = 1 category = "file" From 837674fa25383df0dd4212ca858642c98d907329 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 21:30:49 +0000 Subject: [PATCH 31/53] perf(processor): pre-warm YARA in supervisor so forked children inherit it (COW) autoprocess now compiles the YARA ruleset once before the engine forks any workers. Every forked child inherits the compiled rules via copy-on-write fork (0s + shared read-only pages) and its init_worker File.init_yara() short-circuits on the idempotency guard. Prefork forks one child per task, so without this each task paid the full compile (~0.3-3s depending on ruleset); measured ~321ms -> ~0.05ms per child. Safe: compilation is single-threaded, preserving the prefork single-threaded-before-fork invariant. --- tests/test_autoprocess_yara_prewarm.py | 35 ++++++++++++++++++++++++++ utils/process.py | 12 +++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/test_autoprocess_yara_prewarm.py diff --git a/tests/test_autoprocess_yara_prewarm.py b/tests/test_autoprocess_yara_prewarm.py new file mode 100644 index 00000000000..12efbc7e44c --- /dev/null +++ b/tests/test_autoprocess_yara_prewarm.py @@ -0,0 +1,35 @@ +"""The supervisor must compile YARA once before the engine forks any workers, so +forked children inherit the compiled ruleset via COW instead of each paying the +~3s recompile (critical for prefork's one-task-per-child model).""" +import lib.cuckoo.core.processing_engine as pe +import lib.cuckoo.core.processing_engine.source as pe_source +import utils.process as process +from lib.cuckoo.common.objects import File + + +def test_autoprocess_prewarms_yara_before_engine_runs(monkeypatch): + File.yara_initialized = False + observed = {} + + def fake_init_yara(cls, *a, **k): + cls.yara_initialized = True + + monkeypatch.setattr(File, "init_yara", classmethod(fake_init_yara)) + + class _FakeEngine: + max_count = 0 + max_tasks = 0 + + def run(self): + # Record whether YARA was already compiled at the moment the engine + # (which is what forks workers) starts running. + observed["yara_ready_at_run"] = File.yara_initialized + + monkeypatch.setattr(pe, "get_engine", lambda *a, **k: _FakeEngine()) + monkeypatch.setattr(pe_source, "TaskSource", lambda *a, **k: object()) + monkeypatch.setattr(process, "memory_limit", lambda *a, **k: None) + + process.autoprocess(engine="prefork", disable_memory_limit=True) + + assert observed.get("yara_ready_at_run") is True, \ + "YARA must be compiled before the engine runs (before any fork)" diff --git a/utils/process.py b/utils/process.py index c2d6a7c49d0..93a012c7081 100644 --- a/utils/process.py +++ b/utils/process.py @@ -437,6 +437,18 @@ def autoprocess( if not disable_memory_limit: memory_limit() log.info("Processing analysis data (engine=%s)", engine) + + # Compile the YARA ruleset ONCE in the supervisor, before the engine forks any + # workers. Every forked child then inherits the compiled rules via copy-on-write + # fork (0s + shared read-only pages), and its init_worker() File.init_yara() call + # short-circuits on the idempotency guard. Without this, prefork — which forks a + # fresh child per task — would pay the ~3s compile on every single task. Safe to + # call here: compilation is single-threaded, so it does not break the prefork + # single-threaded-before-fork invariant. + from lib.cuckoo.common.objects import File + + File.init_yara() + source = TaskSource(db, failed_processing=failed_processing) eng = get_engine( engine, From 241e6c214e185cf1bf341014502d9720f602d621 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 16:55:23 -0500 Subject: [PATCH 32/53] rooter: fail-closed hardening (resolve re-entry clears stale binding + nexthop_init blackhole seed) - _resolve_nexthop clears self.nexthop_* on entry so a failed resolve (route->drop) cannot leave a stale binding that _dispatch_nexthop still installs on route_network re-entry/retry. - nexthop_init seeds a blackhole default before the real replace, so a failed route setup leaves the gateway table fail-closed instead of empty (falling through to main), even with fail_closed=no. --- lib/cuckoo/core/analysis_manager.py | 4 ++++ tests/test_rooter_nexthop.py | 2 ++ tests/test_route_network_nexthop.py | 22 ++++++++++++++++++++++ utils/rooter.py | 5 +++++ 4 files changed, 33 insertions(+) diff --git a/lib/cuckoo/core/analysis_manager.py b/lib/cuckoo/core/analysis_manager.py index d56327d6ae1..b2612cfc0a4 100644 --- a/lib/cuckoo/core/analysis_manager.py +++ b/lib/cuckoo/core/analysis_manager.py @@ -559,6 +559,10 @@ def _resolve_nexthop(self, routing): """ self.interface = None # nexthop owns enable/disable; skip the generic block self.rt_table = None + # Clear any binding a PRIOR resolve left, so a failure path here (route -> "drop") cannot + # leave a stale self.nexthop_* that _dispatch_nexthop would still install -- fail-open on + # route_network re-entry / machine retry (Copilot). + self.nexthop_id = self.nexthop_interface = self.nexthop_rt_table = self.nexthop_priority = None # Resolve the selector, FAILING CLOSED on a typo'd/unknown route (review B1 + gemini #14): # - an explicit gateway id, or a policy token (roundrobin/random), selects directly; # - ONLY the node's configured DEFAULT route falls back to default_policy; diff --git a/tests/test_rooter_nexthop.py b/tests/test_rooter_nexthop.py index 0cc6a507220..ccbf47111d4 100644 --- a/tests/test_rooter_nexthop.py +++ b/tests/test_rooter_nexthop.py @@ -47,6 +47,7 @@ def test_nexthop_init_onlink(rec): rooter.nexthop_init("201", "ens6", "onlink") assert rec["run"] == [ ("ip", "route", "flush", "table", "201"), + ("ip", "route", "replace", "blackhole", "default", "table", "201"), ("ip", "route", "replace", "default", "dev", "ens6", "onlink", "table", "201"), ] @@ -55,6 +56,7 @@ def test_nexthop_init_via(rec): rooter.nexthop_init("202", "ens7", "10.30.72.1") assert rec["run"] == [ ("ip", "route", "flush", "table", "202"), + ("ip", "route", "replace", "blackhole", "default", "table", "202"), ("ip", "route", "replace", "default", "via", "10.30.72.1", "dev", "ens7", "table", "202"), ] diff --git a/tests/test_route_network_nexthop.py b/tests/test_route_network_nexthop.py index 59120a4927f..d87eee05a8e 100644 --- a/tests/test_route_network_nexthop.py +++ b/tests/test_route_network_nexthop.py @@ -225,3 +225,25 @@ def test_unroute_noop_when_not_nexthop(mgr): mgr.nexthop_id = None mgr._unroute_nexthop() assert [c for c, _ in mgr._calls if c == "nexthop_disable"] == [] + + +def test_resolve_clears_stale_binding_on_reentry_failure(mgr, monkeypatch): + # route_network re-entry (e.g. machine retry): a first resolve binds a gateway; a later resolve + # that fails (empty/all-down pool) must FORCE drop AND clear self.nexthop_* so _dispatch_nexthop + # does not install a stale binding despite the drop decision (Copilot fail-open). + prof = type("P", (), {"name": "gw1", "interface": "ens6", "rt_table": "201", "priority": 0})() + routing = _fake_routing(default_policy="roundrobin", default_route="nexthop") + monkeypatch.setattr(am, "_select_gateway", lambda r: prof, raising=False) + monkeypatch.setattr(am, "gateways", {"gw1": prof}, raising=False) + mgr.route = "gw1" + assert mgr._resolve_nexthop(routing) is True + assert mgr.nexthop_id == "gw1" + # pool now empty -> second resolve fails: drop + stale binding cleared + monkeypatch.setattr(am, "_select_gateway", lambda r: None, raising=False) + mgr.route = "nexthop" + assert mgr._resolve_nexthop(routing) is False + assert mgr.route == "drop" + assert mgr.nexthop_id is None + mgr._calls.clear() + mgr._dispatch_nexthop() # no-op: no stale binding to install + assert "nexthop_enable" not in [c for c, _ in mgr._calls] diff --git a/utils/rooter.py b/utils/rooter.py index 96dc0a2fd24..fcfe2d26a84 100644 --- a/utils/rooter.py +++ b/utils/rooter.py @@ -697,6 +697,11 @@ def nexthop_init(rt_table, egress_if, next_hop): log.error("nexthop_init refusing to flush reserved routing table %r (fix the [gwX] rt_table)", rt_table) return run(settings.ip, "route", "flush", "table", rt_table) + # Fail-closed seed: never leave the table empty. If the real default replace below fails (bad + # egress_if/next_hop), this blackhole remains so the VM's policy lookup drops instead of falling + # through to `main` and egressing outside the gateway -- true even when [nexthop] fail_closed=no + # (codex P2: run() records but does not raise on a failed ip route replace). + run(settings.ip, "route", "replace", "blackhole", "default", "table", rt_table) if next_hop == "onlink": run(settings.ip, "route", "replace", "default", "dev", egress_if, "onlink", "table", rt_table) else: From bf495cd765e5a7bcbb92fb294100c8630634e279 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 17:14:00 -0500 Subject: [PATCH 33/53] rooter: startup config validation (dirty-line table gating when enabled + default_policy) - reserve the [routing] dirty-line rt_table only when the dirty line is enabled (internet != none), so a nexthop-only node reusing that id is not falsely rejected at startup. - validate [nexthop] default_policy is roundrobin/random or a configured gateway id (else every pool task silently drops). --- lib/cuckoo/core/startup.py | 16 ++++++++- tests/test_nexthop_selection.py | 59 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/lib/cuckoo/core/startup.py b/lib/cuckoo/core/startup.py index bb1ffcf9bad..4ca6725ff34 100644 --- a/lib/cuckoo/core/startup.py +++ b/lib/cuckoo/core/startup.py @@ -115,7 +115,11 @@ def load_nexthop_profiles(routing_cfg): if vrt is not None: owned_tables[str(vrt)] = f"VPN '{vname}'" dirty_rt = getattr(getattr(routing_cfg, "routing", None), "rt_table", None) - if dirty_rt is not None and str(dirty_rt): + dirty_internet = str(getattr(getattr(routing_cfg, "routing", None), "internet", "none") or "none") + # Only reserve the [routing] dirty-line table when the dirty-line route is actually enabled (its + # init_rttable runs only when internet != "none"). A nexthop-only node (internet = none) with a + # preconfigured rt_table must not falsely reject a [gwX] reusing that id and fail to start (codex P2). + if dirty_rt is not None and str(dirty_rt) and dirty_internet != "none": owned_tables.setdefault(str(dirty_rt), "the [routing] dirty-line table") # Pass 1: parse + validate + register profiles (no rooter side effects yet). profiles = [] @@ -188,6 +192,16 @@ def load_nexthop_profiles(routing_cfg): # loudly at startup instead (gemini medium). if not profiles: raise CuckooStartupError("[nexthop] is enabled but no gateways were configured in routing.conf") + # default_policy must resolve: a pool selector (roundrobin/random) or one of the configured gateway + # ids. Otherwise _select_gateway() returns None for every default/route=nexthop task and they all + # silently drop; reject a typo'd/dangling policy at startup instead of shipping a dead pool (codex P2). + default_policy = str(getattr(routing_cfg.nexthop, "default_policy", "") or "") + gateway_ids = {p.name for p in profiles} + if default_policy not in _POLICY_TOKENS and default_policy not in gateway_ids: + raise CuckooStartupError( + f"[nexthop] default_policy '{default_policy}' must be 'roundrobin', 'random', or a configured " + f"gateway id ({', '.join(sorted(gateway_ids))})" + ) vm_net = str(routing_cfg.nexthop.vm_net) tables_csv = ",".join(p.rt_table for p in profiles) # Record sweep state for SIGTERM, then sweep any STALE state from a prior run diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py index dbae8baaf7e..894a1b5a655 100644 --- a/tests/test_nexthop_selection.py +++ b/tests/test_nexthop_selection.py @@ -412,7 +412,8 @@ def test_gwx_rt_table_collides_with_dirty_line_raises(monkeypatch): class _GwDirtyLineTable(_FakeRouting): def __init__(self): super().__init__() - self.routing = _DictSection(route="none", rt_table="201") # dirty-line table == gw table + # dirty-line ENABLED (internet != none) and its table == the gw table -> collision + self.routing = _DictSection(route="internet", internet="ens_wan", rt_table="201") self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) startup.gateways.clear() @@ -423,6 +424,62 @@ def __init__(self): assert "gw1" not in startup.gateways +def test_gwx_rt_table_reused_when_dirty_line_disabled_ok(monkeypatch): + # codex P2: when the dirty line is DISABLED (internet = none) its rt_table is never built, so a + # [gwX] reusing that id must NOT be rejected -- a nexthop-only node must be able to start. + import lib.cuckoo.core.startup as startup + + class _GwDirtyLineDisabled(_FakeRouting): + def __init__(self): + super().__init__() + self.routing = _DictSection(route="none", internet="none", rt_table="201") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + startup.load_nexthop_profiles(_GwDirtyLineDisabled()) # must NOT raise + assert "gw1" in startup.gateways + + +def test_invalid_default_policy_raises(monkeypatch): + # codex P2: default_policy that is neither roundrobin/random nor a configured gateway id resolves + # to None in _select_gateway -> every pool task silently drops. Reject it at startup. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _BadPolicy(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="gw2", + fail_closed=True, vm_net="192.168.100.0/24") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_BadPolicy()) + + +def test_gateway_id_default_policy_ok(monkeypatch): + # default_policy may name a configured gateway id (pin the pool default to one exit). + import lib.cuckoo.core.startup as startup + + class _GwPolicy(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="gw1", + fail_closed=True, vm_net="192.168.100.0/24") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + startup.load_nexthop_profiles(_GwPolicy()) # must NOT raise + assert "gw1" in startup.gateways + + def test_gateway_named_like_policy_token_raises(monkeypatch): # A [gwX] must not be named a pool-policy token (roundrobin/random): _select_gateway would # treat the name as the policy, not the id, so the profile could never be explicitly selected. From 50dffbb1e518157bb38a8ecdd0adba5a4595f560 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 22:27:52 +0000 Subject: [PATCH 34/53] docs: drop internal planning docs from upstream PR; make code comments self-contained --- .../2026-05-27-extractor-subpool-spike.md | 506 --------- .../2026-05-27-processor-engine-ab-runbook.md | 276 ----- .../2026-05-27-processor-prefork-engine.md | 986 ------------------ ...7-processor-concurrency-redesign-design.md | 249 ----- .../common/integrations/file_extra_info.py | 6 +- lib/cuckoo/core/processing_engine/prefork.py | 2 +- 6 files changed, 5 insertions(+), 2020 deletions(-) delete mode 100644 docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md delete mode 100644 docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md delete mode 100644 docs/superpowers/plans/2026-05-27-processor-prefork-engine.md delete mode 100644 docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md diff --git a/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md b/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md deleted file mode 100644 index a1b2894f012..00000000000 --- a/docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md +++ /dev/null @@ -1,506 +0,0 @@ -# Extractor Sub-Pool Spawn Strategy: Spike Results - -**Date:** 2026-05-27 -**Scope:** Resolve design-spec §4.2(d) — which `multiprocessing` start method -(`forkserver` vs `fork`) the per-task `_EXTRACTOR_POOL` should use under the -prefork supervisor (Tasks 5–8). -**Status:** DECIDED — use **`fork`** -**Implements:** Task 9 of the processor-prefork-engine plan. -**Next:** Task 10 rewrites `_EXTRACTOR_POOL` in -`lib/cuckoo/common/integrations/file_extra_info.py` using -`multiprocessing.get_context("fork")`. - ---- - -## Decision Criterion - -Choose the faster strategy that leaves **zero survivors after `killpg`** and -runs cleanly with `os._exit()`. If latency is a tie, prefer `forkserver` for -stronger isolation; otherwise take the winner on latency. - ---- - -## Environment - -| Item | Value | -|---|---| -| Host Python | 3.12.3 (GCC 13.3.0) | -| pebble | 5.1.0 | -| Available start methods | `fork`, `spawn`, `forkserver` | -| `MAX_WORKERS` tested | 6 (matches planned `_EXTRACTOR_POOL`) | -| Iterations per context | 7 clean samples (8th occasionally lost to fork-pipe flush; stats below are over the captured samples) | - ---- - -## Results - -### Latency (pool-create + first-result, ms) - -Each iteration: create a fresh `pebble.ProcessPool(max_workers=6, context=ctx)`, -schedule one `trivial_extractor` call (200× SHA-256 rounds on 256 random bytes), -collect result, `pool.close(); pool.join()`. Measured inside a forked child -that is single-threaded at the time of pool creation (mirrors production: the -task child is forked from the supervisor before doing any threaded work). - -| context | mean | median | p95 | min | max | errors | -|---|---|---|---|---|---|---| -| `fork` | **115.7 ms** | 117.0 ms | 118.3 ms | 112.9 ms | 118.3 ms | 0 | -| `forkserver` | **125.5 ms** | 112.3 ms | **219.0 ms** | 111.3 ms | 219.0 ms | 0 | - -Key observations: -- `fork` is stable: all 7 samples land within a 5.5 ms band (112.9–118.3 ms). -- `forkserver` shows a **~107 ms cold-start penalty on iteration 1** (219 ms vs - the steady-state ~112 ms). This is the forkserver helper process booting. - Because each task child is ephemeral (it exits via `os._exit()` after the task - completes), **every task pays the cold-start cost** — the forkserver process - does not survive across tasks. The steady-state median (~112 ms) is therefore - misleading; the operative number for one-shot task children is the p95 - (219 ms). -- `fork` mean (115.7 ms) vs `forkserver` mean (125.5 ms): **9.8 ms faster** on - mean; the true per-task advantage is larger due to the cold-start effect. - -### Orphan Check (killpg safety) - -For each context: fork a child that calls `os.setsid()`, creates the sub-pool -with `MAX_WORKERS=6` workers all running, then sleeps. Parent records the -child's `pgid`, waits 3 s, sends `SIGKILL` to the process group -(`os.killpg(pgid, SIGKILL)`), waits 0.5 s, then runs `pgrep -g ` to -check for survivors. - -| context | child pgid | survivors after killpg | result | -|---|---|---|---| -| `fork` | 3435871 | (none) | **PASS** | -| `forkserver` | 3435882 | (none) | **PASS** | - -Both contexts leave zero survivors. The forkserver helper process and all pool -workers are members of the child's process group (they inherit the session from -the `setsid()` call) and are swept cleanly by `SIGKILL`. - -### Side Note: forkserver Requires Importable Task Functions - -During direct (`-c`) testing, `forkserver` workers failed with: - -``` -AttributeError: Can't get attribute 'trivial_extractor' on - )> -``` - -This is expected behaviour: the forkserver re-imports `__main__` to find the -callable, and inline `-c` code has no importable module. In production, -`_EXTRACTOR_POOL` schedules functions defined in -`lib/cuckoo/common/integrations/file_extra_info.py`, which IS a real importable -module, so this is not a blocker. It is noted here because it would surface -immediately in tests that define callables in `__main__` without writing a real -module file. - ---- - -## Decision - -**Chosen context: `fork`** - -Rationale: - -1. **Latency wins on every metric that matters for ephemeral task children.** - `fork` mean (115.7 ms) is 9.8 ms faster than `forkserver` mean (125.5 ms). - More importantly, `forkserver` p95 is 219 ms vs `fork` p95 of 118 ms — a - 100 ms difference — because the forkserver helper process must be created - fresh for every task child. - -2. **Both contexts pass the orphan/killpg safety check** (zero survivors after - `SIGKILL` to the process group). Safety is not a differentiator here. - -3. **Fork-from-warm-child is safe in this context.** The design mandates that - `_EXTRACTOR_POOL` is created immediately after the task child is forked from - the supervisor, before the child spawns any threads. At that point the child - is single-threaded, so `fork` is free of the fork-after-thread deadlock risk. - -4. **Forkserver's isolation benefit is not needed here.** The task child itself - is already isolated (it exits via `os._exit()` and is killed via `killpg`). - The additional process-level isolation that forkserver provides over fork - adds latency cost for no measurable safety gain. - ---- - -## Spike Script - -Paste below for reproducibility. Run with the CAPEv2 venv Python: - -``` -/home/cape/.cache/pypoetry/virtualenvs/capev2-Cxvx6Y4B-py3.12/bin/python3.12 /tmp/extractor_spike.py -``` - -```python -#!/usr/bin/env python3 -""" -Throwaway spike script: measure pebble.ProcessPool creation+first-result latency -for 'forkserver' vs 'fork' start methods, and verify orphan behaviour under killpg. - -Usage: run with the CAPEv2 venv Python, e.g. - /home/cape/.cache/pypoetry/virtualenvs/capev2-Cxvx6Y4B-py3.12/bin/python3.12 /tmp/extractor_spike.py - -Results are printed to stdout. -""" - -import hashlib -import multiprocessing -import os -import signal -import subprocess -import sys -import time - -import pebble - -ITERATIONS = 8 # latency iterations per context inside the child -MAX_WORKERS = 6 # mirrors the planned _EXTRACTOR_POOL size -ORPHAN_SLEEP = 3.0 # seconds the orphan child sleeps while we killpg it - - -# --------------------------------------------------------------------------- -# Trivial extractor-like work function (must be importable at module top level -# for forkserver, which re-imports the module to bootstrap workers). -# --------------------------------------------------------------------------- -def trivial_extractor(payload: bytes) -> str: - """Simulate cheap extractor work: hash a small buffer, spin briefly.""" - # ~200 ms of CPU-ish work without importing CAPE modules - digest = payload - for _ in range(200): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -# --------------------------------------------------------------------------- -# Latency measurement -# --------------------------------------------------------------------------- -def measure_latency(context_name: str, n: int = ITERATIONS) -> dict: - """ - Create a fresh ProcessPool, schedule one task, collect the result, - then close+join. Repeat n times. Returns stats dict. - Must be called from a single-threaded process so 'fork' is safe. - """ - ctx = multiprocessing.get_context(context_name) - payload = os.urandom(256) - samples = [] - errors = 0 - - for i in range(n): - t0 = time.perf_counter() - try: - pool = pebble.ProcessPool(max_workers=MAX_WORKERS, context=ctx) - future = pool.schedule(trivial_extractor, args=(payload,)) - result = future.result(timeout=30) - pool.close() - pool.join() - except Exception as exc: - errors += 1 - print(f" [{context_name}] iter {i}: ERROR {exc}", file=sys.stderr) - continue - elapsed = time.perf_counter() - t0 - samples.append(elapsed) - print(f" [{context_name}] iter {i+1}/{n}: {elapsed*1000:.1f} ms result={result[:8]}...") - - if not samples: - return {"context": context_name, "n": n, "errors": errors} - - samples_sorted = sorted(samples) - n_s = len(samples_sorted) - mean_ms = sum(samples) / n_s * 1000 - median_ms = samples_sorted[n_s // 2] * 1000 - p95_ms = samples_sorted[int(n_s * 0.95)] * 1000 - min_ms = samples_sorted[0] * 1000 - max_ms = samples_sorted[-1] * 1000 - - return { - "context": context_name, - "n": n_s, - "errors": errors, - "mean_ms": round(mean_ms, 2), - "median_ms": round(median_ms, 2), - "p95_ms": round(p95_ms, 2), - "min_ms": round(min_ms, 2), - "max_ms": round(max_ms, 2), - } - - -# --------------------------------------------------------------------------- -# Orphan check -# --------------------------------------------------------------------------- -def orphan_child_main(context_name: str, ready_w: int, done_r: int) -> None: - """ - Child process body for the orphan test: - 1. setsid() to become its own process group leader. - 2. Tell parent our pgid via the write end of the pipe. - 3. Create a sub-pool, schedule a long-sleeping task. - 4. Sleep ORPHAN_SLEEP seconds while parent sends SIGKILL to the group. - 5. os._exit(0) -- should never reach here if killpg works. - """ - os.setsid() - pgid = os.getpgid(0) - - # Signal parent that we're ready and tell it our pgid - os.write(ready_w, f"{pgid}\n".encode()) - os.close(ready_w) - os.close(done_r) - - ctx = multiprocessing.get_context(context_name) - try: - pool = pebble.ProcessPool(max_workers=MAX_WORKERS, context=ctx) - # Schedule tasks that will outlive the killpg - futures = [pool.schedule(trivial_extractor, args=(os.urandom(256),)) - for _ in range(MAX_WORKERS)] - # Sleep long enough for parent to killpg us - time.sleep(ORPHAN_SLEEP * 2) - pool.close() - pool.join() - except Exception: - pass - os._exit(0) - - -def run_orphan_check(context_name: str) -> dict: - """ - Fork a child that creates the sub-pool; the child setsid()s. - Parent waits for ready signal, then killpg(SIGKILL), waits, then - checks pgrep for survivors in the child's pgroup. - """ - ready_r, ready_w = os.pipe() - done_r, done_w = os.pipe() # placeholder - - pid = os.fork() - if pid == 0: - # child - os.close(ready_r) - os.close(done_w) - orphan_child_main(context_name, ready_w, done_r) - os._exit(0) # never reached - - # parent - os.close(ready_w) - os.close(done_r) - os.close(done_w) - - # Read pgid from child - data = b"" - while True: - chunk = os.read(ready_r, 64) - if not chunk: - break - data += chunk - if b"\n" in data: - break - os.close(ready_r) - - child_pgid = int(data.strip()) - print(f" [{context_name}] child pgid={child_pgid}; sleeping {ORPHAN_SLEEP}s to let sub-pool spin up...") - time.sleep(ORPHAN_SLEEP) - - print(f" [{context_name}] sending SIGKILL to pgid {child_pgid}") - try: - os.killpg(child_pgid, signal.SIGKILL) - except ProcessLookupError: - pass # already dead - - # Wait for the direct child to be reaped - try: - os.waitpid(pid, 0) - except ChildProcessError: - pass - - # Give OS a moment to clean up - time.sleep(0.5) - - # Check for survivors in the process group - pgrep_result = subprocess.run( - ["pgrep", "-g", str(child_pgid)], - capture_output=True, text=True - ) - survivors = pgrep_result.stdout.strip() - survivor_count = len(survivors.splitlines()) if survivors else 0 - - return { - "context": context_name, - "child_pgid": child_pgid, - "survivors": survivors or "(none)", - "survivor_count": survivor_count, - "pass": survivor_count == 0, - } - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- -def main(): - print("=" * 60) - print(f"pebble version : {pebble.__version__}") - print(f"Python : {sys.version}") - print(f"PID : {os.getpid()}") - print(f"MAX_WORKERS : {MAX_WORKERS}") - print(f"ITERATIONS : {ITERATIONS}") - print("=" * 60) - - # ----------------------------------------------------------------------- - # Phase 1: Latency measurement - # We fork a clean child for each context so that: - # - 'fork' measurements start from a single-threaded process (as it will - # in production: the task child is forked from the supervisor before - # doing any threaded work). - # - Both measurements run from equivalent baseline states. - # ----------------------------------------------------------------------- - latency_results = {} - for ctx_name in ("fork", "forkserver"): - print(f"\n--- Latency: {ctx_name} ---") - r_pipe, w_pipe = os.pipe() - pid = os.fork() - if pid == 0: - # child - os.close(r_pipe) - import json - result = measure_latency(ctx_name, n=ITERATIONS) - payload = json.dumps(result).encode() + b"\n" - os.write(w_pipe, payload) - os.close(w_pipe) - os._exit(0) - else: - os.close(w_pipe) - import json - data = b"" - while True: - chunk = os.read(r_pipe, 4096) - if not chunk: - break - data += chunk - os.close(r_pipe) - os.waitpid(pid, 0) - result = json.loads(data.strip()) - latency_results[ctx_name] = result - - # ----------------------------------------------------------------------- - # Phase 2: Orphan check - # ----------------------------------------------------------------------- - orphan_results = {} - for ctx_name in ("fork", "forkserver"): - print(f"\n--- Orphan check: {ctx_name} ---") - orphan_results[ctx_name] = run_orphan_check(ctx_name) - - # ----------------------------------------------------------------------- - # Summary - # ----------------------------------------------------------------------- - print("\n" + "=" * 60) - print("LATENCY SUMMARY") - print("=" * 60) - fmt = "{:<12} {:>10} {:>10} {:>10} {:>10} {:>10} {:>8}" - print(fmt.format("context", "mean_ms", "median_ms", "p95_ms", "min_ms", "max_ms", "errors")) - print("-" * 70) - for ctx_name in ("fork", "forkserver"): - r = latency_results[ctx_name] - print(fmt.format( - ctx_name, - r.get("mean_ms", "N/A"), - r.get("median_ms", "N/A"), - r.get("p95_ms", "N/A"), - r.get("min_ms", "N/A"), - r.get("max_ms", "N/A"), - r.get("errors", 0), - )) - - print("\n" + "=" * 60) - print("ORPHAN CHECK SUMMARY") - print("=" * 60) - for ctx_name in ("fork", "forkserver"): - r = orphan_results[ctx_name] - status = "PASS (zero survivors)" if r["pass"] else f"FAIL ({r['survivor_count']} survivors)" - print(f" {ctx_name:<12}: pgid={r['child_pgid']} survivors={r['survivors']} -> {status}") - - print("\n" + "=" * 60) - print("DECISION") - print("=" * 60) - fork_pass = orphan_results["fork"]["pass"] - forksvr_pass = orphan_results["forkserver"]["pass"] - fork_mean = latency_results["fork"].get("mean_ms", float("inf")) - forksvr_mean = latency_results["forkserver"].get("mean_ms", float("inf")) - - if not fork_pass and not forksvr_pass: - decision = "NEITHER context passes orphan check -- escalate." - elif not fork_pass: - decision = f"forkserver (fork failed orphan check)" - elif not forksvr_pass: - decision = f"fork (forkserver failed orphan check)" - elif fork_mean <= forksvr_mean: - delta = forksvr_mean - fork_mean - decision = f"fork (faster by {delta:.1f} ms mean; both pass orphan check)" - else: - delta = fork_mean - forksvr_mean - decision = f"forkserver (faster by {delta:.1f} ms mean; both pass orphan check)" - - print(f" Chosen context: {decision}") - - -if __name__ == "__main__": - main() -``` - ---- - -## Raw Output - -``` -============================================================ -pebble version : 5.1.0 -Python : 3.12.3 (main, Mar 23 2026, 19:04:32) [GCC 13.3.0] -PID : 3435721 -MAX_WORKERS : 6 -ITERATIONS : 8 -============================================================ - ---- Latency: fork --- - [fork] iter 1/8: 112.8 ms result=7ee30c1c... - [fork] iter 2/8: 117.2 ms result=7ee30c1c... - [fork] iter 3/8: 117.0 ms result=7ee30c1c... - [fork] iter 4/8: 118.3 ms result=7ee30c1c... - [fork] iter 5/8: 113.9 ms result=7ee30c1c... - [fork] iter 6/8: 114.2 ms result=7ee30c1c... - [fork] iter 7/8: 118.0 ms result=7ee30c1c... - ---- Latency: forkserver --- - [forkserver] iter 1/8: 219.0 ms result=2596f03c... - [forkserver] iter 2/8: 111.3 ms result=2596f03c... - [forkserver] iter 3/8: 111.9 ms result=2596f03c... - [forkserver] iter 4/8: 112.0 ms result=2596f03c... - [forkserver] iter 5/8: 112.3 ms result=2596f03c... - [forkserver] iter 6/8: 112.2 ms result=2596f03c... - [forkserver] iter 7/8: 112.3 ms result=2596f03c... - ---- Orphan check: fork --- - [fork] child pgid=3435871; sleeping 3.0s to let sub-pool spin up... - [fork] sending SIGKILL to pgid 3435871 - ---- Orphan check: forkserver --- - [forkserver] child pgid=3435882; sleeping 3.0s to let sub-pool spin up... - [forkserver] sending SIGKILL to pgid 3435882 - -============================================================ -LATENCY SUMMARY -============================================================ -context mean_ms median_ms p95_ms min_ms max_ms errors ----------------------------------------------------------------------- -fork 115.66 116.95 118.3 112.85 118.3 0 -forkserver 125.52 112.29 218.97 111.27 218.97 0 - -============================================================ -ORPHAN CHECK SUMMARY -============================================================ - fork : pgid=3435871 survivors=(none) -> PASS (zero survivors) - forkserver : pgid=3435882 survivors=(none) -> PASS (zero survivors) - -============================================================ -DECISION -============================================================ - Chosen context: fork (faster by 9.9 ms mean; both pass orphan check) -``` - -Note: the script uses `os.fork()` to run each context measurement in a clean child -process. As a side effect, forked children inherit the parent's buffered stdout -and may re-print the banner header; the actual latency data is transmitted via a -pipe and aggregated correctly in the parent. The raw output above shows only the -relevant lines per context. diff --git a/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md b/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md deleted file mode 100644 index 75612a183e5..00000000000 --- a/docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md +++ /dev/null @@ -1,276 +0,0 @@ -# Processor Engine A/B Runbook - -**Date:** 2026-05-27 -**Scope:** How to flip between `pebble` and `prefork` processing engines, what to measure, how to declare a winner, when to retire the `-mc 0` stopgap. -**Status:** Ready for A/B — branch `feature/processor-prefork-engine` implementation complete. -**Audience:** Engineer running the A/B in production. - ---- - -## Preconditions - -Before running the A/B, confirm: - -1. Branch `feature/processor-prefork-engine` is deployed. Implementation spans commits `47576a166` through `beb829e67` (14 commits: `TaskSource`, engine registry/base, `run_task` adapter, `PebbleEngine`, `PreforkEngine` scaffolding, fork-per-task execution, wall-clock timeout + `killpg`, extractor sub-pool rewrite, and a docs revert). - -2. The existing stopgap drop-in is in place at `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf`: - ``` - # STOPGAP (2026-05-27): maxtasksperchild=0 disables worker recycling. - # Worker recycle at max_tasks>0 deadlocks in multiprocessing _exit_function - # joining the nested _EXTRACTOR_POOL children. See cape-processor redesign. - # Tradeoff: no per-worker memory reclaim; monitor RSS / pair with periodic restart. - [Service] - ExecStart= - ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 - ``` - This drop-in does **not** pass `--engine`, so the pebble engine (argparse default) is active. - -3. `cape-processor.service` is running and healthy (check for recent `Reports generation completed for Task #N` lines in `/opt/CAPEv2/log/process.log`). - -4. You have `sudo` on this host. - ---- - -## Database Note (important) - -The A/B runs against the **production PostgreSQL database**. Both engines share the module-level `db = Database()` instance configured from `conf/cuckoo.conf` (`postgresql://...`). There is no separate staging database path. - -The `sqlite://` database in `tests/conftest.py` is the unit-test fixture only. It has no production code path and is completely irrelevant to the A/B. - ---- - -## Flipping Engines - -### To enable the prefork engine - -Edit the drop-in at `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf`. Add `--engine prefork` to the `ExecStart` line: - -``` -# STOPGAP (2026-05-27): maxtasksperchild=0 disables worker recycling. -# Worker recycle at max_tasks>0 deadlocks in multiprocessing _exit_function -# joining the nested _EXTRACTOR_POOL children. See cape-processor redesign. -# Tradeoff: no per-worker memory reclaim; monitor RSS / pair with periodic restart. -# NOTE: -mc 0 is a pebble-specific workaround; it is harmless but meaningless -# under prefork (different lifecycle, no worker-recycle exit path). -[Service] -ExecStart= -ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 --engine prefork -``` - -Then reload and restart: - -```bash -sudo systemctl daemon-reload -sudo systemctl restart cape-processor.service -``` - -Verify it started with the right engine — look for this log line within a few seconds of restart: - -```bash -sudo journalctl -u cape-processor.service -f --since "1 minute ago" -grep -m1 "engine" /opt/CAPEv2/log/process.log | tail -5 -``` - -Expected: a log line referencing `PreforkEngine` or `engine=prefork` at startup (exact text from `utils/process.py`). - -### To switch back to pebble - -Remove `--engine prefork` from the `ExecStart` line (or explicitly add `--engine pebble`), then: - -```bash -sudo systemctl daemon-reload -sudo systemctl restart cape-processor.service -``` - ---- - -## Rollback - -If at any point you need to revert to the exact pre-A/B pebble configuration, restore the drop-in to its original content (no `--engine` flag): - -``` -[Service] -ExecStart= -ExecStart=/etc/poetry/bin/poetry run python process.py -p20 auto -pt 900 -mc 0 -``` - -Then `sudo systemctl daemon-reload && sudo systemctl restart cape-processor.service`. This fully restores the pebble+`-mc 0` baseline. - ---- - -## Metrics to Compare - -Run each engine for at least **48 hours** (or 500 completed tasks, whichever comes first) under the same workload before drawing conclusions. Collect all four metrics for both engines. - -### 1. Throughput (tasks/min) - -Parse `Reports generation completed` completions from the log, compute rate per minute: - -```bash -grep "Reports generation completed" /opt/CAPEv2/log/process.log \ - | awk '{print $1, $2}' \ - | awk -F'[: ]' '{print $1"T"$2":"$3":00"}' \ - | sort | uniq -c \ - | awk '{printf "%s completions=%d\n", $2, $1}' -``` - -Or a simpler rolling summary over the last hour: - -```bash -SINCE=$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S') -awk -v since="$SINCE" '$0 >= since && /Reports generation completed/' \ - /opt/CAPEv2/log/process.log | wc -l -``` - -Target: prefork throughput within 5% of pebble. Any significant regression needs investigation before declaring prefork the winner. - -### 2. Wedge incidents - -A "wedge" is a period of >5 minutes with no `Reports generation completed` lines while tasks are still scheduled. The `-mc 0` stopgap eliminated wedges for pebble; prefork should also have zero. - -Check for silent gaps in the last 24 hours: - -```bash -grep "Reports generation completed" /opt/CAPEv2/log/process.log \ - | awk '{print $1" "$2}' \ - | awk 'NR>1 { - cmd="date -d \""$1"\" +%s"; cmd | getline ts; close(cmd) - cmd2="date -d \""prev"\" +%s"; cmd2 | getline pts; close(cmd2) - gap = ts - pts - if (gap > 300) printf "GAP %d s ending at %s\n", gap, $1 - prev=$1" "$2 - } - NR==1 {prev=$1" "$2}' -``` - -Alternatively, watch the heartbeat log line that `PreforkEngine` emits periodically (`in_flight=N oldest=Xs`). A heartbeat without completions for >5 minutes while `in_flight > 0` is an early wedge indicator. - -**Decision criterion:** any wedge under either engine is a failure for that engine. Both should show zero over the observation window. - -### 3. Peak and sustained RSS - -Sum the RSS of all python workers under the processor's main PID at 5-minute intervals. First find the service's MainPID: - -```bash -MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) -echo "MainPID: $MAIN_PID" -``` - -Then poll (run in a loop, or via cron, or as a one-shot snapshot): - -```bash -# One snapshot (GB): -MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) -ps --no-headers -o rss --ppid "$MAIN_PID" \ - | awk '{sum+=$1} END {printf "RSS %.2f GB (%d workers)\n", sum/1048576, NR}' -``` - -For a continuous 5-minute polling log: - -```bash -while true; do - MAIN_PID=$(systemctl show cape-processor.service --property=MainPID --value) - RSS=$(ps --no-headers -o rss --ppid "$MAIN_PID" \ - | awk '{sum+=$1} END {printf "%.2f", sum/1048576}') - echo "$(date '+%Y-%m-%dT%H:%M:%S') RSS=${RSS}GB" - sleep 300 -done | tee /tmp/rss-log.txt -``` - -**Expected behavior difference:** -- Pebble with `-mc 0`: RSS grows gradually over time (workers never recycle, C-lib leaks accumulate). This is the known trade-off of the `-mc 0` stopgap. -- Prefork: RSS should be roughly steady (each task child exits via `os._exit()`, reclaiming its memory per-task). A monotonically growing RSS trend under prefork is a regression. - -**Decision criterion:** prefork RSS should not trend upward continuously over 48 hours. Pebble RSS growing is expected and acceptable (it is a known stopgap trade-off). A periodic `RuntimeMaxSec` systemd restart can bound pebble RSS if needed. - -### 4. Orphan count after induced timeouts - -This metric requires inducing a task that exceeds `-pt 900` (or temporarily reducing `-pt` for a test). It validates that `killpg` in prefork actually cleans up the entire process tree and leaves no grandchildren reparented to init. - -**Procedure:** - -1. Before the test, note the current `in_flight` from logs or the heartbeat line. -2. Submit (or identify) a task known to run longer than the current `-pt` value. Alternatively: temporarily set `-pt 60` in the drop-in to trigger timeouts faster, then restore. -3. When a `Processing timeout` line appears in `process.log`, immediately check for orphans: - -```bash -# Check for python processes reparented to init (PPID=1): -pgrep -P 1 | xargs -I{} ps -o pid,ppid,cmd -p {} 2>/dev/null \ - | grep -i python - -# Or check for any surviving members of the task's process group. -# Get the task child PID from the timeout log line first, then: -# (The supervisor logs the child PID and PGID in the SIGTERM/SIGKILL log lines.) -TASK_PGID= -pgrep -g "$TASK_PGID" -``` - -**Expected results:** -- Prefork: `pgrep` returns nothing. The `killpg` swept the entire process group (task child + extractor sub-pool workers). Zero orphans. -- Pebble: `pgrep` may return orphaned grandchildren. Pebble's `stop_process` sends a signal to one worker PID; it does not sweep the extractor sub-pool children, which may reparent to init. - -**Decision criterion:** prefork must show zero orphans after any induced timeout. Non-zero is a regression in the primary correctness guarantee of the new engine. - ---- - -## Decision Criterion - -Declare prefork the **winner** if, over the observation window (48 hours or 500 tasks): - -- Throughput within 5% of pebble (tasks/min). -- Zero wedge incidents (same as pebble with `-mc 0`). -- RSS steady-state, not trending upward. -- Zero orphaned processes after any induced or naturally-occurring timeout. - -Declare pebble the **winner** (and treat prefork as needing more work) if any of: - -- Prefork throughput is >5% lower than pebble. -- Prefork wedges (even once). -- Prefork RSS trends upward monotonically over 48 hours. -- Prefork leaves orphans after a timeout. - -If there is a prefork regression, file an issue, roll back to pebble (one `systemctl` command — see Rollback above), and investigate before re-attempting the A/B. - ---- - -## Stopgap Retirement - -The `-mc 0` flag in the drop-in exists solely to work around the pebble worker-recycle deadlock (`multiprocessing._exit_function` joining the nested `_EXTRACTOR_POOL` children). It is a **pebble-specific stopgap**. - -Under `--engine prefork`, the `-mc 0` flag is meaningless (prefork has no worker-recycle exit path) but harmless. It does not need to be removed immediately. - -**When to remove `-mc 0`:** once `--engine prefork` is made the argparse default in `utils/process.py` (a one-line code change flipping the `--engine` default from `pebble` to `prefork`, done after A/B declares prefork the winner), the pebble engine will no longer be the active engine on startup and the `-mc 0` stopgap becomes dead weight. At that point: - -1. Remove the `-mc 0` argument from the drop-in's `ExecStart` line. -2. The drop-in itself can eventually be deleted entirely once `--engine prefork` is also baked into the unit file's `ExecStart` (or once the old pebble path is removed from the codebase). - -Do not remove `-mc 0` earlier than this. It is harmless under prefork and removing it prematurely would leave the pebble engine unprotected if someone ever toggles back. - ---- - -## Open Follow-ups - -These items are not part of the A/B itself but are outstanding work to track: - -1. **Uncommitted `resolver_pool` lazy fix.** `lib/cuckoo/common/cleaners_utils.py` contains the `_LazyThreadPool` fix that defers `ThreadPool(50)` creation to the first call (reduces supervisor thread count 73 → 20, making prefork forks clean). This fix is in the working tree on `feature/processor-prefork-engine` but has not been committed as a standalone commit. It should be committed on a separate branch or PR — see §9 of the spec (`docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md`). - -2. **Proctitle/formatter cleanup deferral.** Task 4 (`d944dcc4d`) restored per-task `proctitle` and logging formatter cleanup in `run_task`. The prefork engine's per-task children already have isolated address spaces (no shared log handlers), so the cleanup is technically redundant under prefork — but it was kept for parity and correctness under the pebble path. A future cleanup could remove it from `run_task` and make it engine-specific if it shows measurable cost. - -3. **`--tasks-per-child` knob.** The design spec (§4.2) documents this as a YAGNI future knob: recycling a prefork child across N>1 tasks requires a supervisor→child dispatch channel, which was deliberately not built. If the measured per-task fork cost ever becomes material, this is the documented path to amortize it. Current measured cost: ~0.35 s/task (~1.6% of p50 22 s task) — not material. - -4. **pyattck `techniques` recompute.** Observed burning CPU in `_get_relationship_objects` on each access. The fix (pre-compute in the warm base so children inherit the cached result via COW) was deferred as an A/B-measured optimization. If the A/B reveals CPU spikes on prefork attributable to pyattck, this is the first place to look. - ---- - -## Related Artifacts - -| Artifact | Path | -|---|---| -| Concurrency redesign spec | `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` | -| Implementation plan (11 tasks) | `docs/superpowers/plans/2026-05-27-processor-prefork-engine.md` | -| Extractor sub-pool spike (fork vs forkserver) | `docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md` | -| Stopgap drop-in (live) | `/etc/systemd/system/cape-processor.service.d/stopgap-no-recycle.conf` | -| Processor entry point | `utils/process.py` | -| Engine implementations | `lib/cuckoo/core/processing_engine/pebble.py`, `lib/cuckoo/core/processing_engine/prefork.py` | -| Engine registry | `lib/cuckoo/core/processing_engine/__init__.py` | -| Extractor sub-pool (rewritten in Task 10) | `lib/cuckoo/common/integrations/file_extra_info.py` | diff --git a/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md b/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md deleted file mode 100644 index 8d63c652769..00000000000 --- a/docs/superpowers/plans/2026-05-27-processor-prefork-engine.md +++ /dev/null @@ -1,986 +0,0 @@ -# Processor Prefork Engine Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the deadlock-prone pebble worker pool in `utils/process.py` with a pluggable engine layer: the existing pebble loop preserved as an A/B control, and a new single-threaded **prefork supervisor** (fork-per-task children, `os._exit`, process-group timeout kill) as the reliability target. - -**Architecture:** A thin `autoprocess()` dispatches to a `ProcessingEngine` chosen by `--engine {pebble,prefork}`. Both engines share a `TaskSource` (DB poll + status writes) and a `run_task(task)` adapter. `PebbleEngine` wraps today's loop unchanged. `PreforkEngine` is a single-threaded supervisor that forks one child per task; the child `os.setsid()`s, runs the task, and `os._exit()`s; the supervisor reaps children and enforces a launch-relative wall-clock timeout via `os.killpg`. No supervisor↔child channel exists. - -**Tech Stack:** Python 3.12, SQLAlchemy (CAPE `Database`), pytest 7.2.2 (`poetry run pytest`, sqlite `db` fixture), `os.fork`/`waitpid`/`killpg`/`setsid`. - -**Spec:** `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` - -**Conventions for every task below:** -- Run tests with: `cd /opt/CAPEv2 && poetry run pytest -v` -- Commit only the files listed in the task (the repo working tree has many unrelated dirty files — never `git add -A`). -- Status constants import from `lib.cuckoo.core.data.task` (`TASK_COMPLETED`, `TASK_FAILED_PROCESSING`, `TASK_REPORTED`, `Task`). - ---- - -## Phase 1 — Engine seam (behavior-preserving) - -After Phase 1, behavior is unchanged: `--engine pebble` (the default) runs exactly today's loop, now behind the seam. - -### Task 1: `TaskSource` — DB polling + status writes - -**Files:** -- Create: `lib/cuckoo/core/processing_engine/__init__.py` (empty) -- Create: `lib/cuckoo/core/processing_engine/source.py` -- Test: `tests/test_processing_engine_source.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_processing_engine_source.py -import pytest -from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING -from lib.cuckoo.core.processing_engine.source import TaskSource - - -def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): - t1 = db.add_path(temp_pe32) - t2 = db.add_path(temp_pe32) - db.set_status(t1, TASK_COMPLETED) - db.set_status(t2, TASK_COMPLETED) - - src = TaskSource(db) - got = src.fetch(limit=10, exclude_ids={t2}) - ids = [t.id for t in got] - assert t1 in ids and t2 not in ids - - -def test_mark_failed_sets_status(db, temp_pe32): - t1 = db.add_path(temp_pe32) - db.set_status(t1, TASK_COMPLETED) - - src = TaskSource(db) - src.mark_failed(t1) - assert db.view_task(t1).status == TASK_FAILED_PROCESSING -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_processing_engine_source.py -v` -Expected: FAIL with `ModuleNotFoundError: lib.cuckoo.core.processing_engine.source` - -- [ ] **Step 3: Write minimal implementation** - -```python -# lib/cuckoo/core/processing_engine/source.py -"""Shared task source for processing engines: pulls tasks needing processing -from the DB and writes terminal status. Engines differ in how they *run* tasks, -not in how they pull them.""" -import logging - -from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, Task - -log = logging.getLogger(__name__) - - -class TaskSource: - def __init__(self, db, failed_processing=False): - self.db = db - self._status = TASK_FAILED_PROCESSING if failed_processing else TASK_COMPLETED - - def fetch(self, limit, exclude_ids): - """Return up to `limit` tasks needing processing, excluding `exclude_ids` - (in-flight). Tasks are expunged so they are safe to use after the txn.""" - if limit <= 0: - return [] - with self.db.session.begin(): - tasks = self.db.list_tasks(status=self._status, limit=limit, order_by=Task.completed_on.asc()) - self.db.session.expunge_all() - return [t for t in tasks if t.id not in exclude_ids] - - def mark_failed(self, task_id): - with self.db.session.begin(): - self.db.set_status(task_id, TASK_FAILED_PROCESSING) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_processing_engine_source.py -v` -Expected: PASS (2 passed) - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/__init__.py lib/cuckoo/core/processing_engine/source.py tests/test_processing_engine_source.py -git commit -m "feat(processor): add TaskSource for engine task polling/status" -``` - ---- - -### Task 2: `ProcessingEngine` base + `get_engine()` registry - -**Files:** -- Create: `lib/cuckoo/core/processing_engine/base.py` -- Modify: `lib/cuckoo/core/processing_engine/__init__.py` -- Test: `tests/test_processing_engine_registry.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_processing_engine_registry.py -import pytest -from lib.cuckoo.core.processing_engine import get_engine -from lib.cuckoo.core.processing_engine.base import ProcessingEngine - - -def _noop(*a, **k): - pass - - -def test_get_engine_returns_requested_class(): - eng = get_engine("pebble", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) - assert isinstance(eng, ProcessingEngine) - - -def test_get_engine_unknown_raises(): - with pytest.raises(ValueError): - get_engine("nope", task_fn=_noop, worker_init=_noop, source=None, parallel=2, timeout=900) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_processing_engine_registry.py -v` -Expected: FAIL with `ImportError`/`cannot import name 'get_engine'` - -- [ ] **Step 3: Write minimal implementation** - -```python -# lib/cuckoo/core/processing_engine/base.py -"""Abstract processing engine: drives the per-task lifecycle for autoprocess. -Concrete engines (pebble, prefork) differ only in worker isolation.""" - - -class ProcessingEngine: - def __init__(self, task_fn, worker_init, source, parallel, timeout): - self.task_fn = task_fn # task_fn(task) -> None: runs ONE task to completion - self.worker_init = worker_init # called in worker context (pool init / post-fork) - self.source = source # TaskSource - self.parallel = parallel - self.timeout = timeout - - def run(self): - raise NotImplementedError -``` - -```python -# lib/cuckoo/core/processing_engine/__init__.py -from lib.cuckoo.core.processing_engine.base import ProcessingEngine - - -def get_engine(name, **kwargs): - """Construct the named engine. Imports are local so that selecting one engine - never imports the other's dependencies.""" - if name == "pebble": - from lib.cuckoo.core.processing_engine.pebble import PebbleEngine - return PebbleEngine(**kwargs) - if name == "prefork": - from lib.cuckoo.core.processing_engine.prefork import PreforkEngine - return PreforkEngine(**kwargs) - raise ValueError("unknown processing engine: %r" % name) - - -__all__ = ["ProcessingEngine", "get_engine"] -``` - -NOTE: this task references `pebble.PebbleEngine` (Task 4) and `prefork.PreforkEngine` (Task 5). The registry test only exercises the `pebble` branch, which Task 4 makes importable. Run Task 2's test again after Task 4. For now, add a temporary stub so Task 2 passes in isolation: - -```python -# lib/cuckoo/core/processing_engine/pebble.py (TEMPORARY STUB — replaced in Task 4) -from lib.cuckoo.core.processing_engine.base import ProcessingEngine - - -class PebbleEngine(ProcessingEngine): - def run(self): - raise NotImplementedError -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_processing_engine_registry.py -v` -Expected: PASS (2 passed) - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/base.py lib/cuckoo/core/processing_engine/__init__.py lib/cuckoo/core/processing_engine/pebble.py tests/test_processing_engine_registry.py -git commit -m "feat(processor): add ProcessingEngine base + get_engine registry" -``` - ---- - -### Task 3: Extract `run_task(task)` adapter in `process.py` - -The per-task argument building currently inlined in `autoprocess()` (`utils/process.py` ~lines 463–470: sample-hash lookup + building `args`/`kwargs` for `process()`) becomes a standalone `run_task(task)` so both engines share it. - -**Files:** -- Modify: `utils/process.py` (add `run_task`; reference existing `process()` at line 107 and `db`) -- Test: `tests/test_run_task_adapter.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_run_task_adapter.py -import types -import utils.process as proc - - -def test_run_task_calls_process_with_task_and_auto(monkeypatch, db, temp_pe32): - captured = {} - - def fake_process(target=None, sample_sha256=None, task=None, report=False, auto=False, **kw): - captured["task_id"] = task.id - captured["auto"] = auto - captured["report"] = report - - monkeypatch.setattr(proc, "process", fake_process) - monkeypatch.setattr(proc, "db", db) - - tid = db.add_path(temp_pe32) - task = db.view_task(tid) - proc.run_task(task) - - assert captured["task_id"] == tid - assert captured["auto"] is True and captured["report"] is True -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_run_task_adapter.py -v` -Expected: FAIL with `AttributeError: module 'utils.process' has no attribute 'run_task'` - -- [ ] **Step 3: Write minimal implementation** - -Add to `utils/process.py` (near `process()`), preserving the existing sample-hash logic from `autoprocess`: - -```python -def run_task(task, memory_debugging=False, debug=False): - """Run exactly one completed task to completion (processing -> report). - Extracted from autoprocess so every engine shares identical per-task setup.""" - sample_hash = "" - if task.category != "url": - with db.session.begin(): - sample = db.view_sample(task.sample_id) - if sample: - sample_hash = sample.sha256 - process( - task.target, - sample_hash, - report=True, - auto=True, - task=task, - memory_debugging=memory_debugging, - debug=debug, - ) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_run_task_adapter.py -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add utils/process.py tests/test_run_task_adapter.py -git commit -m "refactor(processor): extract run_task(task) adapter shared by engines" -``` - ---- - -### Task 4: `PebbleEngine` (relocate today's loop) + wire `autoprocess` + `--engine` flag - -Move the current pebble pool loop (`utils/process.py` `autoprocess` body, ~lines 430–508, plus `processing_finished`, ~lines 376–407) into `PebbleEngine`, calling injected `task_fn`/`worker_init`/`source` instead of the inline DB/process code. Behavior must be identical to today (including `-mc 0` default). - -**Files:** -- Modify: `lib/cuckoo/core/processing_engine/pebble.py` (replace the Task 2 stub) -- Modify: `utils/process.py` (`autoprocess()` → build `TaskSource` + `get_engine`; add `--engine` arg; keep `init_worker`, `process`, `run_task`) -- Test: `tests/test_pebble_engine.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_pebble_engine.py -import threading -import time -from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_REPORTED -from lib.cuckoo.core.processing_engine.pebble import PebbleEngine -from lib.cuckoo.core.processing_engine.source import TaskSource - - -def test_pebble_engine_processes_one_task(db, temp_pe32, monkeypatch): - tid = db.add_path(temp_pe32) - db.set_status(tid, TASK_COMPLETED) - - ran = threading.Event() - - def fake_task_fn(task): - with db.session.begin(): - db.set_status(task.id, TASK_REPORTED) - ran.set() - - eng = PebbleEngine(task_fn=fake_task_fn, worker_init=lambda: None, - source=TaskSource(db), parallel=2, timeout=30, max_count=1) - eng.run() - assert ran.wait(timeout=20) - assert db.view_task(tid).status == TASK_REPORTED -``` - -NOTE: `max_count` lets the loop exit after N scheduled tasks for tests (mirror the existing `cfg.cuckoo.max_analysis_count` exit; default 0 = run forever in production). - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_pebble_engine.py -v` -Expected: FAIL with `NotImplementedError` (Task 2 stub) or `TypeError` (no `max_count`). - -- [ ] **Step 3: Write minimal implementation** - -```python -# lib/cuckoo/core/processing_engine/pebble.py -"""Pebble-pool engine — preserved as the A/B control. Mirrors the historical -autoprocess loop, parameterized behind the engine seam. max_tasks defaults to 0 -(no recycle): worker recycling deadlocks in multiprocessing _exit_function while -joining the nested extractor pool — see the redesign spec.""" -import logging -import time - -import pebble - -from lib.cuckoo.core.processing_engine.base import ProcessingEngine - -log = logging.getLogger(__name__) - - -class PebbleEngine(ProcessingEngine): - def __init__(self, task_fn, worker_init, source, parallel, timeout, - max_tasks=0, max_count=0): - super().__init__(task_fn, worker_init, source, parallel, timeout) - self.max_tasks = max_tasks - self.max_count = max_count - self._pending = {} # future -> task_id - - def _done(self, future): - task_id = self._pending.pop(future, None) - try: - future.result() - except Exception: - log.exception("[%s] pebble: task failed", task_id) - if task_id is not None: - self.source.mark_failed(task_id) - - def run(self): - count = 0 - with pebble.ProcessPool(max_workers=self.parallel, max_tasks=self.max_tasks, - initializer=self.worker_init) as pool: - while not self.max_count or count < self.max_count: - if len(self._pending) >= self.parallel: - time.sleep(1) - continue - tasks = self.source.fetch(limit=self.parallel, - exclude_ids=set(self._pending.values())) - added = False - for task in tasks: - future = pool.schedule(self.task_fn, args=(task,), timeout=self.timeout) - self._pending[future] = task.id - future.add_done_callback(self._done) - count += 1 - added = True - break - if not added and not self.max_count: - time.sleep(1) - if not added and self.max_count: - break - # drain - while self._pending: - time.sleep(0.2) -``` - -In `utils/process.py`, replace the body of `autoprocess()` with engine dispatch (keep `memory_limit()`/`free_space_monitor` setup as today), and add the CLI flag: - -```python -# autoprocess() body (replace the pebble loop): -from lib.cuckoo.core.processing_engine import get_engine -from lib.cuckoo.core.processing_engine.source import TaskSource - -def autoprocess(parallel=1, failed_processing=False, maxtasksperchild=0, - memory_debugging=False, processing_timeout=300, debug=False, - disable_memory_limit=False, engine="pebble"): - if not disable_memory_limit: - memory_limit() - log.info("Processing analysis data (engine=%s)", engine) - source = TaskSource(db, failed_processing=failed_processing) - eng = get_engine( - engine, - task_fn=lambda task: run_task(task, memory_debugging=memory_debugging, debug=debug), - worker_init=init_worker, - source=source, - parallel=parallel, - timeout=processing_timeout, - ) - if engine == "pebble": - eng.max_tasks = maxtasksperchild - eng.run() -``` - -```python -# argparse (near the other autoprocess args): -parser.add_argument("--engine", choices=["pebble", "prefork"], default="pebble", - help="Processing engine: pebble (default, A/B control) or prefork.") -# main() autoprocess(...) call: add engine=args.engine -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_pebble_engine.py tests/test_processing_engine_registry.py -v` -Expected: PASS (3 passed) - -- [ ] **Step 5: Smoke-test the real CLI (no behavior change)** - -Run: `poetry run python utils/process.py --help` — Expected: shows `--engine {pebble,prefork}`. -Run (brief, then Ctrl-C): `poetry run python utils/process.py -p2 auto -pt 900 --engine pebble` — Expected: logs `Processing analysis data (engine=pebble)` and processes as before. - -- [ ] **Step 6: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/pebble.py utils/process.py tests/test_pebble_engine.py -git commit -m "feat(processor): PebbleEngine behind engine seam + --engine flag (default pebble)" -``` - ---- - -## Phase 2 — PreforkEngine - -A single-threaded supervisor. All tests inject a fake `task_fn` (sleep/crash/normal) so the real processing pipeline is not needed. - -### Task 5: Supervisor scaffolding + concurrency cap + single-threaded invariant - -**Files:** -- Create: `lib/cuckoo/core/processing_engine/prefork.py` (replace any stub) -- Test: `tests/test_prefork_engine.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_prefork_engine.py -import os -import threading -import time - -from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING, TASK_REPORTED -from lib.cuckoo.core.processing_engine.prefork import PreforkEngine -from lib.cuckoo.core.processing_engine.source import TaskSource - - -def test_single_threaded_invariant_raises_when_extra_thread(db): - eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, - source=TaskSource(db), parallel=2, timeout=30) - stop = threading.Event() - t = threading.Thread(target=stop.wait) - t.start() - try: - import pytest - with pytest.raises(RuntimeError, match="single-threaded"): - eng._assert_single_threaded() - finally: - stop.set() - t.join() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_prefork_engine.py -v` -Expected: FAIL with `ModuleNotFoundError`/`AttributeError`. - -- [ ] **Step 3: Write minimal implementation** - -```python -# lib/cuckoo/core/processing_engine/prefork.py -"""Single-threaded prefork supervisor: forks one child per task, each child runs -exactly one task then os._exit(); the supervisor reaps children and enforces a -launch-relative wall-clock timeout via process-group kill. No supervisor<->child -channel exists. See docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md.""" -import logging -import os -import signal -import threading -import time - -from lib.cuckoo.core.processing_engine.base import ProcessingEngine - -log = logging.getLogger(__name__) - - -class _Child: - __slots__ = ("task_id", "pid", "start", "pgid", "timed_out", "kill_deadline") - - def __init__(self, task_id, pid, start, pgid): - self.task_id = task_id - self.pid = pid - self.start = start - self.pgid = pgid - self.timed_out = False - self.kill_deadline = None - - -class PreforkEngine(ProcessingEngine): - def __init__(self, task_fn, worker_init, source, parallel, timeout, - heartbeat_interval=30, term_grace=5, max_count=0, poll_interval=0.2): - super().__init__(task_fn, worker_init, source, parallel, timeout) - self.heartbeat_interval = heartbeat_interval - self.term_grace = term_grace - self.max_count = max_count - self.poll_interval = poll_interval - self._inflight = {} # pid -> _Child - - def _assert_single_threaded(self): - n = threading.active_count() - if n != 1: - raise RuntimeError( - "prefork supervisor must be single-threaded before fork; " - "active_count=%d (a thread/Mongo client/pool leaked into the supervisor)" % n) - - def _inflight_task_ids(self): - return {c.task_id for c in self._inflight.values()} - - def _heartbeat(self): - if self._inflight: - oldest = max(time.monotonic() - c.start for c in self._inflight.values()) - else: - oldest = 0.0 - log.info("prefork heartbeat: in_flight=%d oldest=%.0fs", len(self._inflight), oldest) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_prefork_engine.py -v` -Expected: PASS (1 passed) - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py -git commit -m "feat(processor): PreforkEngine scaffolding + single-threaded invariant" -``` - ---- - -### Task 6: Fork-per-task execution + reap + status (child sets / supervisor overrides) - -**Files:** -- Modify: `lib/cuckoo/core/processing_engine/prefork.py` -- Test: `tests/test_prefork_engine.py` - -- [ ] **Step 1: Write the failing test** - -```python -# append to tests/test_prefork_engine.py - -def test_normal_task_runs_and_status_set_by_child(db, temp_pe32): - tid = db.add_path(temp_pe32) - db.set_status(tid, TASK_COMPLETED) - - # child runs this; it must set status itself (child sets, supervisor overrides only on failure) - def task_fn(task): - # fresh DB handle in child after fork - from lib.cuckoo.core.database import Database - Database().set_status(task.id, TASK_REPORTED) - - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, - source=TaskSource(db), parallel=2, timeout=30, max_count=1) - eng.run() - assert db.view_task(tid).status == TASK_REPORTED - - -def test_crashing_task_marked_failed_by_supervisor(db, temp_pe32): - tid = db.add_path(temp_pe32) - db.set_status(tid, TASK_COMPLETED) - - def task_fn(task): - os._exit(3) # simulate abnormal exit / crash - - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, - source=TaskSource(db), parallel=2, timeout=30, max_count=1) - eng.run() - assert db.view_task(tid).status == TASK_FAILED_PROCESSING -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_prefork_engine.py -v` -Expected: FAIL (`run`/`_launch`/`_reap` not implemented). - -- [ ] **Step 3: Write minimal implementation** - -```python -# add methods to PreforkEngine - - def _child_main(self, task): - os.setsid() # own session/process group; supervisor killpg sweeps the subtree - try: - self.worker_init() - self.task_fn(task) - return 0 - except BaseException: - log.exception("prefork child: task %s crashed", getattr(task, "id", "?")) - return 1 - - def _launch(self, task): - self._assert_single_threaded() - pid = os.fork() - if pid == 0: - code = 1 - try: - code = self._child_main(task) - finally: - os._exit(code) # NEVER return: skip multiprocessing atexit join - self._inflight[pid] = _Child(task_id=task.id, pid=pid, start=time.monotonic(), pgid=pid) - log.debug("prefork: launched task %d as pid %d", task.id, pid) - - def _reap(self): - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - return - if pid == 0: - return - child = self._inflight.pop(pid, None) - if child is None: - continue - if child.timed_out: - continue # already marked failed during timeout enforcement - ok = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 - if not ok: - log.warning("prefork: task %d (pid %d) abnormal exit status=%d -> FAILED_PROCESSING", - child.task_id, pid, status) - self.source.mark_failed(child.task_id) - - def run(self): - count = 0 - last_hb = 0.0 - while True: - self._reap() - if self.max_count and count >= self.max_count and not self._inflight: - return - free = self.parallel - len(self._inflight) - launchable = free if not self.max_count else min(free, self.max_count - count) - if launchable > 0: - tasks = self.source.fetch(limit=launchable, exclude_ids=self._inflight_task_ids()) - for task in tasks[:launchable]: - self._launch(task) - count += 1 - now = time.monotonic() - if now - last_hb >= self.heartbeat_interval: - self._heartbeat() - last_hb = now - time.sleep(self.poll_interval) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_prefork_engine.py -v` -Expected: PASS (3 passed) - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py -git commit -m "feat(processor): prefork fork-per-task execution + reap + failure status" -``` - ---- - -### Task 7: Wall-clock timeout via `killpg` + no-orphan cleanup - -**Files:** -- Modify: `lib/cuckoo/core/processing_engine/prefork.py` -- Test: `tests/test_prefork_engine.py` - -- [ ] **Step 1: Write the failing test** - -```python -# append to tests/test_prefork_engine.py - -def test_timeout_kills_process_group_no_orphans(db, temp_pe32): - tid = db.add_path(temp_pe32) - db.set_status(tid, TASK_COMPLETED) - - marker = "/tmp/prefork_orphan_%d" % os.getpid() - - def task_fn(task): - # spawn a grandchild that would outlive the worker, then hang - import subprocess, sys - subprocess.Popen([sys.executable, "-c", - "import time,os;open(%r,'w').close();time.sleep(120)" % marker]) - time.sleep(120) - - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, source=TaskSource(db), - parallel=1, timeout=1, term_grace=1, max_count=1, poll_interval=0.1) - if os.path.exists(marker): - os.unlink(marker) - eng.run() - - assert db.view_task(tid).status == TASK_FAILED_PROCESSING - # grandchild must have been swept by killpg (give it a moment) - time.sleep(2) - # nothing holding the marker's process group alive: assert no python sleeping on it - # (best-effort: the grandchild process should be gone) - import subprocess - out = subprocess.run(["pgrep", "-f", marker], capture_output=True, text=True) - assert out.stdout.strip() == "", "orphaned grandchild survived killpg" - if os.path.exists(marker): - os.unlink(marker) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_prefork_engine.py::test_timeout_kills_process_group_no_orphans -v` -Expected: FAIL (timeout enforcement not implemented → test hangs to its own safety or task not failed). If it hangs, that confirms missing enforcement; add enforcement in Step 3. - -- [ ] **Step 3: Write minimal implementation** - -```python -# add to PreforkEngine - - def _enforce_timeouts(self): - now = time.monotonic() - for child in list(self._inflight.values()): - if child.timed_out or now - child.start <= self.timeout: - continue - log.error("prefork: task %d (pid %d) exceeded %ds -> killpg", - child.task_id, child.pid, self.timeout) - child.timed_out = True - self.source.mark_failed(child.task_id) - try: - os.killpg(child.pgid, signal.SIGTERM) - except ProcessLookupError: - continue - child.kill_deadline = now + self.term_grace - - def _escalate_kills(self): - now = time.monotonic() - for child in list(self._inflight.values()): - if child.timed_out and child.kill_deadline and now > child.kill_deadline: - try: - os.killpg(child.pgid, signal.SIGKILL) - except ProcessLookupError: - pass - child.kill_deadline = None -``` - -Wire both into `run()`'s loop, immediately after `self._reap()`: - -```python - self._reap() - self._enforce_timeouts() - self._escalate_kills() -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_prefork_engine.py -v` -Expected: PASS (4 passed) - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py -git commit -m "feat(processor): prefork wall-clock timeout via killpg + grace escalation" -``` - ---- - -### Task 8: Post-fork child reinit + wire `--engine prefork` - -`init_worker()` today is the pebble pool initializer (disposes the SQLAlchemy engine post-fork, resets log handlers, pre-compiles YARA). For prefork it runs in `_child_main` via the injected `worker_init`. Confirm it is fork-safe to call in the child (it already does `db.engine.dispose(close=False)` and resets handlers without lock-acquiring calls). - -**Files:** -- Modify: `utils/process.py` (`autoprocess` already passes `worker_init=init_worker` and `engine=args.engine` from Task 4 — verify prefork path constructs cleanly) -- Test: `tests/test_prefork_engine.py` - -- [ ] **Step 1: Write the failing test** - -```python -# append to tests/test_prefork_engine.py - -def test_worker_init_called_in_child(db, temp_pe32, tmp_path): - tid = db.add_path(temp_pe32) - db.set_status(tid, TASK_COMPLETED) - flag = str(tmp_path / "winit_ran") - - def worker_init(): - open(flag, "w").close() - - def task_fn(task): - assert os.path.exists(flag), "worker_init must run before task_fn in child" - from lib.cuckoo.core.database import Database - Database().set_status(task.id, TASK_REPORTED) - - eng = PreforkEngine(task_fn=task_fn, worker_init=worker_init, source=TaskSource(db), - parallel=1, timeout=30, max_count=1) - eng.run() - assert db.view_task(tid).status == TASK_REPORTED -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_prefork_engine.py::test_worker_init_called_in_child -v` -Expected: PASS already if Task 6 calls `self.worker_init()` before `self.task_fn(task)` in `_child_main`. If it FAILS, fix ordering in `_child_main`. - -- [ ] **Step 3: Verify ordering (no code change expected)** - -Confirm `_child_main` calls `self.worker_init()` then `self.task_fn(task)`. If not, reorder. - -- [ ] **Step 4: Smoke-test the real prefork CLI against the live DB (read-only-ish)** - -Run (brief, then Ctrl-C): `poetry run python utils/process.py -p2 auto -pt 900 --engine prefork` -Expected: logs `Processing analysis data (engine=prefork)`, a `prefork heartbeat: in_flight=...` line, and tasks transition to reported. Verify no `_exit_function`/`join` stacks via `py-spy dump` on a child (should show the task running, not multiprocessing atexit). - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/core/processing_engine/prefork.py tests/test_prefork_engine.py -git commit -m "feat(processor): verify post-fork worker_init ordering for prefork children" -``` - ---- - -## Phase 3 — Extractor sub-pool + finalize - -### Task 9: SPIKE — resolve §4.2(d) extractor sub-pool spawn strategy by measurement - -The nested extractor pool in `lib/cuckoo/common/integrations/file_extra_info.py` (`_EXTRACTOR_POOL`) must, under prefork, be: created inside the task child, explicitly `close()`+`join()`d before the child `os._exit()`s, and entirely within the child's process group (so `killpg` sweeps it). The open question is the spawn context: `forkserver` (clean, re-imports per task) vs `fork` from the warm single-threaded child (inherits modules cheaply, must be created before the child spawns threads). - -**Files:** -- Create: `docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md` (findings only; no production code) - -- [ ] **Step 1: Measure both strategies** - -Write a throwaway script that, inside a forked child, builds (a) `pebble.ProcessPool(context=multiprocessing.get_context("forkserver"))` and (b) `...get_context("fork")`, schedules a trivial extractor-like function across `max_workers=6`, and records: pool-create+first-result latency, and whether `killpg` of the child's group leaves any survivors (`pgrep`). Run on a representative extracted-file workload. - -- [ ] **Step 2: Record the decision** - -Write findings + the chosen context to the notes file, with the latency numbers and the orphan check. Decision criterion: choose the faster strategy that leaves zero survivors after `killpg` and runs cleanly with `os._exit()`. - -- [ ] **Step 3: Commit** - -```bash -git add docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md -git commit -m "docs(processor): spike results for extractor sub-pool spawn strategy" -``` - ---- - -### Task 10: Make `_EXTRACTOR_POOL` prefork-safe (explicit teardown + process-group membership) - -**Files:** -- Modify: `lib/cuckoo/common/integrations/file_extra_info.py` (`_get_extractor_pool`/`generic_file_extractors`, ~lines 436–520) -- Test: `tests/test_extractor_pool_teardown.py` - -- [ ] **Step 1: Write the failing test** - -```python -# tests/test_extractor_pool_teardown.py -import os -import time -import multiprocessing as mp - - -def _busy(_): - time.sleep(60) - - -def test_extractor_pool_is_swept_by_killpg(tmp_path): - """A child that creates the extractor pool and is killpg'd must leave no survivors.""" - marker = str(tmp_path / "epool_child") - - def child(): - os.setsid() - open(marker, "w").close() - import pebble - ctx = mp.get_context("forkserver") # or the strategy chosen in Task 9 - pool = pebble.ProcessPool(max_workers=2, context=ctx) - for _ in range(2): - pool.schedule(_busy, args=(1,), timeout=60) - time.sleep(60) - - pid = os.fork() - if pid == 0: - try: - child() - finally: - os._exit(0) - while not os.path.exists(marker): - time.sleep(0.05) - time.sleep(1) - os.killpg(pid, 15) - time.sleep(2) - import subprocess - out = subprocess.run(["pgrep", "-g", str(pid)], capture_output=True, text=True) - assert out.stdout.strip() == "", "extractor sub-pool survived killpg" -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `poetry run pytest tests/test_extractor_pool_teardown.py -v` -Expected: PASS or FAIL depending on context choice; if FAIL (survivors), the chosen context/teardown is wrong — adjust per Task 9 findings. - -- [ ] **Step 3: Implement teardown + per-call lifecycle** - -In `generic_file_extractors`, replace the process-wide cached `_EXTRACTOR_POOL` with a pool created and torn down per call (under prefork the child is ephemeral, so per-task lifetime is correct), using the Task-9 context, and `pool.close(); pool.join()` in a `finally`. Keep the per-future timeouts. - -```python -def generic_file_extractors(file, destination_folder, data_dictionary, options, results, duplicated, tests=False): - ... - import multiprocessing - ctx = multiprocessing.get_context("forkserver") # per Task 9 decision - pool = pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) - try: - ... # schedule extractors, collect futures, gather results (unchanged logic) - finally: - pool.close() - pool.join() -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `poetry run pytest tests/test_extractor_pool_teardown.py -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add lib/cuckoo/common/integrations/file_extra_info.py tests/test_extractor_pool_teardown.py -git commit -m "fix(extractors): per-call pool with explicit teardown, swept by process-group kill" -``` - ---- - -### Task 11: Docs + A/B systemd drop-in + stopgap retirement note - -**Files:** -- Create: `docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md` -- Modify: `docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md` (mark Phase status) - -- [ ] **Step 1: Write the A/B runbook** - -Document: how to flip engines (systemd drop-in `ExecStart ... --engine prefork`), the metrics to compare (throughput tasks/min, wedge incidents, peak RSS, orphan count after induced timeouts via `pgrep`), how to roll back (remove drop-in line), and the condition to retire the `-mc 0` stopgap (once `prefork` is the default). **State explicitly that the A/B runs against the production PostgreSQL database — both engines share the same module-level `Database()` (configured `postgresql://...`); the `sqlite://` `db` fixture in `tests/conftest.py` is a unit-test harness only and has no production path.** - -- [ ] **Step 2: Update spec status + commit** - -```bash -git add docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md -git commit -m "docs(processor): A/B runbook + mark redesign implementation status" -``` - ---- - -## Self-review notes (author) - -- **Spec coverage:** pluggable seam (T1–T4), PebbleEngine control (T4), Prefork supervisor incl. fork-per-task/os._exit/killpg/single-threaded invariant/status (T5–T8), warm-base inheritance (relies on `main()` init before `run()`; T8 smoke-test), extractor sub-pool §4.2(d) (T9–T10), A/B + metrics + heartbeat (T6 heartbeat, T11 runbook). Memory model is design-level (no code). -- **Deferred-by-design:** §4.2(d) spawn context is resolved by the T9 spike before T10 codes it — intentional, not a placeholder. -- **Type consistency:** `ProcessingEngine(task_fn, worker_init, source, parallel, timeout)` used uniformly; `task_fn(task)`, `source.fetch(limit, exclude_ids)`, `source.mark_failed(task_id)` consistent across T1/T4/T5–T8. -- **Risk:** the live `/opt/CAPEv2` tree has many dirty files — every commit step lists explicit paths; never `git add -A`. diff --git a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md b/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md deleted file mode 100644 index f49700f544a..00000000000 --- a/docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md +++ /dev/null @@ -1,249 +0,0 @@ -# Processor Concurrency Redesign — Pluggable Engines (Pebble + Prefork) - -**Date:** 2026-05-27 -**Component:** `utils/process.py` (CAPE report processor, systemd `cape-processor.service`) -**Branch:** `feature/processor-prefork-engine` (off `feature/guac-auth-evtx-snapshots`) -**Status:** Design — awaiting review before implementation planning - -**Status (2026-05-28):** implementation complete on branch `feature/processor-prefork-engine` (commits `47576a166` through `beb829e67`); A/B runbook at `docs/superpowers/notes/2026-05-27-processor-engine-ab-runbook.md`. - ---- - -## 1. Background & Problem - -`utils/process.py auto` continuously pulls `TASK_COMPLETED` analyses from the DB and runs -`process()` on each (processing modules → signatures → reporting), `-p20` in parallel, -with a `-pt 900` per-task timeout. It uses a `pebble.ProcessPool` (default **fork** start -method, `maxtasksperchild=7`). Each worker also lazily builds a **nested** -`pebble.ProcessPool` (`_EXTRACTOR_POOL`, 6 procs) for file extractors. - -On 2026-05-27 the processor wedged: all 20 worker slots stuck, scheduler parked in its -"pool full" sleep, log silent for hours, and **zero `Processing timeout` lines ever** -despite `-pt 900`. Diagnosis (py-spy + strace + log forensics) found **two independent -deadlocks**, both rooted in forking a process that owns threads and child processes: - -1. **Startup thread-soup fork hazard.** `lib/cuckoo/common/cleaners_utils.py` created - `resolver_pool = ThreadPool(50)` at *import* time; `process.py` imports that module, so - the parent held 50+ threads when pebble forked workers. Forked children inherited locks - held by now-dead threads. *(Mitigated: `resolver_pool` made lazy — main thread count - 73 → 20.)* - -2. **Worker-recycle exit deadlock (primary).** pebble's `worker_process` *returns normally* - when it hits `max_tasks`, so the worker runs `multiprocessing.util._exit_function`, which - `join()`s the worker's child processes — the nested `_EXTRACTOR_POOL` workers, which never - terminate → hang forever. Every worker deadlocks on its 7th-task recycle (observed: - exactly `140 completed ÷ 20 workers = 7`). *(Stopgap: `-mc 0` disables recycling so the - exit path is never reached; tradeoff is no per-worker memory reclaim.)* - -3. **Timeout enforcement gap.** pebble only times out *acknowledged* tasks - (`TaskManager.timeout()` requires `task.started`). A worker that hangs before/while - starting is immortal, and even when the timeout fires it kills one PID, not the worker's - descendant tree (extractors, external tools orphan to init). - -These are properties of the **fork + long-lived pool + pool-inside-a-worker** model, not -one-off bugs. This redesign replaces that model while keeping the old one runnable for A/B. - -## 2. Goals / Non-Goals - -**Goals** -- A processing model where a single hung/crashy task or extractor can never wedge the pool. -- A per-task timeout that **always** fires and fully cleans up the task's entire process tree. -- No fork-from-multithreaded-process hazard; no orphaned grandchildren; no immortal tasks. -- Keep the existing (pebble) model selectable so the new model can be A/B-tested against it - in production with a flag flip and measurable comparison. -- Per-task init cost stays near zero (preserve the warm-state benefit of long-lived workers). - -**Non-Goals** -- Changing what `process()` does (the processing/signature/reporting pipeline is unchanged). -- Changing the DB task lifecycle/statuses. -- Reworking `cape.service` or other CAPE components. - -## 3. Workload facts (measured, this deployment) -- Task duration: p50 22s, p90 52s, p99 107s, **max 129s** — none near the 900s limit. -- Throughput: a few tasks/min, bursty; ~160 tasks over a working day. -- Host: 16 CPU, 125 GB RAM; worker RSS ~1.8 GB. Memory is not the binding constraint. -- Implication: tasks are short and heavyweight; warm workers exist only to amortize - cold init (YARA compile ~3s + importing all processing/signature/reporting modules). - -## 4. Architecture — pluggable engines - -A clean seam splits "pull & track work" from "isolate & run one task." Both engines share -everything except worker isolation. - -**Shared (unchanged):** -- `process(task, report=True, auto=True, ...)` — the per-task work. -- DB poll for `TASK_COMPLETED`, concurrency accounting (`parallel`), status update helpers, - config (`-pt`, `-p`). - -**Selection:** new CLI flag `--engine {pebble,prefork}` (default `pebble` until prefork wins -the A/B; then flip the default). systemd `ExecStart` / drop-in carries the flag. Each engine -tags log lines with its name and emits per-task timing so A/B is measurable. - -``` - ┌───────────────────────────┐ - │ autoprocess(engine=...) │ DB poll, concurrency, status - └─────────────┬─────────────┘ - ┌────────────────┴────────────────┐ - ┌─────▼──────┐ ┌──────▼───────┐ - │ PebbleEngine│ (control/baseline)│ PreforkEngine│ (target) - └────────────┘ └──────────────┘ -``` - -### 4.1 `PebbleEngine` (preserved baseline) -Today's implementation, kept intact as the A/B control. Default `max_tasks=0` (recycling is -what deadlocks it; with 0 the exit path is never hit). It is explicitly *not* the long-term -target — it remains for comparison and rollback. - -### 4.2 `PreforkEngine` (target — Approach A) - -A single-threaded **supervisor** owning the full lifecycle. No library-internal pool, no -channels, no background manager threads. - -**(a) Warm base (once):** supervisor calls `init_database()` + `init_modules()` + YARA -compile, then **stays single-threaded** — no MongoClient, no thread pools in the supervisor. -Invariant enforced by `assert threading.active_count() == 1` immediately before each fork. - -**(b) Scheduler loop (supervisor main thread):** -- Poll DB for up to `parallel - in_flight` `TASK_COMPLETED` tasks. -- For each: `fork()` a task child (inherits warm modules/YARA copy-on-write ≈ zero init). -- Track `in_flight = {pid: (task_id, start_monotonic, pgid)}`. -- Reap finished children non-blocking (`os.waitpid(-1, WNOHANG)` / `pidfd`); on reap, apply - status (see (e)). -- **Timeout:** for each in-flight child where `now - start > processing_timeout`, - `os.killpg(pgid, SIGTERM)`, then `SIGKILL` after a short grace; mark task - `FAILED_PROCESSING`. Timer starts at *launch* — no "started" bookkeeping to be blind to. -- Emit a periodic heartbeat (`in_flight=N oldest=Xs`) so a stall is never silent. - -**(c) Task child:** -- `os.setsid()` → own session/process group, so the supervisor's `killpg` sweeps the entire - subtree (extractor sub-pool, external tools like sigma/suricata). -- Post-fork re-init: dispose+recreate SQLAlchemy engine connection, reset log handlers - (reuse current `init_worker` logic), establish its own Mongo client lazily. -- Run `process(task, ...)` **exactly once**. -- Terminate with **`os._exit(code)`** — never `return`/`sys.exit`, so the multiprocessing - atexit `join` (the exact thing that deadlocked pebble) never runs. - -**Lifecycle decision — fork-per-task, no dispatch channel.** Each child handles exactly -one task. The supervisor only forks, reaps, and times-out; there is **no supervisor→child -dispatch channel**, which deliberately eliminates the inter-process channel whose deadlock -motivated this redesign. Recycling a child across N>1 tasks would *require* such a channel; -the measured lazy-reload cost (§4.4) is ~0.35 s/task (~1.6%), so amortizing it is not worth -that complexity/failure surface. `--tasks-per-child N` is therefore a **documented future -knob, not built now** (YAGNI until a measured need appears). Fork-per-task also gives maximal -leak containment (C-lib leaks reclaimed every task), which was the primary concern. - -**(d) Extractors — per-task process sub-pool (replaces `_EXTRACTOR_POOL`):** -- Created **inside** the task child, used for the file extractors, then **explicitly - `close()`+`join()`d** before the child exits. -- All sub-pool processes live in the task child's process group → swept by the supervisor's - `killpg` on timeout. The child's `os._exit()` is the backstop if teardown is imperfect. -- Each extractor keeps its existing per-tool subprocess timeout. -- **Open implementation detail (resolve with measurement, not guesswork):** how the sub-pool - is spawned. `forkserver` gives clean isolation but re-imports modules per task (cost on a - 22s task); `fork` from the *warm, still-single-threaded* task child inherits modules - cheaply but must be created before the child spawns any thread. Decision criterion: measure - per-task extractor-pool startup cost both ways on a representative task; pick the faster one - that preserves the os._exit + process-group-kill safety properties. - -**(e) Status (child sets, supervisor overrides):** -- The child runs `process()`, which performs the existing status writes (e.g. → reported). -- The supervisor overrides to `FAILED_PROCESSING` only on timeout (killpg) or abnormal child - exit (non-zero / signal). One proven status path; supervisor adds the failure cases. - -### 4.3 Why this removes all three hazards -- Fork-from-multithreaded: base is single-threaded (asserted) → safe forks. -- Immortal tasks: supervisor owns a launch-relative wall-clock timeout → always fires. -- Orphans / exit deadlock: `os._exit()` in the child (no atexit join) + `killpg` of the - process group → complete cleanup, nothing to hang on. - -### 4.4 Memory model & shared state (audited 2026-05-27) - -The heavy load-once structures were audited for post-load mutation. All are **read-only -after load** or per-worker caches never shared across workers: - -| Structure | Loaded at | Post-load writes | Verdict | -|---|---|---|---| -| `File.yara_rules` (compiled YARA) | `init_yara()` (idempotent) | none — read at scan (`objects.py:593`) | read-only | -| `_MAXMINDDB_CLIENT` (geoip) | module-level, mtime-guarded | reload only if DB file changes | read-only (mmap) | -| capa `RuleSet` | import-time `get_rules()` | none — read at match | read-only | -| `mitre`/pyattck (`BASE_OBJECTS`, `RELATIONSHIP_MAP`) | `Attck()` at `abstracts.py:71` import | none — `techniques` recomputes views; caches nothing | read-only | -| `DECRYPTORS` (vbadeobf) | import-time decorator registry | none — read via `.get` | read-only | -| `_CLAMAV_CACHE` | during processing | per-worker, self-`clear()`s (`clamav.py:113`) | per-worker, not shared | - -**Nothing requires a live mutable instance shared across concurrent workers.** Decisive -proof: today's pebble model already shares these *only* via COW-fork (parent loads → workers -inherit, diverging on write). If anything needed cross-worker mutation it would already be -broken in production. - -Consequence for Approach A: the "load once, share" model is **preserved identically** — the -single-threaded warm base loads them where `init_modules`/import does today; per-task children -inherit by COW. Per-task `fork()` copies page tables, not the GBs of data. Short-lived -children also retain COW sharing better than 7-task workers, which gradually un-share heavy -pages via CPython refcount writes + cyclic GC. - -The leak concern is *better* served by A: a fresh process per task reclaims C-lib leaks every -task (recycle-every-task), strictly stronger than `maxtasksperchild=7`, without the -`_exit_function` deadlock. - -**Known per-task recompute cost:** pyattck `techniques` recomputes on each access (observed -burning CPU in `_get_relationship_objects`) — same in both models. Mitigation if material: -pre-compute/cache it in the warm base so children inherit the result. Treated as an -A/B-measured optimization, not a correctness item. - -**Two load categories (the audit's real taxonomy):** -- *Category 1 — loaded at init/import:* YARA incl. yara-forge (`custom/yara/binaries/yara-rules-extended.yar`, - 17 MB, compiled by `init_yara()`), MITRE/pyattck (45 MB, `abstracts.py` import), capa (5.6 MB). - The warm base loads these **once** before forking → COW-shared to all children. Strictly - better than today, where `init_worker` compiles YARA **per worker** (1× vs 20×). -- *Category 2 — lazy, loaded on first use:* `_LOLD_CACHE` (loldrivers, **29 MB JSON**), - `_TOOLS_CACHE` (security_tools), etc. Reloaded per task under fork-per-task. - -**Measured Category-2 cost (2026-05-27, isolated fresh process):** -| Loader | time | RSS | -|---|---|---| -| `_load_loldrivers()` (29 MB JSON → 619 entries) | **0.35 s** | +41 MB | -| `_load_tools()` (security_tools) | ~0 s | +0 MB | -| maliciousmacrobot (ML model) | **disabled** | — | -| sigma rules (~28 MB) | n/a — loaded inside a shelled-out subprocess, process-group-swept, own timeout | - -So per-task lazy reload ≈ **0.35 s (~1.6% of a p50 22 s task)**; 41 MB × 20 children ≈ 0.8 GB -of 125 GB. Conclusion: **no pre-warm registry needed** — the cost it would save is negligible -and an exhaustive registry is the fragile thing we want to avoid (this audit already missed -loldrivers once). Fork-per-task absorbs it. - -## 5. A/B testing & metrics -- Run identical workload under each engine (flag flip + restart). Compare: - - throughput (tasks/min), per-task wall-clock (p50/p90/p99), - - wedge incidents (should be 0 for prefork), peak/sustained RSS, - - orphaned-process count after induced timeouts. -- Engine name + per-task timing logged for both so comparison is from the same source. - -## 6. Risks & mitigations -| Risk | Mitigation | -|---|---| -| Supervisor accidentally goes multithreaded before fork (Mongo/thread pool import side effect) | `assert active_count()==1` pre-fork; keep Mongo/cleaners out of supervisor; test it. | -| `init_modules()` import side effects differ under the supervisor vs `main()` | Supervisor performs the same init `main()` did; integration test runs a real task end-to-end. | -| Per-task extractor sub-pool startup cost erodes throughput | Measure both spawn strategies (§4.2d); choose empirically; this is the A/B's job to catch. | -| Post-fork SQLAlchemy/Mongo/logging state | Reuse the proven `init_worker` re-init; child establishes its own connections. | -| `-mc 0` stopgap on PebbleEngine grows memory until prefork ships | Monitor RSS; add a periodic clean systemd restart (`RuntimeMaxSec`) if it climbs. | - -## 7. Testing strategy -- **Timeout:** task that sleeps > `-pt` → asserted `killpg`'d at limit, task `FAILED_PROCESSING`, - no orphan processes remain, pool keeps running. -- **Crash:** task that raises / segfaults → supervisor detects abnormal exit, marks failed, - other tasks unaffected. -- **External-tool cleanup:** task whose extractor spawns a long subprocess → on timeout the - subprocess is killed (process-group sweep), zero orphans (PPID 1 check). -- **Concurrency:** N>parallel queued → max in-flight never exceeds `parallel`. -- **Fork-safety invariant:** assert supervisor single-threaded before each fork. -- **A/B parity:** same corpus through both engines → identical task outputs/statuses. - -## 8. Rollout -1. Land `PreforkEngine` behind `--engine prefork`, default remains `pebble`. -2. A/B in production via flag; collect §5 metrics. -3. Flip default to `prefork` once it wins; keep `pebble` selectable for one release. -4. Remove the `-mc 0` stopgap drop-in once `prefork` is the default. - -## 9. Out of scope / follow-ups -- Committing the already-applied `resolver_pool` lazy fix on the mainline. -- Broader processing-pipeline changes inside `process()`. diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 5e1d7b75f42..0e95c476491 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -439,8 +439,10 @@ def _extracted_files_metadata( def _new_extractor_pool(): - """Create a fresh fork-context pebble pool (spike decision: fork from the - warm child — see docs/superpowers/notes/2026-05-27-extractor-subpool-spike.md).""" + """Create a fresh fork-context pebble pool. Fork (not forkserver/spawn) is + deliberate: the task child is single-threaded when the pool is created, so + fork is deadlock-safe here, ~10ms/pool faster than forkserver, and leaves no + orphans (workers are swept by the supervisor's process-group kill).""" ctx = multiprocessing.get_context("fork") return pebble.ProcessPool(max_workers=int(integration_conf.general.max_workers), context=ctx) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 91eb803033e..dc0823eb9a5 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -1,7 +1,7 @@ """Single-threaded prefork supervisor: forks one child per task, each child runs exactly one task then os._exit(); the supervisor reaps children and enforces a launch-relative wall-clock timeout via process-group kill. No supervisor<->child -channel exists. See docs/superpowers/specs/2026-05-27-processor-concurrency-redesign-design.md.""" +channel exists.""" import logging import os import signal From 1386c080e1f6bd6560cdbd82ce7e5891cb5278da Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 17:50:35 -0500 Subject: [PATCH 35/53] rooter: rebind ip-rule cleanup is table-agnostic + reject fail-table collision at startup - nexthop_enable deletes the VM's prior rule by (from,priority) regardless of table, so a rebind to a different gateway cannot leave a stale old-table rule that wins by priority. - load_nexthop_profiles rejects fail_closed when NEXTHOP_FAIL_TABLE collides with a VPN/dirty-line table. --- lib/cuckoo/core/startup.py | 8 +++++++ tests/test_nexthop_selection.py | 39 +++++++++++++++++++++++++++++++++ tests/test_rooter_nexthop.py | 2 +- utils/rooter.py | 6 ++++- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lib/cuckoo/core/startup.py b/lib/cuckoo/core/startup.py index 4ca6725ff34..ad8667bb8b6 100644 --- a/lib/cuckoo/core/startup.py +++ b/lib/cuckoo/core/startup.py @@ -121,6 +121,14 @@ def load_nexthop_profiles(routing_cfg): # preconfigured rt_table must not falsely reject a [gwX] reusing that id and fail to start (codex P2). if dirty_rt is not None and str(dirty_rt) and dirty_internet != "none": owned_tables.setdefault(str(dirty_rt), "the [routing] dirty-line table") + # The fail-closed blackhole is installed into NEXTHOP_FAIL_TABLE; if a VPN or the (enabled) + # dirty-line already owns that table, the blackhole would overwrite ITS default route and silently + # drop its traffic. Reject at startup -- only relevant when fail_closed is on (codex P2). + if routing_cfg.nexthop.fail_closed and str(NEXTHOP_FAIL_TABLE) in owned_tables: + raise CuckooStartupError( + f"[nexthop] fail_closed uses table '{NEXTHOP_FAIL_TABLE}' which collides with " + f"{owned_tables[str(NEXTHOP_FAIL_TABLE)]}; the blackhole would overwrite its default route" + ) # Pass 1: parse + validate + register profiles (no rooter side effects yet). profiles = [] claimed_tables = {} # rt_table -> gateway id, to reject duplicates (each [gwX] needs its own) diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py index 894a1b5a655..6bb09fb7c01 100644 --- a/tests/test_nexthop_selection.py +++ b/tests/test_nexthop_selection.py @@ -532,3 +532,42 @@ def _fake_rooter(cmd, *a): assert core_rooter._gw_live(_Profile("gw1", "ens-up", "201", 0)) is True assert core_rooter._gw_live(_Profile("gw2", "ens-down", "202", 0)) is False assert "nic_up" in calls and "nic_available" not in calls + + +def test_fail_closed_table_collides_with_vpn_raises(monkeypatch): + # codex P2: fail_closed installs a blackhole into NEXTHOP_FAIL_TABLE; if a VPN already uses that + # table, the blackhole would overwrite the VPN's default route and drop its traffic. Reject at load. + import lib.cuckoo.core.startup as startup + from lib.cuckoo.common.exceptions import CuckooStartupError + + class _FailTableVpn(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="roundrobin", + fail_closed=True, vm_net="192.168.100.0/24") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {"vpn0": type("V", (), {"rt_table": startup.NEXTHOP_FAIL_TABLE})()}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + with pytest.raises(CuckooStartupError): + startup.load_nexthop_profiles(_FailTableVpn()) + + +def test_fail_closed_table_collision_ignored_when_fail_closed_off(monkeypatch): + # the collision only matters when fail_closed arms the blackhole; with fail_closed=no a VPN on + # table 250 is fine (we never write that table), so startup must NOT reject it. + import lib.cuckoo.core.startup as startup + + class _FailTableVpnOff(_FakeRouting): + def __init__(self): + super().__init__() + self.nexthop = _DictSection(enabled=True, gateways="gw1", default_policy="roundrobin", + fail_closed=False, vm_net="192.168.100.0/24") + self.gw1 = _DictSection(interface="ens6", next_hop="onlink", rt_table=201) + + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {"vpn0": type("V", (), {"rt_table": startup.NEXTHOP_FAIL_TABLE})()}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) + startup.load_nexthop_profiles(_FailTableVpnOff()) # must NOT raise + assert "gw1" in startup.gateways diff --git a/tests/test_rooter_nexthop.py b/tests/test_rooter_nexthop.py index ccbf47111d4..df47ce928e9 100644 --- a/tests/test_rooter_nexthop.py +++ b/tests/test_rooter_nexthop.py @@ -82,7 +82,7 @@ def test_nexthop_enable_argv(rec): # iproute2 + conntrack go through run(); nat/filter through run_iptables() assert rec["run"] == [ ("conntrack", "-D", "-s", "192.168.100.42"), # pre-bind flush - ("ip", "rule", "del", "from", "192.168.100.42", "lookup", "201", "priority", "10042"), # idempotent pre-clean + ("ip", "rule", "del", "from", "192.168.100.42", "priority", "10042"), # table-agnostic pre-clean (drops a stale old-table rebind rule too) ("ip", "rule", "add", "from", "192.168.100.42", "lookup", "201", "priority", "10042"), ] # Each iptables rule is delete-until-gone (idempotent; Copilot) then added once. The forward diff --git a/utils/rooter.py b/utils/rooter.py index fcfe2d26a84..87eee52dad6 100644 --- a/utils/rooter.py +++ b/utils/rooter.py @@ -723,7 +723,11 @@ def nexthop_enable(vm_ip, ingress_if, egress_if, rt_table, priority): a retried bind (or route_network called twice) must NOT stack rules, so the ip rule and both iptables rules are delete-then-add'd; the iptables deletes loop until gone.""" run("conntrack", "-D", "-s", vm_ip) # drop stale flows so a recycled IP starts clean (best-effort) - run(settings.ip, "rule", "del", "from", vm_ip, "lookup", rt_table, "priority", priority) # idempotent + # Delete any existing rule for this VM at its (deterministic per-IP) priority -- TABLE-AGNOSTIC, + # so a rebind to a DIFFERENT gateway after a crashed/partial unroute removes the stale + # `from priority

lookup ` rule too. A `lookup ` del would miss + # it, leaving it to win by priority and route the task via the previous exit (codex). Then add fresh. + run(settings.ip, "rule", "del", "from", vm_ip, "priority", priority) run(settings.ip, "rule", "add", "from", vm_ip, "lookup", rt_table, "priority", priority) # SNAT the VM out its gateway. Delete-until-gone first so a retry can't stack duplicate NAT rules # that nexthop_disable's delete would then leave behind (Copilot). From 6d069ea7c1e8a64ef25a31c2b06cb401844181e6 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 23:04:56 +0000 Subject: [PATCH 36/53] fix(prefork): don't reap timed-out children before SIGKILL escalation; killpg fallback; idle backoff - _reap now iterates per-pid and holds a timed-out child until its process group has been escalated to SIGKILL (kill_deadline cleared). Reaping it earlier popped it from _inflight so _escalate_kills never SIGKILLed the group, orphaning any surviving grandchildren. - _enforce_timeouts falls back to os.kill(pid) when killpg raises ProcessLookupError (group not yet created / already gone), and only skips setting kill_deadline when the process is truly gone, so escalation still runs. - run() backs off to idle_poll_interval (5s) when fully idle instead of a 0.2s hot loop hammering the DB ~5x/sec. --- lib/cuckoo/core/processing_engine/prefork.py | 46 ++++++++++--- tests/test_prefork_reaping.py | 72 ++++++++++++++++++++ 2 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 tests/test_prefork_reaping.py diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index dc0823eb9a5..30a2aaa1cf4 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -29,12 +29,14 @@ def __init__(self, task_id, pid, start, pgid): class PreforkEngine(ProcessingEngine): def __init__(self, task_fn, worker_init, source, parallel, timeout, - heartbeat_interval=30, term_grace=5, max_count=0, poll_interval=0.2): + heartbeat_interval=30, term_grace=5, max_count=0, poll_interval=0.2, + idle_poll_interval=5.0): super().__init__(task_fn, worker_init, source, parallel, timeout) self.heartbeat_interval = heartbeat_interval self.term_grace = term_grace self.max_count = max_count self.poll_interval = poll_interval + self.idle_poll_interval = idle_poll_interval self._inflight = {} # pid -> _Child def _assert_single_threaded(self): @@ -54,6 +56,14 @@ def _assert_single_threaded(self): def _inflight_task_ids(self): return {c.task_id for c in self._inflight.values()} + def _sleep_interval(self, launched): + # Fully idle (no in-flight work and nothing launched this tick) -> back off + # to avoid hammering the DB ~5x/sec. Otherwise poll tightly so children are + # reaped and timeouts enforced promptly. + if not self._inflight and not launched: + return self.idle_poll_interval + return self.poll_interval + def _heartbeat(self): if self._inflight: oldest = max(time.monotonic() - c.start for c in self._inflight.values()) @@ -95,16 +105,24 @@ def _launch(self, task): log.debug("prefork: launched task %d as pid %d", task.id, pid) def _reap(self): - while True: + # Per-pid reap (not waitpid(-1)) so we can hold a timed-out child until its + # process group has been SIGKILL-escalated. Reaping it earlier would pop it + # from _inflight, so _escalate_kills would never SIGKILL the group and any + # surviving grandchildren would orphan. + for pid in list(self._inflight.keys()): + child = self._inflight.get(pid) + if child is None: + continue + if child.timed_out and child.kill_deadline is not None: + continue # SIGTERM sent, escalation pending — don't reap yet try: - pid, status = os.waitpid(-1, os.WNOHANG) + reaped_pid, status = os.waitpid(pid, os.WNOHANG) except ChildProcessError: - return - if pid == 0: - return - child = self._inflight.pop(pid, None) - if child is None: + self._inflight.pop(pid, None) continue + if reaped_pid == 0: + continue # still running + self._inflight.pop(pid, None) if child.timed_out: continue # already marked failed during timeout enforcement ok = os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 @@ -133,7 +151,13 @@ def _enforce_timeouts(self): try: os.killpg(child.pgid, signal.SIGTERM) except ProcessLookupError: - continue + # Group may not exist yet (child hasn't reached os.setsid()) or is + # already gone. Fall back to signaling the pid directly so the child + # is still killed and escalation still runs. + try: + os.kill(child.pid, signal.SIGTERM) + except ProcessLookupError: + continue # truly gone; _reap will collect it except OSError as e: log.warning("prefork: killpg(SIGTERM) failed for task %d (pid %d): %s", child.task_id, child.pid, e) @@ -171,13 +195,15 @@ def run(self): return free = self.parallel - len(self._inflight) launchable = free if not self.max_count else min(free, self.max_count - count) + launched = 0 if launchable > 0: tasks = self.source.fetch(limit=launchable, exclude_ids=self._inflight_task_ids()) for task in tasks[:launchable]: self._launch(task) count += 1 + launched += 1 now = time.monotonic() if now - last_hb >= self.heartbeat_interval: self._heartbeat() last_hb = now - time.sleep(self.poll_interval) + time.sleep(self._sleep_interval(launched)) diff --git a/tests/test_prefork_reaping.py b/tests/test_prefork_reaping.py new file mode 100644 index 00000000000..18cc1da3879 --- /dev/null +++ b/tests/test_prefork_reaping.py @@ -0,0 +1,72 @@ +"""Timeout/reap state-machine correctness for the prefork supervisor, exercised +with monkeypatched os calls (no real forks): a timed-out child must not be reaped +before its process group is SIGKILL-escalated (else grandchildren orphan), and a +killpg that hits a not-yet-setsid child must fall back to signaling the pid.""" +import os +import signal + +from lib.cuckoo.core.processing_engine.prefork import PreforkEngine, _Child + + +class _FakeSource: + def __init__(self): + self.failed = [] + + def mark_failed(self, task_id): + self.failed.append(task_id) + + +def _engine(): + return PreforkEngine( + task_fn=lambda t: None, worker_init=lambda: None, + source=_FakeSource(), parallel=2, timeout=10, term_grace=5, + ) + + +def test_reap_defers_timed_out_child_until_escalated(monkeypatch): + eng = _engine() + child = _Child(task_id=1, pid=111, start=0.0, pgid=111) + child.timed_out = True + child.kill_deadline = 1e9 # SIGTERM sent, escalation not yet done + eng._inflight[111] = child + + # waitpid WOULD report it exited; reap must not collect it yet. + monkeypatch.setattr(os, "waitpid", lambda pid, flags: (pid, 0)) + eng._reap() + assert 111 in eng._inflight, "timed-out child reaped before SIGKILL escalation -> grandchildren orphan" + + # once escalation clears kill_deadline, reap collects it + child.kill_deadline = None + eng._reap() + assert 111 not in eng._inflight + + +def test_idle_supervisor_backs_off_polling(): + """Fully idle (nothing in-flight, nothing launched) -> long sleep, not a 0.2s + hot loop hammering the DB ~5x/sec.""" + eng = PreforkEngine( + task_fn=lambda t: None, worker_init=lambda: None, source=_FakeSource(), + parallel=2, timeout=10, poll_interval=0.2, idle_poll_interval=5.0, + ) + assert eng._sleep_interval(launched=0) == 5.0 # idle -> back off + + # Actively working: launched something OR has in-flight -> tight poll. + assert eng._sleep_interval(launched=1) == 0.2 + eng._inflight[1] = _Child(task_id=1, pid=1, start=0.0, pgid=1) + assert eng._sleep_interval(launched=0) == 0.2 # in-flight -> reap promptly + + +def test_enforce_timeouts_falls_back_to_kill_when_killpg_missing(monkeypatch): + eng = _engine() + child = _Child(task_id=2, pid=222, start=0.0, pgid=222) # start=0 => far past timeout + eng._inflight[222] = child + + killed = [] + monkeypatch.setattr(os, "killpg", lambda pgid, sig: (_ for _ in ()).throw(ProcessLookupError())) + monkeypatch.setattr(os, "kill", lambda pid, sig: killed.append((pid, sig))) + + eng._enforce_timeouts() + + assert child.timed_out is True + assert (222, signal.SIGTERM) in killed, "must fall back to os.kill(pid) when killpg raises ProcessLookupError" + assert child.kill_deadline is not None, "kill_deadline must be set so SIGKILL escalation still runs" From a1901ad5b1eec6a53558a151d6c4336288db57da Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 23:04:56 +0000 Subject: [PATCH 37/53] fix(processor/source): widen DB fetch limit by exclude_ids to avoid worker starvation Querying only `limit` rows then filtering out in-flight ids could return fewer (or zero) tasks when the oldest tasks are all in-flight, idling free workers. Fetch `limit + len(exclude_ids)` and slice the filtered result back to limit. --- lib/cuckoo/core/processing_engine/source.py | 14 ++++---- tests/test_processing_engine_source.py | 39 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/lib/cuckoo/core/processing_engine/source.py b/lib/cuckoo/core/processing_engine/source.py index 9210a60c890..5fead66c7d2 100644 --- a/lib/cuckoo/core/processing_engine/source.py +++ b/lib/cuckoo/core/processing_engine/source.py @@ -17,16 +17,18 @@ def fetch(self, limit, exclude_ids): """Return up to `limit` tasks needing processing, excluding `exclude_ids` (in-flight). Tasks are expunged so they are safe to use after the txn. - Note: `limit` is applied at the DB level before `exclude_ids` filtering, - so the returned list may contain fewer than `limit` tasks even when more - eligible tasks exist in the DB (those in `exclude_ids` are still counted - against `limit`).""" + The DB query fetches `limit + len(exclude_ids)` rows so that, after + dropping the in-flight ids, up to `limit` eligible tasks remain. Querying + only `limit` would starve workers: if the oldest tasks are all in-flight + they'd be filtered out, returning fewer (or zero) tasks while other + eligible work waits. The result is sliced back to `limit`.""" if limit <= 0: return [] + db_limit = limit + len(exclude_ids) with self.db.session.begin(): - tasks = self.db.list_tasks(status=self._status, limit=limit, order_by=Task.completed_on.asc()) + tasks = self.db.list_tasks(status=self._status, limit=db_limit, order_by=Task.completed_on.asc()) self.db.session.expunge_all() - return [t for t in tasks if t.id not in exclude_ids] + return [t for t in tasks if t.id not in exclude_ids][:limit] def mark_failed(self, task_id): # Always writes TASK_FAILED_PROCESSING regardless of which status this diff --git a/tests/test_processing_engine_source.py b/tests/test_processing_engine_source.py index bca34f41eea..52869188486 100644 --- a/tests/test_processing_engine_source.py +++ b/tests/test_processing_engine_source.py @@ -1,7 +1,46 @@ +import contextlib + from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_FAILED_PROCESSING from lib.cuckoo.core.processing_engine.source import TaskSource +class _FakeTask: + def __init__(self, tid): + self.id = tid + + +class _FakeSession: + def begin(self): + return contextlib.nullcontext() + + def expunge_all(self): + pass + + +class _FakeDB: + def __init__(self, tasks): + self._tasks = tasks + self.limit_seen = None + self.session = _FakeSession() + + def list_tasks(self, status, limit, order_by): + self.limit_seen = limit + return self._tasks[:limit] + + +def test_fetch_widens_db_limit_to_avoid_starvation(): + """DB limit must be widened by len(exclude_ids): if the oldest tasks are all + in-flight, querying only `limit` returns nothing eligible even though other + work waits. Result is sliced back to `limit`.""" + db = _FakeDB([_FakeTask(i) for i in range(1, 6)]) # ids 1..5; 1..3 in-flight + src = TaskSource(db) + + got = src.fetch(limit=2, exclude_ids={1, 2, 3}) + + assert db.limit_seen == 5, "DB query must fetch limit + len(exclude_ids) rows" + assert [t.id for t in got] == [4, 5], "eligible tasks returned, sliced to limit" + + def test_fetch_returns_completed_tasks_excluding_inflight(db, temp_pe32): with db.session.begin(): t1 = db.add_path(temp_pe32) From 735448024317d71eeab31e9fe5d0ae653db3e11f Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Thu, 9 Jul 2026 23:04:56 +0000 Subject: [PATCH 38/53] fix(extractors/dedup): bound post-stop pool join; close QR image fd - _teardown_extractor_pool now bounds the join AFTER stop() too: a grandchild wedged in uninterruptible (D-state) IO survives SIGKILL, so an unbounded reap join would still hang. Abandon after EXTRACTOR_POOL_STOP_JOIN_TIMEOUT. - _qr_decode_zxing opens the image via `with` so its fd is released promptly. --- .../common/integrations/file_extra_info.py | 12 +++++- modules/processing/deduplication.py | 4 +- tests/test_deduplication_qr.py | 34 ++++++++++++++++ tests/test_shared_extractor_pool.py | 40 ++++++++++++++++--- 4 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 tests/test_deduplication_qr.py diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 0e95c476491..95620ece080 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -436,6 +436,10 @@ def _extracted_files_metadata( # (the pebble task timeout only covers task execution, not teardown). Wait this # long for a graceful drain, then force-stop. EXTRACTOR_POOL_JOIN_TIMEOUT = 60 +# Shorter bound for the reap AFTER force-stop; stop() already SIGKILLed the workers, +# so this only waits for the OS to reap them. Kept finite so a D-state straggler +# can't wedge the worker even here. +EXTRACTOR_POOL_STOP_JOIN_TIMEOUT = 5 def _new_extractor_pool(): @@ -462,7 +466,13 @@ def _teardown_extractor_pool(pool): EXTRACTOR_POOL_JOIN_TIMEOUT, ) pool.stop() - pool.join() + # Bound the post-stop reap too: a grandchild wedged in uninterruptible + # (D-state) IO survives SIGKILL, so an unbounded join would still hang. + # Abandon after the bound — the process-group kill sweeps the rest. + try: + pool.join(timeout=EXTRACTOR_POOL_STOP_JOIN_TIMEOUT) + except Exception: + log.error("extractor pool did not reap after force-stop; abandoning (swept by process-group kill)") except Exception: log.debug("_teardown_extractor_pool: teardown raised", exc_info=True) diff --git a/modules/processing/deduplication.py b/modules/processing/deduplication.py index c75daf606b3..3c5ef6d1fe3 100644 --- a/modules/processing/deduplication.py +++ b/modules/processing/deduplication.py @@ -77,8 +77,8 @@ def reindex_screenshots(shots_path): def _qr_decode_zxing(image_path): """Decode QR codes using zxing-cpp. Supports multiple barcodes per image.""" try: - img = Image.open(image_path) - results = zxingcpp.read_barcodes(img) + with Image.open(image_path) as img: + results = zxingcpp.read_barcodes(img) urls = [] for result in results: text = result.text diff --git a/tests/test_deduplication_qr.py b/tests/test_deduplication_qr.py new file mode 100644 index 00000000000..ac06e46ad82 --- /dev/null +++ b/tests/test_deduplication_qr.py @@ -0,0 +1,34 @@ +"""QR decode must open the image as a context manager so its file descriptor is +released promptly (screenshots are decoded in a loop).""" +import modules.processing.deduplication as dedup + + +class _FakeImg: + def __init__(self): + self.exited = False + + def __enter__(self): + return self + + def __exit__(self, *a): + self.exited = True + return False + + +class _FakeResult: + text = "http://evil.example/x" + + +def test_qr_decode_zxing_closes_image(monkeypatch, tmp_path): + img = _FakeImg() + monkeypatch.setattr(dedup, "Image", type("I", (), {"open": staticmethod(lambda p: img)})) + monkeypatch.setattr( + dedup, "zxingcpp", + type("Z", (), {"read_barcodes": staticmethod(lambda i: [_FakeResult()])}), + raising=False, + ) + + urls = dedup._qr_decode_zxing(str(tmp_path / "shot.png")) + + assert img.exited is True, "Image must be used as a context manager so the fd is closed" + assert urls == ["http://evil.example/x"] diff --git a/tests/test_shared_extractor_pool.py b/tests/test_shared_extractor_pool.py index 60880bdcc2a..2d55d6cfb28 100644 --- a/tests/test_shared_extractor_pool.py +++ b/tests/test_shared_extractor_pool.py @@ -72,7 +72,7 @@ def test_shutdown_is_idempotent_when_no_pool(monkeypatch): class _HangingPool: - """join(timeout) times out (wedged grandchild); stop()+unbounded join reaps.""" + """Graceful join times out (wedged grandchild); after stop() the reap succeeds.""" def __init__(self): self.closed = False @@ -84,8 +84,9 @@ def close(self): def join(self, timeout=None): self.join_timeouts.append(timeout) - if timeout is not None: - raise TimeoutError("pool still draining") + if len(self.join_timeouts) == 1: + raise TimeoutError("pool still draining") # graceful join times out + # post-stop reap succeeds (stop() SIGKILLed the workers) def stop(self): self.stopped = True @@ -111,8 +112,8 @@ def test_teardown_force_stops_when_join_times_out(): fx._teardown_extractor_pool(pool) assert pool.closed is True assert pool.stopped is True, "a wedged pool must be force-stopped, not left to hang" - assert pool.join_timeouts[0] is not None, "first join must be bounded by a timeout" - assert pool.join_timeouts[-1] is None, "after stop() the final reap join is unbounded" + # Both the graceful join and the post-stop reap are bounded (no unbounded wait). + assert pool.join_timeouts == [fx.EXTRACTOR_POOL_JOIN_TIMEOUT, fx.EXTRACTOR_POOL_STOP_JOIN_TIMEOUT] def test_teardown_does_not_stop_when_join_succeeds(): @@ -126,6 +127,35 @@ def test_teardown_none_is_noop(): fx._teardown_extractor_pool(None) # must not raise +class _AlwaysHangingPool: + """Every join times out — even the post-stop reap (grandchild stuck in D-state). + Teardown must still return instead of blocking forever.""" + + def __init__(self): + self.closed = False + self.stopped = False + self.join_timeouts = [] + + def close(self): + self.closed = True + + def join(self, timeout=None): + self.join_timeouts.append(timeout) + raise TimeoutError("never drains") + + def stop(self): + self.stopped = True + + +def test_teardown_bounds_post_stop_join_and_returns(): + pool = _AlwaysHangingPool() + fx._teardown_extractor_pool(pool) # must return, not hang + assert pool.stopped is True + # both the initial join AND the post-stop reap must be bounded (no None/unbounded). + assert len(pool.join_timeouts) == 2 + assert all(t is not None for t in pool.join_timeouts) + + class _Task: id = 1 category = "file" From ea2e615a3a93e187d5477973bb23d50144d2b6bf Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 00:34:23 +0000 Subject: [PATCH 39/53] style: drop unused imports orphaned by the engine refactor; fix E401 in test The pool loop moved into the engine modules, leaving time/TimeoutError/ free_space_monitor/TASK_COMPLETED/TASK_FAILED_PROCESSING/Task unused in process.py (ruff F401). Keep the pebble dep guard with a noqa. Split a two-on-one-line import in test_prefork_engine.py (E401). --- tests/test_prefork_engine.py | 3 ++- utils/process.py | 10 ++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index 88ebbcf33b2..7e14dec2b64 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -107,7 +107,8 @@ def test_timeout_kills_process_group_no_orphans(db, temp_pe32): def task_fn(task): # spawn a grandchild that would outlive the worker, then hang - import subprocess, sys + import subprocess + import sys subprocess.Popen([sys.executable, "-c", "import time,os;open(%r,'w').close();time.sleep(120)" % marker]) time.sleep(120) diff --git a/utils/process.py b/utils/process.py index 93a012c7081..5f58ae7ed29 100644 --- a/utils/process.py +++ b/utils/process.py @@ -12,7 +12,6 @@ import resource import signal import sys -import time from contextlib import suppress log = logging.getLogger() @@ -28,26 +27,21 @@ sys.exit(1) try: - import pebble + import pebble # noqa: F401 # fail fast with a clear message if the dep is missing (engines import it) except ImportError: log.critical("Missed pebble dependency. Run: poetry install") sys.exit(1) sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), "..")) -from concurrent.futures import TimeoutError -from lib.cuckoo.common.cleaners_utils import free_space_monitor from lib.cuckoo.common.config import Config from lib.cuckoo.common.constants import CUCKOO_ROOT from lib.cuckoo.common.path_utils import path_delete, path_exists, path_mkdir from lib.cuckoo.common.utils import get_options, option_dict_enabled from lib.cuckoo.core.database import Database, init_database from lib.cuckoo.core.data.task import ( - TASK_COMPLETED, - TASK_FAILED_PROCESSING, TASK_FAILED_REPORTING, - TASK_REPORTED, - Task + TASK_REPORTED ) from lib.cuckoo.core.plugins import RunProcessing, RunReporting, RunSignatures from lib.cuckoo.core.startup import ConsoleHandler, check_linux_dist, init_modules From 43498934c11124342b27107458fd839f23b6bbb9 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 19:35:31 -0500 Subject: [PATCH 40/53] nexthop: only the scheduler applies rooter state; web/vpncheck never sweep init_routing() is called by cuckoo.py (scheduler), web settings (Django startup) and vpncheck. The nexthop stale-sweep flushed the per-task rule band on EVERY call, so a web restart mid-analysis tore down running tasks' egress. Gate all nexthop rooter mutations behind an opt-in apply flag; only the scheduler passes init_routing(apply_nexthop_state=True). --- cuckoo.py | 2 +- lib/cuckoo/core/startup.py | 29 +++++++++++++++++++++-------- tests/test_nexthop_selection.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/cuckoo.py b/cuckoo.py index fa458a6067d..64529915c76 100644 --- a/cuckoo.py +++ b/cuckoo.py @@ -84,7 +84,7 @@ def cuckoo_init(quiet=False, debug=False, artwork=False, test=False): with Database().session.begin(): init_tasks() init_rooter() - init_routing() + init_routing(apply_nexthop_state=True) # scheduler owns the nexthop sweep/build/arm (codex P2) check_tcpdump_permissions() # This is just a temporary hack, we need an actual test suite to integrate with Travis-CI. diff --git a/lib/cuckoo/core/startup.py b/lib/cuckoo/core/startup.py index ad8667bb8b6..e7f52c7b772 100644 --- a/lib/cuckoo/core/startup.py +++ b/lib/cuckoo/core/startup.py @@ -89,10 +89,13 @@ _RESERVED_RT_TABLES = ("local", "main", "default", "0", "253", "254", "255") -def load_nexthop_profiles(routing_cfg): - """Parse [nexthop]/[gwX] sections into the rooter.gateways global, sweep stale - policy-routing state, then arm fail-closed. No-op when [nexthop] is absent - or disabled (review M4 hasattr guard).""" +def load_nexthop_profiles(routing_cfg, apply_rooter_state=False): + """Parse [nexthop]/[gwX] sections into the rooter.gateways global and validate them. When + apply_rooter_state is True (ONLY the scheduler's init_routing passes this), ALSO sweep stale + policy-routing state and build the gateway tables + arm fail-closed. Non-owning callers (the + web/API process via web.settings, and vpncheck) leave it False: they must NOT issue the rooter + sweep, which flushes the 10000-10255 per-task band and would tear down the egress/fail-closed of + analyses currently running under the scheduler (codex P2). No-op when [nexthop] absent/disabled.""" if not hasattr(routing_cfg, "nexthop") or not routing_cfg.nexthop.enabled: return # [nexthop] is enabled: it MUST define gateways + vm_net (gemini #14 MEDIUM). CAPE's config @@ -210,6 +213,11 @@ def load_nexthop_profiles(routing_cfg): f"[nexthop] default_policy '{default_policy}' must be 'roundrobin', 'random', or a configured " f"gateway id ({', '.join(sorted(gateway_ids))})" ) + if not apply_rooter_state: + # Parsed + validated + gateways populated; skip ALL rooter mutations. Only the scheduler + # (cuckoo.py -> init_routing(apply_nexthop_state=True)) owns the sweep/build/arm -- doing it + # from the web/API startup would flush the per-task band and drop live analyses (codex P2). + return vm_net = str(routing_cfg.nexthop.vm_net) tables_csv = ",".join(p.rt_table for p in profiles) # Record sweep state for SIGTERM, then sweep any STALE state from a prior run @@ -720,8 +728,12 @@ def init_rooter(): -def init_routing(): - """Initialize and check whether the routing information is correct.""" +def init_routing(apply_nexthop_state=False): + """Initialize and check whether the routing information is correct. + + apply_nexthop_state: only the SCHEDULER (cuckoo.py) passes True, to sweep+build+arm the nexthop + rooter state. The web/API process and vpncheck call this to populate vpns/socks5s/gateways and + validate config, but must leave it False so they don't tear down live analyses (codex P2).""" # Check whether all VPNs exist if configured and make their configuration # available through the vpns variable. Also enable NAT on each interface. @@ -764,8 +776,9 @@ def init_routing(): rooter("flush_rttable", entry.rt_table) rooter("init_rttable", entry.rt_table, entry.interface) - # Load [gwX] next-hop egress profiles, arm fail-closed (no-op when [nexthop] absent/disabled). - load_nexthop_profiles(routing) + # Load [gwX] next-hop egress profiles; apply rooter state (sweep/build/arm) only for the scheduler + # (no-op when [nexthop] absent/disabled). + load_nexthop_profiles(routing, apply_rooter_state=apply_nexthop_state) # If we are storage and webgui only but using as default route one of the workers exitnodes if dist_conf.distributed.master_storage_only: diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py index 6bb09fb7c01..f7e6d6b4689 100644 --- a/tests/test_nexthop_selection.py +++ b/tests/test_nexthop_selection.py @@ -126,7 +126,7 @@ def test_gwx_loader_populates_and_coerces(monkeypatch): # as core_rooter.gateways unless replaced). Clear it in place so both refs see the reset. startup.gateways.clear() monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append((cmd, a)) or {}, raising=False) - startup.load_nexthop_profiles(_FakeRouting()) + startup.load_nexthop_profiles(_FakeRouting(), apply_rooter_state=True) # Check via startup.gateways (the dict the function mutated) assert "gw1" in startup.gateways assert startup.gateways["gw1"].rt_table == "201" # coerced to str @@ -376,7 +376,7 @@ def __init__(self): startup.gateways.clear() monkeypatch.setattr(startup, "vpns", {}, raising=False) monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {}, raising=False) - startup.load_nexthop_profiles(_NexthopNoFailClosed()) + startup.load_nexthop_profiles(_NexthopNoFailClosed(), apply_rooter_state=True) assert "nexthop_intra_exception_enable" in recorded assert "nexthop_fail_closed_enable" not in recorded @@ -571,3 +571,28 @@ def __init__(self): monkeypatch.setattr(startup, "rooter", lambda *a, **k: {}, raising=False) startup.load_nexthop_profiles(_FailTableVpnOff()) # must NOT raise assert "gw1" in startup.gateways + + +def test_web_startup_does_not_sweep_live_rules(monkeypatch): + # codex P2 (cross-process): the web/API process calls init_routing() -> load_nexthop_profiles with + # apply_rooter_state=False. It must parse+validate+populate gateways but issue NO rooter mutations, + # so a web/gunicorn restart cannot flush the scheduler's live per-task nexthop rules mid-analysis. + import lib.cuckoo.core.startup as startup + recorded = [] + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {}, raising=False) + startup.load_nexthop_profiles(_FakeRouting()) # default apply_rooter_state=False (web/vpncheck path) + assert "gw1" in startup.gateways # parsed + populated (harmless) + assert recorded == [] # but NO rooter mutations + + +def test_scheduler_startup_applies_rooter_state(monkeypatch): + # counterpart: the scheduler (apply_rooter_state=True) DOES sweep/build/arm. + import lib.cuckoo.core.startup as startup + recorded = [] + startup.gateways.clear() + monkeypatch.setattr(startup, "vpns", {}, raising=False) + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {}, raising=False) + startup.load_nexthop_profiles(_FakeRouting(), apply_rooter_state=True) + assert "nexthop_teardown" in recorded and "nexthop_init" in recorded From 15aef0d7ea11b7ea747f962c27b81e86ea7ba177 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Thu, 9 Jul 2026 19:59:31 -0500 Subject: [PATCH 41/53] nexthop: gate init_rooter state-reset to the scheduler (opt-in apply_state) init_rooter() is also called by web/web/settings.py before init_routing; its cleanup_rooter/ forward_drop/state_* reset CAPE-rooter iptables incl. live per-task nexthop rules. A web/gunicorn restart on a nexthop node therefore tore down running analyses. Gate the reset behind apply_state (default False); only the scheduler passes init_rooter(apply_state=True). Reachability still checked. --- cuckoo.py | 2 +- lib/cuckoo/core/startup.py | 17 ++++++++++--- tests/test_nexthop_selection.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/cuckoo.py b/cuckoo.py index 64529915c76..eedcbeafdf0 100644 --- a/cuckoo.py +++ b/cuckoo.py @@ -83,7 +83,7 @@ def cuckoo_init(quiet=False, debug=False, artwork=False, test=False): init_modules() with Database().session.begin(): init_tasks() - init_rooter() + init_rooter(apply_state=True) # scheduler owns the rooter state reset (codex P1) init_routing(apply_nexthop_state=True) # scheduler owns the nexthop sweep/build/arm (codex P2) check_tcpdump_permissions() diff --git a/lib/cuckoo/core/startup.py b/lib/cuckoo/core/startup.py index e7f52c7b772..c46c518b281 100644 --- a/lib/cuckoo/core/startup.py +++ b/lib/cuckoo/core/startup.py @@ -640,9 +640,14 @@ def check_snapshot_state(): conn.close() -def init_rooter(): - """If required, check whether the rooter is running and whether we can - connect to it.""" +def init_rooter(apply_state=False): + """If required, check whether the rooter is running and whether we can connect to it. + + apply_state: only the SCHEDULER (cuckoo.py) passes True, to RESET rooter state at its startup + (cleanup_rooter / cleanup_vrf / forward_drop / state_*). The web/API process and vpncheck verify + the rooter is reachable but must leave it False -- running cleanup_rooter cross-process would + remove the live per-task nexthop (and VPN) iptables rules of analyses owned by the scheduler + (codex P1). Reachability is still checked for all callers (fail-fast if the rooter is required).""" # The default configuration doesn't require the rooter to be ran. # A nexthop-only node still needs the rooter for forward_drop() + fail-closed arm. @@ -692,6 +697,12 @@ def init_rooter(): raise CuckooStartupError(f"Unknown rooter error: {e}") + if not apply_state: + # Reachability verified above; do NOT reset rooter state. cleanup_rooter/forward_drop/state_* + # would tear down the scheduler's live per-task nexthop (and VPN) rules on a web/API/gunicorn + # restart (codex P1). Only the scheduler (init_rooter(apply_state=True)) owns that reset. + return + rooter("cleanup_rooter") rooter("cleanup_vrf", routing.routing.internet) diff --git a/tests/test_nexthop_selection.py b/tests/test_nexthop_selection.py index f7e6d6b4689..a7f0de77070 100644 --- a/tests/test_nexthop_selection.py +++ b/tests/test_nexthop_selection.py @@ -596,3 +596,48 @@ def test_scheduler_startup_applies_rooter_state(monkeypatch): monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {}, raising=False) startup.load_nexthop_profiles(_FakeRouting(), apply_rooter_state=True) assert "nexthop_teardown" in recorded and "nexthop_init" in recorded + + +def test_init_rooter_web_does_not_reset_state(monkeypatch): + # codex P1: init_rooter() is called by the web/API (web.settings) BEFORE init_routing. On a + # nexthop-enabled node it now connects, but must NOT run cleanup_rooter/forward_drop/state_* -- + # those remove the scheduler's live per-task iptables rules on a web/gunicorn restart. Only the + # scheduler (init_rooter(apply_state=True)) resets rooter state. + import lib.cuckoo.core.startup as startup + + class _FakeSock: + def connect(self, *a): + pass + + class _R: + class vpn: + enabled = False + + class tor: + enabled = False + + class inetsim: + enabled = False + + class socks5: + enabled = False + + class routing: + route = "none" + internet = "none" + + class nexthop: + enabled = True + + monkeypatch.setattr(startup.socket, "socket", lambda *a, **k: _FakeSock()) + monkeypatch.setattr(startup, "routing", _R, raising=False) + monkeypatch.setattr(startup.subprocess, "run", + lambda *a, **k: type("P", (), {"returncode": 1, "stdout": "", "stderr": ""})()) + recorded = [] + monkeypatch.setattr(startup, "rooter", lambda cmd, *a, **k: recorded.append(cmd) or {"output": True}, raising=False) + + startup.init_rooter() # web path (apply_state=False) + assert recorded == [] # reachability checked, but NO state-reset mutations + + startup.init_rooter(apply_state=True) # scheduler path + assert "cleanup_rooter" in recorded and "forward_drop" in recorded From 2f13418d9a5724d4496fae3b5390bd24bf6dfae0 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 01:13:03 +0000 Subject: [PATCH 42/53] =?UTF-8?q?fix(cleaners):=20lazy=20resolver=20Thread?= =?UTF-8?q?Pool=20=E2=80=94=20importing=20cleaners=5Futils=20no=20longer?= =?UTF-8?q?=20spawns=2050=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Module-level ThreadPool(50) spawned 53 threads in every process that imported cleaners_utils, including the processor. PreforkEngine.run() imports it (for free_space_monitor) before the first fork, so those threads tripped the single-threaded-before-fork invariant — prefork couldn't launch a single task on any upstream-based checkout. Defer pool creation to first .map() use; only the cleaner/deletion paths need it. Prefork prerequisite. --- lib/cuckoo/common/cleaners_utils.py | 38 +++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/cuckoo/common/cleaners_utils.py b/lib/cuckoo/common/cleaners_utils.py index 74bfd20267e..07a5fc89390 100644 --- a/lib/cuckoo/common/cleaners_utils.py +++ b/lib/cuckoo/common/cleaners_utils.py @@ -3,6 +3,7 @@ import os import shutil import sys +import threading import time from contextlib import suppress from datetime import datetime, timedelta @@ -32,8 +33,41 @@ config = Config() repconf = Config("reporting") webconf = Config("web") -resolver_pool = ThreadPool(50) -atexit.register(resolver_pool.close) +class _LazyThreadPool: + """Defers ThreadPool(50) creation until the first .map()/attribute use. + + Importing this module used to eagerly spawn 53 threads (50 workers + 3 + handler threads) in *every* process that imports it -- including + utils/process.py, which only needs free_space_monitor() and never touches + the resolver pool. Those live threads sat in the processor before it forked + its workers, and forking a heavily-multithreaded process deadlocks the + forked children in multiprocessing bootstrap/atexit (_exit_function -> join + -> waitpid), and trips the prefork single-threaded-before-fork invariant. + Only the cleaner/deletion paths actually call resolver_pool.map(), so the + pool is now created on demand by those callers and the processor forks clean. + """ + + _pool = None + _lock = threading.Lock() + + def _get(self): + if self._pool is None: + with self._lock: + if self._pool is None: + self._pool = ThreadPool(50) + return self._pool + + def __getattr__(self, name): + return getattr(self._get(), name) + + +resolver_pool = _LazyThreadPool() + + +@atexit.register +def _close_resolver_pool(): + if resolver_pool._pool is not None: + resolver_pool._pool.close() HAVE_TMPFS = False if hasattr(config, "tmpfs"): From 291bf4a9d2e1404877fcb7cad72b90cc0949a103 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 01:13:03 +0000 Subject: [PATCH 43/53] revert(dedup): drop out-of-scope screenshot mtime-sort reindex_screenshots' chronological mtime sort is unrelated to the processing engine and broke the upstream test (which mocks listdir/rename, not getmtime). Restore upstream's sorted(os.listdir(...)) behavior. --- modules/processing/deduplication.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/processing/deduplication.py b/modules/processing/deduplication.py index 3c5ef6d1fe3..33a414ccc34 100644 --- a/modules/processing/deduplication.py +++ b/modules/processing/deduplication.py @@ -60,12 +60,7 @@ def _load_imagehash(): def reindex_screenshots(shots_path): - # Sort by modification time so interleaved capture sources (guest JPGs - # with 4-digit padded names and host PNGs with unpadded integer names) - # end up in true chronological order after renumbering. - entries = os.listdir(shots_path) - entries.sort(key=lambda f: os.path.getmtime(os.path.join(shots_path, f))) - for i, cur_basename in enumerate(entries): + for i, cur_basename in enumerate(sorted(os.listdir(shots_path))): extension = os.path.splitext(cur_basename)[-1] new_basename = "%s%s" % (str(i).rjust(4, "0"), extension) log.debug("renaming %s to %s", cur_basename, new_basename) From 1f27c6716e7d4d90696604965fb1bb16fdffb7f2 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 01:13:03 +0000 Subject: [PATCH 44/53] test(processor): isolate prefork engine tests in a single-threaded subprocess The pytest process is multi-threaded (other suites leak clients/pools), so it can't satisfy the prefork single-threaded-before-fork invariant nor safely fork. Run PreforkEngine in a fresh interpreter via a helper; expire the parent session before reads (subprocess writes over a separate SQLite connection). Also stub free_space_monitor in the pebble test (run() sys.exit()s on missing storage/analyses). --- tests/_prefork_engine_subproc.py | 91 +++++++++++++++++++++ tests/test_pebble_engine.py | 4 + tests/test_prefork_engine.py | 132 +++++++++++++------------------ 3 files changed, 152 insertions(+), 75 deletions(-) create mode 100644 tests/_prefork_engine_subproc.py diff --git a/tests/_prefork_engine_subproc.py b/tests/_prefork_engine_subproc.py new file mode 100644 index 00000000000..c8a80b7546d --- /dev/null +++ b/tests/_prefork_engine_subproc.py @@ -0,0 +1,91 @@ +"""Subprocess entrypoint for the PreforkEngine behaviour tests. + +The pytest process is multi-threaded (other suites leak Mongo clients, native +C-extension pools, etc.), so it can neither satisfy the prefork single-threaded- +before-fork invariant nor safely os.fork(). Running the engine in a fresh +interpreter here gives a guaranteed single-threaded process. + +Usage: python tests/_prefork_engine_subproc.py +Exit code 0 = engine ran to completion; the parent asserts on DB state / files. +""" +import os +import sys +import time + +# Make the repo root importable when run as a bare script (sys.path[0] is tests/). +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +# Neutralise the real disk-space check: PreforkEngine.run() calls +# free_space_monitor(storage/analyses, ...), which sys.exit()s when that path is +# absent (as in test/CI checkouts). Disk policy is not under test here. +import lib.cuckoo.common.cleaners_utils as _cu # noqa: E402 + +_cu.free_space_monitor = lambda *a, **k: None + +from lib.cuckoo.core.data.task import TASK_REPORTED # noqa: E402 +from lib.cuckoo.core.database import Database, init_database # noqa: E402 +from lib.cuckoo.core.processing_engine.prefork import PreforkEngine # noqa: E402 +from lib.cuckoo.core.processing_engine.source import TaskSource # noqa: E402 + +_AUX_ENV = "_PREFORK_AUX" + + +def _set_reported(task): + db = Database() + with db.session.begin(): + db.set_status(task.id, TASK_REPORTED) + + +def _normal(task): + _set_reported(task) + + +def _crash(task): + os._exit(3) # abnormal exit -> supervisor marks FAILED_PROCESSING + + +def _timeout(task): + # Spawn a grandchild that would outlive the worker, then hang. The grandchild + # inherits the child's process group (the child os.setsid()s), so killpg sweeps it. + import subprocess + + marker = os.environ[_AUX_ENV] + subprocess.Popen([sys.executable, "-c", "import time;open(%r,'w').close();time.sleep(120)" % marker]) + time.sleep(120) + + +def _worker_init_flag(): + open(os.environ[_AUX_ENV], "w").close() + + +def _worker_init_task(task): + assert os.path.exists(os.environ[_AUX_ENV]), "worker_init must run before task_fn in child" + _set_reported(task) + + +def main(): + scenario, dsn, tid, aux = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] + os.environ[_AUX_ENV] = aux + init_database(dsn=dsn) + src = TaskSource(Database()) + + common = dict(source=src, max_count=1) + if scenario == "normal": + eng = PreforkEngine(task_fn=_normal, worker_init=lambda: None, parallel=2, timeout=30, **common) + elif scenario == "crash": + eng = PreforkEngine(task_fn=_crash, worker_init=lambda: None, parallel=2, timeout=30, **common) + elif scenario == "timeout": + eng = PreforkEngine(task_fn=_timeout, worker_init=lambda: None, parallel=1, timeout=1, + term_grace=1, poll_interval=0.1, **common) + elif scenario == "worker_init": + eng = PreforkEngine(task_fn=_worker_init_task, worker_init=_worker_init_flag, + parallel=1, timeout=30, **common) + else: + print("unknown scenario: %s" % scenario, file=sys.stderr) + sys.exit(2) + + eng.run() + + +if __name__ == "__main__": + main() diff --git a/tests/test_pebble_engine.py b/tests/test_pebble_engine.py index b6a339b5048..a9ee3d9dcb0 100644 --- a/tests/test_pebble_engine.py +++ b/tests/test_pebble_engine.py @@ -31,6 +31,10 @@ def test_pebble_engine_processes_one_task(db, temp_pe32, tmp_path, monkeypatch): tid = db.add_path(temp_pe32) db.set_status(tid, TASK_COMPLETED) + # run() calls free_space_monitor(storage/analyses, ...), which sys.exit()s when + # that path doesn't exist (as in CI). Stub it — disk policy isn't under test here. + monkeypatch.setattr("lib.cuckoo.common.cleaners_utils.free_space_monitor", lambda *a, **k: None) + sentinel = str(tmp_path / "ran.txt") monkeypatch.setenv("_PEBBLE_TEST_SENTINEL", sentinel) diff --git a/tests/test_prefork_engine.py b/tests/test_prefork_engine.py index 7e14dec2b64..c1d3ba04567 100644 --- a/tests/test_prefork_engine.py +++ b/tests/test_prefork_engine.py @@ -1,4 +1,6 @@ import os +import subprocess +import sys import threading import time @@ -9,21 +11,34 @@ from lib.cuckoo.core.processing_engine.prefork import PreforkEngine from lib.cuckoo.core.processing_engine.source import TaskSource +_SUBPROC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_prefork_engine_subproc.py") + @pytest.fixture def db_file(tmp_path): """File-backed SQLite fixture needed for fork tests: in-memory sqlite:// - is per-connection so cross-process writes are invisible to the parent.""" + is per-connection so cross-process writes are invisible to the parent. + Yields ``(db, dsn)`` so the isolated subprocess can open the same file.""" dsn = f"sqlite:///{tmp_path}/test.db" reset_database_FOR_TESTING_ONLY() try: init_database(dsn=dsn) retval = Database() - yield retval + yield retval, dsn finally: reset_database_FOR_TESTING_ONLY() +def _run_engine_subprocess(scenario, dsn, tid, aux="", timeout=60): + """Run PreforkEngine in a fresh single-threaded interpreter (the pytest process + is multi-threaded and cannot satisfy the single-threaded-before-fork invariant + nor safely fork). Returns the CompletedProcess.""" + return subprocess.run( + [sys.executable, _SUBPROC, scenario, dsn, str(tid), aux], + capture_output=True, text=True, timeout=timeout, + ) + + def test_single_threaded_invariant_raises_when_extra_thread(db): eng = PreforkEngine(task_fn=lambda t: None, worker_init=lambda: None, source=TaskSource(db), parallel=2, timeout=30) @@ -63,101 +78,68 @@ def fake_listdir(path): def test_normal_task_runs_and_status_set_by_child(db_file, temp_pe32): - with db_file.session.begin(): - tid = db_file.add_path(temp_pe32) - db_file.set_status(tid, TASK_COMPLETED) - - # child runs this; it must set status itself (child sets, supervisor overrides only on failure) - def task_fn(task): - # fresh DB connection in child (file-backed SQLite so the write is - # visible to the parent); must be wrapped in a transaction to commit - from lib.cuckoo.core.database import Database - db = Database() - with db.session.begin(): - db.set_status(task.id, TASK_REPORTED) - - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, - source=TaskSource(db_file), parallel=2, timeout=30, max_count=1) - eng.run() - with db_file.session.begin(): - assert db_file.view_task(tid).status == TASK_REPORTED - - -def test_crashing_task_marked_failed_by_supervisor(db, temp_pe32): + db, dsn = db_file with db.session.begin(): tid = db.add_path(temp_pe32) db.set_status(tid, TASK_COMPLETED) - def task_fn(task): - os._exit(3) # simulate abnormal exit / crash + r = _run_engine_subprocess("normal", dsn, tid) + assert r.returncode == 0, r.stderr - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, - source=TaskSource(db), parallel=2, timeout=30, max_count=1) - eng.run() + db.session.expire_all() # subprocess wrote via a separate connection; drop cached rows with db.session.begin(): - assert db.view_task(tid).status == TASK_FAILED_PROCESSING + assert db.view_task(tid).status == TASK_REPORTED -def test_timeout_kills_process_group_no_orphans(db, temp_pe32): +def test_crashing_task_marked_failed_by_supervisor(db_file, temp_pe32): + db, dsn = db_file with db.session.begin(): tid = db.add_path(temp_pe32) db.set_status(tid, TASK_COMPLETED) - marker = "/tmp/prefork_orphan_%d" % os.getpid() + r = _run_engine_subprocess("crash", dsn, tid) + assert r.returncode == 0, r.stderr - def task_fn(task): - # spawn a grandchild that would outlive the worker, then hang - import subprocess - import sys - subprocess.Popen([sys.executable, "-c", - "import time,os;open(%r,'w').close();time.sleep(120)" % marker]) - time.sleep(120) + db.session.expire_all() + with db.session.begin(): + assert db.view_task(tid).status == TASK_FAILED_PROCESSING + + +def test_timeout_kills_process_group_no_orphans(db_file, temp_pe32, tmp_path): + db, dsn = db_file + with db.session.begin(): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) - eng = PreforkEngine(task_fn=task_fn, worker_init=lambda: None, source=TaskSource(db), - parallel=1, timeout=1, term_grace=1, max_count=1, poll_interval=0.1) - if os.path.exists(marker): - os.unlink(marker) - eng.run() + marker = str(tmp_path / "orphan_marker") + r = _run_engine_subprocess("timeout", dsn, tid, aux=marker, timeout=60) + assert r.returncode == 0, r.stderr + db.session.expire_all() with db.session.begin(): assert db.view_task(tid).status == TASK_FAILED_PROCESSING - # grandchild must have been swept by killpg (give it a moment) - time.sleep(2) - # Sanity: the grandchild must have started and written the marker - # before being killed; otherwise the pgrep check below would pass vacuously. + # Sanity: the grandchild must have started (written the marker) or the orphan + # check below would pass vacuously. assert os.path.exists(marker), "grandchild never started — test is vacuous" - import subprocess + time.sleep(2) # give any survivor a chance to appear out = subprocess.run(["pgrep", "-f", marker], capture_output=True, text=True) assert out.stdout.strip() == "", "orphaned grandchild survived killpg" - if os.path.exists(marker): - os.unlink(marker) - -def test_worker_init_called_in_child(db_file, temp_pe32): - with db_file.session.begin(): - tid = db_file.add_path(temp_pe32) - db_file.set_status(tid, TASK_COMPLETED) - flag = "/tmp/prefork_winit_ran_%d" % os.getpid() - def worker_init(): - open(flag, "w").close() +def test_worker_init_called_in_child(db_file, temp_pe32, tmp_path): + db, dsn = db_file + with db.session.begin(): + tid = db.add_path(temp_pe32) + db.set_status(tid, TASK_COMPLETED) - def task_fn(task): - assert os.path.exists(flag), "worker_init must run before task_fn in child" - from lib.cuckoo.core.database import Database - db = Database() - with db.session.begin(): - db.set_status(task.id, TASK_REPORTED) + flag = str(tmp_path / "winit_flag") + # The subprocess's task_fn asserts the flag exists before setting status, so a + # REPORTED result proves worker_init ran before task_fn in the child. + r = _run_engine_subprocess("worker_init", dsn, tid, aux=flag) + assert r.returncode == 0, r.stderr - if os.path.exists(flag): - os.unlink(flag) - try: - eng = PreforkEngine(task_fn=task_fn, worker_init=worker_init, source=TaskSource(db_file), - parallel=1, timeout=30, max_count=1) - eng.run() - with db_file.session.begin(): - assert db_file.view_task(tid).status == TASK_REPORTED - finally: - if os.path.exists(flag): - os.unlink(flag) + assert os.path.exists(flag), "worker_init did not run" + db.session.expire_all() + with db.session.begin(): + assert db.view_task(tid).status == TASK_REPORTED From b9c2b38926e1b7f3b5f31a8a5546f149effff954 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 01:17:01 +0000 Subject: [PATCH 45/53] style: drop unused tid binding in prefork test subprocess helper (ruff F841) --- tests/_prefork_engine_subproc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/_prefork_engine_subproc.py b/tests/_prefork_engine_subproc.py index c8a80b7546d..0e896063283 100644 --- a/tests/_prefork_engine_subproc.py +++ b/tests/_prefork_engine_subproc.py @@ -64,7 +64,9 @@ def _worker_init_task(task): def main(): - scenario, dsn, tid, aux = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] + # argv[3] is the task id; the engine fetches tasks from the DB itself, so the + # helper doesn't use it (it's there for a uniform parent-side call signature). + scenario, dsn, _tid, aux = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] os.environ[_AUX_ENV] = aux init_database(dsn=dsn) src = TaskSource(Database()) From a5d07341918b7269faf0c7da113c143c62fe85b9 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 01:28:26 +0000 Subject: [PATCH 46/53] fix(prefork): SIGKILL escalation falls back to os.kill(pid) when killpg misses Mirror the _enforce_timeouts fallback in _escalate_kills: if a child hung before os.setsid() its process group won't exist, so killpg(SIGKILL) raises ProcessLookupError and the child would leak. Signal the pid directly instead. --- lib/cuckoo/core/processing_engine/prefork.py | 7 ++++++- tests/test_prefork_reaping.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/cuckoo/core/processing_engine/prefork.py b/lib/cuckoo/core/processing_engine/prefork.py index 30a2aaa1cf4..f4ef28c21c0 100644 --- a/lib/cuckoo/core/processing_engine/prefork.py +++ b/lib/cuckoo/core/processing_engine/prefork.py @@ -173,7 +173,12 @@ def _escalate_kills(self): try: os.killpg(child.pgid, signal.SIGKILL) except ProcessLookupError: - pass + # Group may not exist (child hung before os.setsid()); signal the + # pid directly so a hung child can't leak. Mirrors _enforce_timeouts. + try: + os.kill(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass except OSError as e: log.warning("prefork: killpg(SIGKILL) failed for task %d (pid %d): %s", child.task_id, child.pid, e) diff --git a/tests/test_prefork_reaping.py b/tests/test_prefork_reaping.py index 18cc1da3879..b122d662459 100644 --- a/tests/test_prefork_reaping.py +++ b/tests/test_prefork_reaping.py @@ -70,3 +70,20 @@ def test_enforce_timeouts_falls_back_to_kill_when_killpg_missing(monkeypatch): assert child.timed_out is True assert (222, signal.SIGTERM) in killed, "must fall back to os.kill(pid) when killpg raises ProcessLookupError" assert child.kill_deadline is not None, "kill_deadline must be set so SIGKILL escalation still runs" + + +def test_escalate_kills_falls_back_to_kill_when_killpg_missing(monkeypatch): + eng = _engine() + child = _Child(task_id=3, pid=333, start=0.0, pgid=333) + child.timed_out = True + child.kill_deadline = -1.0 # deadline already passed -> escalate now + eng._inflight[333] = child + + killed = [] + monkeypatch.setattr(os, "killpg", lambda pgid, sig: (_ for _ in ()).throw(ProcessLookupError())) + monkeypatch.setattr(os, "kill", lambda pid, sig: killed.append((pid, sig))) + + eng._escalate_kills() + + assert (333, signal.SIGKILL) in killed, "SIGKILL must fall back to os.kill(pid) when killpg raises ProcessLookupError" + assert child.kill_deadline is None From d90c6485d229636bb4e92cdd6ca1f89b58ab77a1 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Fri, 10 Jul 2026 10:11:30 -0500 Subject: [PATCH 47/53] central-mode: optional object-store artifact seam + job->worker directory (default OFF) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in "central mode" that lets a CAPE web/API node serve a fleet of workers from a shared data plane, WITHOUT changing single-node behavior. Everything is gated by a new [central_mode] stanza that defaults to enabled = no, so a stock install is byte-for-byte unchanged (verified by an OFF-path + MT-absent import smoke test). What it adds (all lazy / import-optional): - [central_mode] config (conf/default/cuckoo.conf.default + central_mode.py): toggle + vendor-neutral backend settings. boto3 stays an optional, lazily-imported dependency (never core) — the same way the repo's existing modules/machinery/aws.py already treats it. - FS -> object-store artifact seam (storage_backend.py + artifact_storage.py, read side; modules/reporting/centralstore.py, write side). Artifacts key by // on any S3-compatible store (AWS S3 / MinIO / Ceph) OR a shared local/NFS mount. The read seam lazily stages the tree back to the local storage/analyses// dir so the many report features that read the local filesystem work unchanged. A completion marker (.centralstore.done) gates the read-side staging cache and the worker's local cleanup. - Pluggable job -> worker directory (job_directory.py + central_guac.py) so the central node can route interactive Guacamole to the worker hosting a live task's VM. Default backend is the vendor-neutral broker HTTP status API; an optional lazy-boto3 DynamoDB backend ships too. Worker access paths (api-token file, port, ssh user/keyfile) are config-driven. - tenancy-optional facades (tenancy_optional.py x2, analysis/central_scope.py) + a settings.py _users_app_present() shim. Central mode is ORTHOGONAL to multi-tenancy: the facades delegate to the multi-tenant `users` app when it is deployed and degrade to single-tenant see-all (fail-CLOSED on runtime errors) via ImportError when it is not — so this lands independently of any multi-tenant layer. - DocumentDB compatibility knobs in dev_utils/mongodb.py (explicit tls= / retryWrites=, both defaulting to today's behavior, documented in conf/default/reporting.conf.default) and a $facet -> per-category $group hunt aggregation (hunt_query.py) that Amazon DocumentDB accepts; the single-node $facet path is unchanged. The web view/endpoint changes (analysis/views.py, apiv2/views.py, guac/*, submission/views.py) are all `if central_mode_config().enabled:` seam sites with function-local imports; the OFF path is the original code (submission status/remote_session gate the worker-VM fallback on central mode, so single-node is byte-for-byte upstream — no degenerate empty-guest_ip session). guac/views.py additionally derives the interactive VM label + guest IP authoritatively from the task rather than trusting the session_data path segment (prevents a takeover/SSRF via a forged label/IP). Tests: tests/test_central_mode.py (config + store + directory + marker + hunt), tenancy_optional + central_scope facade tests (MT-absent behavior runs for real; MT-present delegation/fail-closed cases importorskip when the users app is absent), test_mt_absent_smoke.py (proves every touched view imports and the facades degrade with the MT layer absent) and test_settings_mt_optional.py. --- conf/default/cuckoo.conf.default | 37 +++ conf/default/reporting.conf.default | 6 + dev_utils/mongodb.py | 15 + lib/cuckoo/common/artifact_storage.py | 205 +++++++++++++ lib/cuckoo/common/central_guac.py | 155 ++++++++++ lib/cuckoo/common/central_mode.py | 146 +++++++++ lib/cuckoo/common/hunt_query.py | 27 ++ lib/cuckoo/common/job_directory.py | 152 ++++++++++ lib/cuckoo/common/storage_backend.py | 232 ++++++++++++++ lib/cuckoo/common/tenancy_optional.py | 100 +++++++ modules/reporting/centralstore.py | 271 +++++++++++++++++ tests/test_central_mode.py | 416 ++++++++++++++++++++++++++ tests/test_mt_absent_smoke.py | 85 ++++++ tests/test_tenancy_optional.py | 64 ++++ web/analysis/central_scope.py | 45 +++ web/analysis/central_views.py | 360 ++++++++++++++++++++++ web/analysis/test_central_scope.py | 85 ++++++ web/analysis/views.py | 242 +++++++++++---- web/apiv2/views.py | 51 ++++ web/guac/consumers.py | 30 +- web/guac/views.py | 36 ++- web/submission/views.py | 39 ++- web/web/settings.py | 19 ++ web/web/tenancy_optional.py | 113 +++++++ web/web/test_settings_mt_optional.py | 36 +++ 25 files changed, 2898 insertions(+), 69 deletions(-) create mode 100644 lib/cuckoo/common/artifact_storage.py create mode 100644 lib/cuckoo/common/central_guac.py create mode 100644 lib/cuckoo/common/central_mode.py create mode 100644 lib/cuckoo/common/hunt_query.py create mode 100644 lib/cuckoo/common/job_directory.py create mode 100644 lib/cuckoo/common/storage_backend.py create mode 100644 lib/cuckoo/common/tenancy_optional.py create mode 100644 modules/reporting/centralstore.py create mode 100644 tests/test_central_mode.py create mode 100644 tests/test_mt_absent_smoke.py create mode 100644 tests/test_tenancy_optional.py create mode 100644 web/analysis/central_scope.py create mode 100644 web/analysis/central_views.py create mode 100644 web/analysis/test_central_scope.py create mode 100644 web/web/tenancy_optional.py create mode 100644 web/web/test_settings_mt_optional.py diff --git a/conf/default/cuckoo.conf.default b/conf/default/cuckoo.conf.default index 3af62a9c577..0b74a1501ab 100644 --- a/conf/default/cuckoo.conf.default +++ b/conf/default/cuckoo.conf.default @@ -246,3 +246,40 @@ analysis = 0 mongo = no # Clean orphan files in mongodb unused_files_in_mongodb = no + +[central_mode] +# Central control-plane mode (off = current single-node behavior; analyses stay on the local +# storage/analyses tree). When on, analysis artifacts are pushed to a shared object store and +# interactive Guac routing resolves job->worker via a directory backend. +enabled = no +# Artifact storage backend when central mode is ON: "s3" (any S3-compatible store — AWS S3, MinIO, +# Ceph RGW — via boto3, which is an optional/lazy import: pip install boto3 to use it) or "local" +# (a shared local/NFS mount, no extra deps). +storage_backend = s3 +# S3-compatible object store (storage_backend=s3). endpoint blank = AWS default; creds blank = +# boto3 default credential chain (e.g. an instance role). Point endpoint/creds at MinIO/Ceph/etc. +s3_bucket = +s3_region = us-east-1 +s3_prefix = results +s3_endpoint_url = +s3_access_key = +s3_secret_key = +# storage_backend=local: root the per-job artifact trees live under (blank = local analyses tree). +central_local_root = +# Job->worker directory for interactive Guac routing: "broker_http" (default; vendor-neutral — +# resolves via the broker's GET /api/status/) or "dynamodb" (AWS; reads broker_table). +job_directory = broker_http +# job_directory=broker_http: broker base URL + optional Bearer API token. +broker_url = +broker_api_token = +# job_directory=dynamodb: the broker job-tracking DynamoDB table name. +broker_table = +# Interactive-Guac worker access: how the central node reaches the worker that hosts a live task's +# VM. Defaults match the standard CAPE deb layout; override for other topologies. +# File holding the worker apiv2 Token (blank/absent => requests go unauthenticated). +worker_api_token_file = /etc/cape/api-token +# The worker's apiv2/web port. +worker_api_port = 8000 +# The central node's libvirt-over-SSH identity onto workers (must be authorized on the workers). +worker_ssh_user = cape +worker_ssh_keyfile = /home/cape/.ssh/id_ed25519 diff --git a/conf/default/reporting.conf.default b/conf/default/reporting.conf.default index 5f5bb47ec0a..2633e4fd1ef 100644 --- a/conf/default/reporting.conf.default +++ b/conf/default/reporting.conf.default @@ -105,6 +105,12 @@ index_filenames = no # Set this value if you are using mongodb with TLS enabled # tlscafile = +# Enable TLS on the connection. Required for managed backends like Amazon DocumentDB +# (a tlscafile alone does NOT enable TLS in pymongo). Default off = today's behavior. +# tls = no +# Retryable writes. Amazon DocumentDB rejects them, so set this to no when pointing at +# DocumentDB. Default yes = pymongo's own default (unchanged for stock MongoDB). +# retrywrites = yes # Automatically delete large dict values that exceed mongos 16MB limitation # Note: This only deletes dict keys from data stored in MongoDB. You would diff --git a/dev_utils/mongodb.py b/dev_utils/mongodb.py index 3dad5be7fc8..1ddf4cc1b28 100644 --- a/dev_utils/mongodb.py +++ b/dev_utils/mongodb.py @@ -13,6 +13,15 @@ mdb = repconf.mongodb.get("db", "cuckoo") +def _truthy(val, default=False): + """Coerce a conf value (bool or 'yes'/'no'/'on'/'off'/...) to bool.""" + if isinstance(val, bool): + return val + if val is None: + return default + return str(val).strip().lower() in ("1", "true", "yes", "on") + + if repconf.mongodb.enabled: from pymongo import MongoClient, version_tuple from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError @@ -30,7 +39,13 @@ def connect_to_mongo() -> MongoClient: username=repconf.mongodb.get("username"), password=repconf.mongodb.get("password"), authSource=repconf.mongodb.get("authsource", "cuckoo"), + # DocumentDB (central mode) needs TLS explicitly on (a CA file alone + # does NOT enable TLS in pymongo) and retryWrites OFF (DocumentDB + # rejects retryable writes). Both default to today's single-node + # behavior: tls off, retryWrites on (pymongo's own default). + tls=_truthy(repconf.mongodb.get("tls", False), False), tlsCAFile=repconf.mongodb.get("tlscafile", None), + retryWrites=_truthy(repconf.mongodb.get("retrywrites", True), True), connect=True, # Force connection now to catch issues serverSelectionTimeoutMS=5000, socketTimeoutMS=30000, diff --git a/lib/cuckoo/common/artifact_storage.py b/lib/cuckoo/common/artifact_storage.py new file mode 100644 index 00000000000..43702a3d3fb --- /dev/null +++ b/lib/cuckoo/common/artifact_storage.py @@ -0,0 +1,205 @@ +"""FS->object-store artifact seam. central_mode OFF -> local storage/analyses// +; ON -> the configured backend (S3-compatible or shared local mount) at +//. The single-node vs central decision + the CAPE-specific bits +(task_id->job_id resolution, tenant scope, traversal guard) live here; the raw object I/O +is delegated to a pluggable ArtifactStore (lib/cuckoo/common/storage_backend.py) so central +mode runs on any S3-compatible store (AWS/MinIO/Ceph) or a shared mount, with AWS purely +config. Validated live on a CAPE box; the branch/seam logic here is the unit-testable part. +""" +import os + +from lib.cuckoo.common.constants import CUCKOO_ROOT +from lib.cuckoo.common.central_mode import central_mode_config +from lib.cuckoo.common.storage_backend import get_artifact_store, ArtifactNotFound + + +def _local_analysis_path(task_id, relpath): + return os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), relpath) + + +def _safe_relpath(relpath): + """Reject traversal/absolute/backslash before a relpath becomes an object key or a + local path segment. Callers today pass regex-constrained (\\w+) or fixed relpaths, but + this keeps the seam safe independent of caller discipline (audit MEDIUM-1).""" + from django.http import Http404 + + if not relpath or relpath.startswith("/") or "\\" in relpath or ".." in relpath.split("/"): + raise Http404(f"invalid artifact path: {relpath!r}") + return relpath + + +def _job_id_for_task(task_id, scope=None): + """Central mode keys the store by the global job_id (the broker passes it in custom, + stamped into info.job_id at reporting; centralstore re-keys info.id to the unique + central task id). Resolve task_id -> job_id via mongo. + + `scope` is the requesting viewer's tenant filter (e.g. entitled_scope_filter): + info.id is a per-worker sequence and collides across workers in a central + deployment, so the lookup is ANDed with the viewer's scope to guarantee the + resolved doc is one the viewer may actually see — not another tenant's analysis + that happens to share the numeric id (audit HIGH: cross-store id collision).""" + from dev_utils.mongodb import mongo_find_one + from django.http import Http404 + + # filereport/full_memory routes capture task_id/analysis_number as \w+ (not \d+), + # so a non-numeric segment must raise Http404 (the views catch it -> clean error), + # not an uncaught ValueError -> HTTP 500. + try: + tid = int(task_id) + except (TypeError, ValueError): + raise Http404("invalid task id") + query = {"info.id": tid} + if scope: + query = {"$and": [query, scope]} + doc = mongo_find_one("analysis", query, {"info.job_id": 1}) + job_id = (doc or {}).get("info", {}).get("job_id") + if not job_id: + raise Http404("no job_id mapping for task") + return job_id + + +def _store_and_container(task_id, scope=None): + """Return (ArtifactStore, container) for an analysis. Single-node: the local-FS store + over storage/analyses, container=. Central: the configured backend (S3/local + mount), container="/" (raises Http404 if the job_id can't resolve).""" + cfg = central_mode_config() + store, is_central = get_artifact_store(cfg) + if not is_central: + return store, str(task_id) + return store, f"{cfg.s3_prefix}/{_job_id_for_task(task_id, scope)}" + + +def artifact_response(task_id, relpath, content_type, filename, chunk=8192, scope=None): + """Return a Django streaming response for an analysis artifact from any backend. + + `scope`: the requesting viewer's tenant filter, threaded into the central + task_id->job_id lookup so a viewer can't pull another tenant's artifact via an + id collision (see _job_id_for_task).""" + from django.http import StreamingHttpResponse, Http404 + + _safe_relpath(relpath) + store, container = _store_and_container(task_id, scope) # may raise Http404 (no job_id) + try: + body_iter, length = store.stream(container, relpath, chunk) + except ArtifactNotFound: + raise Http404(f"artifact not found: {relpath}") + resp = StreamingHttpResponse(body_iter, content_type=content_type) + if length is not None: + resp["Content-Length"] = length + resp["Content-Disposition"] = f"attachment; filename={filename}" + return resp + + +def _stage_tree(task_id, scope, want): + """Shared staging core for central mode: copy artifacts from the central store into the + local storage/analyses// tree so the MANY report features that read the local + filesystem work centrally without rewriting each reader. `want(rel) -> bool` selects which + relpaths to stage. Returns True iff the .centralstore.done completion marker was seen. + Best-effort: never raises. No-op single-node (caller guards).""" + store, container = _store_and_container(task_id, scope) + local = _local_analysis_path(task_id, "") + os.makedirs(local, exist_ok=True) + local_real = os.path.realpath(local) + complete = False + for rel in store.iter_relpaths(container): + if rel == ".centralstore.done": + complete = True + continue # completion marker is a control object, not an artifact — don't stage it + if not rel or rel.endswith("/") or not want(rel): + continue + # Defence-in-depth: an object key suffix must never escape the analysis dir when it + # becomes a LOCAL destination (centralstore only emits in-tree keys today). + try: + _safe_relpath(rel) + except Exception: + continue + dest = os.path.join(local, rel) + dest_real = os.path.realpath(dest) + if dest_real != local_real and not dest_real.startswith(local_real + os.sep): + continue + if os.path.exists(dest): + continue + store.download(container, rel, dest) + return complete + + +def ensure_local_analysis(task_id, scope=None, exclude_prefixes=("memory/", "memory.dmp")): + """Central mode: lazily materialize the central results// tree into the local + storage/analyses// dir so the report features that read the local filesystem + (json report, evtx, ETW aux/*.json, sysmon, process.log, behavior feeds, dropped files, + …) work centrally without rewriting each reader. Cached via a .central_staged marker + (subsequent calls are a cheap stat) — written only after the .centralstore.done marker is + seen, so a listing taken mid-upload isn't cached as complete. Excludes the large on-demand + memory dumps (memory/ subtree + the root memory.dmp[.zip/.strings] full-RAM image) which + would otherwise bloat every report view by GBs; they stage on demand via ensure_local_ + memory / stream via materialize_artifact. No-op single-node. Best-effort: never raises.""" + cfg = central_mode_config() + if not cfg.enabled: + return + marker = os.path.join(_local_analysis_path(task_id, ""), ".central_staged") + if os.path.exists(marker): + return + try: + complete = _stage_tree( + task_id, scope, want=lambda rel: not any(rel.startswith(p) for p in exclude_prefixes) + ) + if complete: + with open(marker, "w") as f: + f.write("staged") + except Exception: + # leave whatever was staged; the per-file seam still covers downloads + pass + + +def ensure_local_memory(task_id, scope=None): + """Central mode: stage the memory dumps (the memory/ per-process subtree AND the root + memory.dmp[.zip/.strings] full-RAM image) — which ensure_local_analysis EXCLUDES from the + bulk stage because they are large — to the local analysis dir, on EXPLICIT demand (the + memory-download endpoints). Idempotent per-file; not marker-gated. Best-effort.""" + cfg = central_mode_config() + if not cfg.enabled: + return + try: + _stage_tree(task_id, scope, want=lambda rel: rel.startswith("memory/") or rel.startswith("memory.dmp")) + except Exception: + pass + + +def artifact_exists(task_id, relpath, scope=None): + """True iff an analysis artifact exists — local (single-node) or via the central store's + existence check. Used to gate optional UI download links (decrypted/mixed pcap, tlskeys, + mitmdump) the worker may or may not have produced; a local-FS check returns False for + central-backed artifacts, hiding links for files that actually exist.""" + try: + _safe_relpath(relpath) + store, container = _store_and_container(task_id, scope) + return store.exists(container, relpath) + except Exception: + return False + + +def materialize_artifact(task_id, relpath, scope=None): + """Return (local_path, is_temp) for an artifact a caller must open as a real file — + random-access slicing (procdump), filtering/regeneration (pcap), or handing to an external + tool (VT upload). Single-node: the real local path, is_temp=False (DO NOT delete it). + Central: the object streamed to a temp file, is_temp=True (caller deletes it in a finally). + Returns (None, False) if absent.""" + try: + _safe_relpath(relpath) + store, container = _store_and_container(task_id, scope) + return store.materialize(container, relpath) + except Exception: + return (None, False) + + +def read_artifact_text(task_id, relpath, max_bytes=100000, scope=None): + """Read a text artifact (e.g. process.log) from any backend, truncated to max_bytes.""" + try: + _safe_relpath(relpath) + store, container = _store_and_container(task_id, scope) + data = store.read_text(container, relpath, max_bytes) + except Exception: + return "" + if len(data) > max_bytes: + return data[:max_bytes] + "\n... [TRUNCATED] ..." + return data diff --git a/lib/cuckoo/common/central_guac.py b/lib/cuckoo/common/central_guac.py new file mode 100644 index 00000000000..248918be675 --- /dev/null +++ b/lib/cuckoo/common/central_guac.py @@ -0,0 +1,155 @@ +"""Central-mode interactive-Guacamole worker routing. + +Single-node: a task's live analysis VM is in local libvirt and guacd is on +localhost. In the broker/autoscaling topology the VM runs on an ephemeral ASG +worker, so the central guac consumer must target THAT worker's guacd + VM. + +worker_ip_for_task() resolves task_id (info.id) -> info.job_id -> the broker job +record's sandbox_worker_ip (recorded by the dispatcher at dispatch time) -> the +worker's private IP. The job record is fetched via a pluggable JobDirectory +(job_directory.py): the broker's HTTP status API by default (vendor-neutral), or +DynamoDB (opt-in, AWS) — so the fork carries no hard DynamoDB/boto3 dependency. Returns None for box-local / +single-node tasks (no broker record), so the consumer/view keep their localhost +path unchanged when central mode is off or the task ran locally. +""" +import logging + +log = logging.getLogger(__name__) + + +def _worker_api_token(token_file): + """Read the worker apiv2 Token from token_file; '' if unset/unreadable (=> no auth header). + Never raises — a missing token file just means unauthenticated requests.""" + try: + with open(token_file) as f: + return f.read().strip() + except Exception: + return "" + + +def _worker_task_view_url(worker_ip, port, cape_task_id): + """The worker's apiv2 task-view URL. port comes from [central_mode] worker_api_port.""" + return "http://%s:%d/apiv2/tasks/view/%d/" % (worker_ip, int(port), int(cape_task_id)) + + +def _libvirt_ssh_dsn(ip, ssh_user, keyfile): + """qemu+ssh libvirt DSN to a worker. ssh_user/keyfile come from [central_mode] + (worker_ssh_user/worker_ssh_keyfile) so the deb defaults aren't hardcoded. keyfile is + URL-quoted (safe='/') so a configured path with a '&'/'?'/space can't corrupt the query.""" + from urllib.parse import quote + + return "qemu+ssh://%s@%s/system?keyfile=%s&no_verify=1" % (ssh_user, ip, quote(keyfile, safe="/")) + + +def _job_id_for_task(task_id): + """Resolve the broker job_id for a task. Prefer the RDS task.custom stamp + ('job_id=ui-', set by the submit-bridge at enqueue) — it exists DURING the + live run, which is exactly when interactive guac is needed. The DocumentDB + analysis doc is only written at reporting (after the VM is gone), so it can't be + relied on here; fall back to it only for non-bridged/seeded tasks.""" + try: + from lib.cuckoo.core.database import Database + + t = Database().view_task(int(task_id)) + custom = getattr(t, "custom", None) if t else None + if custom: + # comma-separated k=v pairs — take ONLY the job_id= value (not the rest of the + # string), matching centralstore.resolve_job_id; a trailing ',foo=bar' would + # otherwise corrupt the DynamoDB key / S3 prefix lookup. + for part in str(custom).split(","): + part = part.strip() + if part.startswith("job_id="): + v = part.split("=", 1)[1].strip() + if v: + return v + except Exception: + pass + try: + from dev_utils.mongodb import mongo_find_one + + doc = mongo_find_one("analysis", {"info.id": int(task_id)}, {"info.job_id": 1}) + return (doc or {}).get("info", {}).get("job_id") + except Exception: + return None + + +def _worker_ip(cfg, task_id): + """Resolve task_id -> the hosting worker's private IP using an ALREADY-loaded cfg, so + worker_ip_for_task and libvirt_dsn_for_task parse [central_mode] once per call instead of + twice. Returns None (local/unresolvable). The IP is validated in job_directory.loc_from_item.""" + from lib.cuckoo.common.job_directory import get_job_directory + + directory = get_job_directory(cfg) + if directory is None: + return None + try: + job_id = _job_id_for_task(task_id) + if not job_id: + return None + loc = directory.lookup(job_id) + return loc.worker_ip if loc else None + except Exception as e: + log.warning("central guac: worker resolution failed for task %s: %s", task_id, e) + return None + + +def worker_ip_for_task(task_id): + """Private IP of the worker hosting this task's live VM, or None (local).""" + from lib.cuckoo.common.central_mode import central_mode_config + + return _worker_ip(central_mode_config(), task_id) + + +def worker_vm_for_task(task_id): + """For a live broker-dispatched interactive task, return (vm_label, guest_ip) of the + VM on the worker — needed to build the guac session_data on the central node, where + the local machines table is empty (the VM lives on the worker). Resolves the broker + record (job_id -> worker IP + the worker-local cape_task_id) then asks that worker's + apiv2 for the task's machine. Returns (None, None) for non-bridged/local tasks.""" + from lib.cuckoo.common.central_mode import central_mode_config + from lib.cuckoo.common.job_directory import get_job_directory + + cfg = central_mode_config() + directory = get_job_directory(cfg) + if directory is None: + return (None, None) + try: + job_id = _job_id_for_task(task_id) + if not job_id: + return (None, None) + loc = directory.lookup(job_id) + if not loc: + return (None, None) + worker_ip = loc.worker_ip + cape_task_id = loc.cape_task_id + if not worker_ip or cape_task_id is None: + return (None, None) + + import requests + + token = _worker_api_token(cfg.worker_api_token_file) + headers = {"Authorization": f"Token {token}"} if token else {} + r = requests.get(_worker_task_view_url(worker_ip, cfg.worker_api_port, cape_task_id), + headers=headers, timeout=10) + data = (r.json() or {}).get("data", {}) + return (data.get("machine"), None) # central guac uses the worker's localhost for VNC + except Exception as e: + log.warning("central guac: worker VM lookup failed for task %s: %s", task_id, e) + return (None, None) + + +def libvirt_dsn_for_task(task_id, local_dsn): + """libvirt DSN to query the VM's VNC port: the worker's libvirt over SSH for a + worker-hosted task, else the local DSN. (Requires the central node's worker_ssh_user + to hold worker_ssh_keyfile, authorized on workers — deploy-time plumbing.)""" + from lib.cuckoo.common.central_mode import central_mode_config + + cfg = central_mode_config() # loaded once; shared with the worker-IP resolution below + ip = _worker_ip(cfg, task_id) + if not ip: + return (local_dsn, None) + # The central node's libvirt-over-SSH identity onto workers comes from [central_mode] + # (worker_ssh_user/worker_ssh_keyfile); no_verify skips host-key prompts for ephemeral + # in-VPC workers. + dsn = _libvirt_ssh_dsn(ip, cfg.worker_ssh_user, cfg.worker_ssh_keyfile) + return (dsn, ip) diff --git a/lib/cuckoo/common/central_mode.py b/lib/cuckoo/common/central_mode.py new file mode 100644 index 00000000000..817a0d30e9e --- /dev/null +++ b/lib/cuckoo/common/central_mode.py @@ -0,0 +1,146 @@ +"""Central-mode toggle. OFF (default) = single-node behavior (local pg/mongo/FS). +ON = the web/api app reads the central data plane (RDS via [database], DocumentDB +via [mongodb]) and serves artifacts from S3. + +The relational + mongo CONNECTIONS are pointed by their existing conf; this flag +gates the code-level behavior that has no conf today — the FS->S3 artifact seam — +and carries the S3 location. Parsing logic is split into the pure `_parse()` helper +so it's unit-testable without importing the CAPE config machinery. +""" +import os +from dataclasses import dataclass + + +def _within(realpath, root): + """True iff `realpath` is `root` itself or lives under it. Uses a separator- + terminated prefix so a sibling like storage/binaries-evil does NOT count as + inside storage/binaries (prefix-collision guard).""" + return realpath == root or realpath.startswith(root.rstrip(os.sep) + os.sep) + + +def upload_target_realpath(full, base_real, trusted_roots): + """Decide whether an analysis file may be shipped to the central store, and if + so, the on-disk path whose CONTENT to upload. + + centralstore walks the analysis tree. Most entries are regular files inside the + tree. A few are symlinks INTO trusted content roots — `binary` -> storage/binaries/ + and any guacrecording symlinked from the analysis dir -> storage/ + guacrecordings/. Those must ship (the read seam serves them), so we resolve and + upload their target content under the file's analysis-relative key. + + But artifacts are partly sample-influenced: a planted symlink (e.g. binary -> + ~/.aws/credentials) would otherwise be read and exfiltrated to S3 (audit + CRITICAL). So we ALLOW a resolved path only when it stays within the analysis + tree itself or one of the trusted_roots; anything resolving elsewhere returns + None (skip). Pure (os.path only) so it is unit-testable without boto3/Django. + """ + realpath = os.path.realpath(full) + for root in [base_real, *trusted_roots]: + if _within(realpath, root): + return realpath + return None + + +def _as_bool(v, default=False): + if isinstance(v, bool): + return v + if v is None: + return default + return str(v).strip().lower() in ("1", "true", "yes", "on") + + +def _as_int(v, default): + """Coerce a config value to int, falling back to `default` on None/blank/garbage so a + typo in [central_mode] degrades to the documented default instead of a startup crash.""" + try: + return int(str(v).strip()) + except (TypeError, ValueError): + return default + + +def _as_port(v, default): + """A TCP port from config. int() accepts syntactically-valid but out-of-range values + (0, -1, 99999) that would then format into a broken URL and silently kill worker resolution; + fall back to `default` unless the value is a real port (1..65535).""" + p = _as_int(v, default) + return p if 1 <= p <= 65535 else default + + +@dataclass +class CentralModeConfig: + enabled: bool = False + s3_bucket: str = "" + s3_region: str = "us-east-1" + s3_prefix: str = "results" # results//... + # Artifact storage backend (when central mode is ON). "s3" = any S3-compatible object + # store (AWS S3, MinIO, Ceph RGW, …) via boto3; "local" = a shared local/NFS mount. + # Single-node (enabled=False) always uses the local storage/analyses tree regardless. + storage_backend: str = "s3" + # S3-compatible endpoint + creds. ALL OPTIONAL: empty s3_endpoint_url -> AWS's default + # endpoint; empty creds -> boto3's default chain (IAM role on AWS). Set them to point at + # MinIO/Ceph/etc. — this is the ONLY thing that made the artifact store AWS-locked. + s3_endpoint_url: str = "" + s3_access_key: str = "" + s3_secret_key: str = "" + # For storage_backend="local" (central with a shared mount): the root the per-job + # artifact trees live under (results//...). Empty -> falls back to the local + # storage/analyses tree. + central_local_root: str = "" + # Broker job-tracking DynamoDB table — lets the central node resolve a live + # job to the worker hosting its VM (interactive Guacamole worker routing). + broker_table: str = "" + # Job->worker directory backend for interactive guac routing (central_guac.py): + # "broker_http" (default; vendor-neutral — resolves via the broker's GET /api/status/, + # so no DynamoDB/boto3) or "dynamodb" (AWS; reads broker_table). + job_directory: str = "broker_http" + # For job_directory="broker_http": the broker base URL + Bearer API token. + broker_url: str = "" + broker_api_token: str = "" + # Interactive-Guac worker access (central_guac.py resolves a live task's VM on the worker that + # hosts it). These carry the deployment's worker conventions so they aren't hardcoded for + # non-deb topologies. worker_api_token_file: file holding the worker apiv2 Token (blank/absent + # => no auth header). worker_api_port: the worker's apiv2/web port. worker_ssh_user + + # worker_ssh_keyfile: the central node's libvirt-over-SSH identity onto workers. + worker_api_token_file: str = "/etc/cape/api-token" + worker_api_port: int = 8000 + worker_ssh_user: str = "cape" + worker_ssh_keyfile: str = "/home/cape/.ssh/id_ed25519" + # The report doc -> central DocumentDB write is the NATIVE mongodb.py reporting module + # pointed at DocumentDB via [mongodb] (tls=yes, retrywrites=no); central_mode therefore + # only carries the FS->S3 artifact location. + + +def _parse(sec) -> "CentralModeConfig": + """Pure: turn a [central_mode] config section (dict-like) into CentralModeConfig.""" + get = sec.get if hasattr(sec, "get") else (lambda k, d=None: d) + return CentralModeConfig( + enabled=_as_bool(get("enabled", False), False), + s3_bucket=str(get("s3_bucket", "") or ""), + s3_region=str(get("s3_region", "us-east-1") or "us-east-1"), + s3_prefix=str(get("s3_prefix", "results") or "results"), + storage_backend=str(get("storage_backend", "s3") or "s3").strip().lower(), + s3_endpoint_url=str(get("s3_endpoint_url", "") or ""), + s3_access_key=str(get("s3_access_key", "") or ""), + s3_secret_key=str(get("s3_secret_key", "") or ""), + central_local_root=str(get("central_local_root", "") or ""), + broker_table=str(get("broker_table", "") or ""), + job_directory=str(get("job_directory", "broker_http") or "broker_http").strip().lower(), + broker_url=str(get("broker_url", "") or ""), + broker_api_token=str(get("broker_api_token", "") or ""), + worker_api_token_file=str(get("worker_api_token_file", "/etc/cape/api-token") or "/etc/cape/api-token"), + worker_api_port=_as_port(get("worker_api_port", 8000), 8000), + worker_ssh_user=str(get("worker_ssh_user", "cape") or "cape"), + worker_ssh_keyfile=str(get("worker_ssh_keyfile", "/home/cape/.ssh/id_ed25519") or "/home/cape/.ssh/id_ed25519"), + ) + + +def central_mode_config() -> "CentralModeConfig": + # Lazy import so module import stays dependency-free (the CAPE config machinery + # is only touched when this is actually called at runtime). + from lib.cuckoo.common.config import Config + + try: + sec = Config("cuckoo").get("central_mode") + except Exception: + sec = {} + return _parse(sec) diff --git a/lib/cuckoo/common/hunt_query.py b/lib/cuckoo/common/hunt_query.py new file mode 100644 index 00000000000..804314ffe21 --- /dev/null +++ b/lib/cuckoo/common/hunt_query.py @@ -0,0 +1,27 @@ +"""Hunt aggregation builder — per-category $group (NO $facet, so it runs on +Amazon DocumentDB, which rejects $facet). Extracted from web/analysis/views.py +so it's unit-testable without importing the full web module graph; views.py +imports build_hunt_facets() and passes its mongo_aggregate in. + +Returns the same shape the old $facet pipeline produced: + {cat_id: [{"_id": , "count": , "task_ids": [...]}, ...]} +so the downstream clean_facets/validator loop is unchanged. +""" + + +def build_hunt_facets(mongo_aggregate, match, hunt_map, categories, min_count): + facets = {} + for cat_id, cat_config in hunt_map.items(): + if not categories.get(cat_id): + continue + stages = [{"$match": match}] + if cat_config.get("db_unwind"): + stages.append({"$unwind": cat_config["db_unwind"]}) + stages.extend([ + {"$group": {"_id": cat_config["db_group"], "count": {"$sum": 1}, "task_ids": {"$addToSet": "$info.id"}}}, + {"$match": cat_config.get("db_match", {"count": {"$gte": min_count}})}, + {"$sort": {"count": -1}}, + {"$limit": 100}, + ]) + facets[cat_id] = list(mongo_aggregate("analysis", stages)) + return facets diff --git a/lib/cuckoo/common/job_directory.py b/lib/cuckoo/common/job_directory.py new file mode 100644 index 00000000000..a93509b6f89 --- /dev/null +++ b/lib/cuckoo/common/job_directory.py @@ -0,0 +1,152 @@ +"""Pluggable job->worker directory for central-mode interactive routing. + +central_guac.py needs to resolve a LIVE task's broker job_id to the worker hosting its +VM — the worker's private IP + the worker-local cape_task_id — so the central node can +target that worker's guacd / libvirt for interactive Guacamole. The broker dispatcher +records this mapping when it pushes the job; in our AWS broker it lives in a DynamoDB +item keyed by job_id (sandbox_worker_ip, cape_task_id). That DynamoDB get_item was the +LAST hard AWS coupling in the CAPE fork's job path (the FS->S3 artifact seam is already +abstracted in storage_backend.py). + +This module pulls it behind a tiny vendor-neutral interface so a non-AWS deployment can +resolve the same mapping over the broker's HTTP status API (or any KV store) instead of +DynamoDB — keeping the fork AWS-free (AWS is one config value), while the AWS broker + +DynamoDB stay in the private IaC repo. The lookup result shape (worker_ip + cape_task_id) +is the only contract; everything else about how the broker stores it is a backend detail. +Django-free + unit-testable, like storage_backend.py. +""" +import logging + +log = logging.getLogger(__name__) + + +class JobLocation: + """Where a live broker job's VM is. worker_ip = the worker's private IP; cape_task_id + = the worker-local CAPE task id (used to query that worker's apiv2 for the VM). Either + may be None when the broker hasn't dispatched the job to a worker yet.""" + + __slots__ = ("worker_ip", "cape_task_id") + + def __init__(self, worker_ip=None, cape_task_id=None): + # "" / missing -> None so callers can do a simple truthiness check; keep + # cape_task_id as-is (it can legitimately be 0, distinct from absent None). + self.worker_ip = worker_ip or None + self.cape_task_id = cape_task_id + + def __eq__(self, other): + return ( + isinstance(other, JobLocation) + and self.worker_ip == other.worker_ip + and self.cape_task_id == other.cape_task_id + ) + + def __repr__(self): + return f"JobLocation(worker_ip={self.worker_ip!r}, cape_task_id={self.cape_task_id!r})" + + +def _valid_worker_ip(ip): + """The broker record's sandbox_worker_ip becomes the netloc of a qemu+ssh libvirt DSN AND of + an authenticated apiv2 URL (central_guac.py). A poisoned/spoofed broker record with a value + like '1.2.3.4/system?keyfile=/attacker/key&no_verify=1&x=' or 'attacker:9999/x?a=' would + otherwise inject libvirt URI params or redirect the Token-bearing apiv2 request to an attacker + host. It is knowably a bare IP, so require it to parse as one; anything else -> None (the caller + then keeps the localhost/single-node path). Validating here — the one normalizer both backends + and both consumers flow through — is the single choke point.""" + if not ip: + return None + import ipaddress + + text = str(ip).strip() + try: + ipaddress.ip_address(text) + except ValueError: + log.warning("job_directory: ignoring non-IP sandbox_worker_ip %r from broker record", ip) + return None + return text + + +def loc_from_item(item): + """Map a broker job record (dict from DynamoDB or the broker HTTP status API — both + use the SAME field names) to a JobLocation. Pure, so it's the unit-testable core both + backends share. The worker IP is validated (see _valid_worker_ip) before it reaches the + DSN/URL builders.""" + item = item or {} + return JobLocation(_valid_worker_ip(item.get("sandbox_worker_ip")), item.get("cape_task_id")) + + +class JobDirectory: + """Abstract job->worker resolver. Backends implement lookup(job_id).""" + + def lookup(self, job_id): + """Return a JobLocation for job_id, or None if it can't be resolved (network/ + config error, or no such job). worker_ip/cape_task_id inside may still be None + if the broker hasn't dispatched the job yet.""" + raise NotImplementedError + + +class DynamoJobDirectory(JobDirectory): + """Read the broker's DynamoDB job item directly (default for our AWS broker — keeps + today's [central_mode] broker_table behavior byte-for-byte).""" + + def __init__(self, table, region="us-east-1"): + self.table = table + self.region = region + + def lookup(self, job_id): + try: + import boto3 + + item = ( + boto3.resource("dynamodb", region_name=self.region) + .Table(self.table) + .get_item(Key={"job_id": job_id}) + .get("Item", {}) + ) + except Exception as e: + log.warning("job_directory(dynamodb): lookup failed for %s: %s", job_id, e) + return None + return loc_from_item(item) + + +class BrokerHttpJobDirectory(JobDirectory): + """Resolve via the broker's HTTP status API — vendor-neutral (no DynamoDB / boto3 in + the fork). The broker's GET {broker_url}/api/status/{job_id} returns the same job + record (sandbox_worker_ip + cape_task_id). Bearer-auth with the broker API token.""" + + def __init__(self, broker_url, token=""): + self.broker_url = (broker_url or "").rstrip("/") + self.token = token or "" + + def lookup(self, job_id): + if not self.broker_url: + return None + try: + import requests + + headers = {"Authorization": f"Bearer {self.token}"} if self.token else {} + r = requests.get(f"{self.broker_url}/api/status/{job_id}", headers=headers, timeout=10) + if r.status_code != 200: + return None + item = r.json() or {} + except Exception as e: + log.warning("job_directory(broker_http): lookup failed for %s: %s", job_id, e) + return None + return loc_from_item(item) + + +def get_job_directory(cfg): + """Return a JobDirectory for the central-mode config, or None when central mode is off + or no directory is configured — in which case central_guac's callers keep their + single-node/localhost path unchanged. Default backend is 'broker_http' (vendor-neutral — + resolves via the broker's HTTP API); 'dynamodb' (AWS) is opt-in.""" + if not getattr(cfg, "enabled", False): + return None + backend = (getattr(cfg, "job_directory", "") or "broker_http").strip().lower() + if backend == "broker_http": + if not getattr(cfg, "broker_url", ""): + return None + return BrokerHttpJobDirectory(cfg.broker_url, getattr(cfg, "broker_api_token", "")) + # 'dynamodb' (opt-in, AWS) — None if broker_table unset (matches the pre-abstraction gate). + if not getattr(cfg, "broker_table", ""): + return None + return DynamoJobDirectory(cfg.broker_table, getattr(cfg, "s3_region", "us-east-1")) diff --git a/lib/cuckoo/common/storage_backend.py b/lib/cuckoo/common/storage_backend.py new file mode 100644 index 00000000000..79547e10a86 --- /dev/null +++ b/lib/cuckoo/common/storage_backend.py @@ -0,0 +1,232 @@ +"""Pluggable artifact-storage backend for central mode. + +The central-mode artifact seam (lib/cuckoo/common/artifact_storage.py read side, +modules/reporting/centralstore.py write side) used boto3's S3 client with AWS defaults +directly — the ONLY hard AWS coupling in the central-mode CAPE code. This module pulls +that behind a small vendor-neutral interface so central mode runs on ANY S3-compatible +object store (AWS S3, MinIO, Ceph RGW, …) or a shared local/NFS mount, with the endpoint +and credentials supplied by config. Nothing AWS-shaped lives here — bucket/endpoint/region/ +creds are values in [central_mode], injected at runtime. + +A `container` is the per-analysis root (single-node: the storage/analyses/ dir; +central: the "/" key prefix); `relpath` is the artifact path within it +(the caller validates it with _safe_relpath before passing it here). The stores are +Django-free — they return (byte-iterator, length) and the caller builds the HTTP response — +so this module is portable + unit-testable without the web stack. +""" +import os +import shutil +import tempfile + + +class ArtifactNotFound(Exception): + """Raised by stream()/read bytes when an artifact key is absent.""" + + +class ArtifactStore: + """Abstract per-analysis object store. Backends implement raw object ops keyed by + (container, relpath).""" + + def exists(self, container, relpath) -> bool: + raise NotImplementedError + + def stream(self, container, relpath, chunk=8192): + """Return (byte_iterator, content_length_or_None). Raise ArtifactNotFound if absent.""" + raise NotImplementedError + + def read_text(self, container, relpath, max_bytes): + """Return up to max_bytes of text (utf-8, errors=replace), or "" if absent.""" + raise NotImplementedError + + def materialize(self, container, relpath): + """Return (local_path, is_temp) for a caller that needs a real file (random access, + external tool). is_temp=True means the caller must delete it. (None, False) if absent.""" + raise NotImplementedError + + def iter_relpaths(self, container): + """Yield each artifact relpath present under `container` (for staging).""" + raise NotImplementedError + + def download(self, container, relpath, dest_abspath): + """Copy one artifact to dest_abspath (creates parent dirs). Best-effort raise on error.""" + raise NotImplementedError + + def put_file(self, local_path, container, relpath): + """Write a local file into the store at (container, relpath).""" + raise NotImplementedError + + +def _iter_file(path, chunk): + with open(path, "rb") as f: + while True: + data = f.read(chunk) + if not data: + break + yield data + + +class LocalFSStore(ArtifactStore): + """Filesystem backend. Used for single-node (base=storage/analyses, container=) + AND for a central deployment on a shared local/NFS mount (base=central_local_root, + container="/").""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + def _path(self, container, relpath): + return os.path.join(self.base_dir, str(container), relpath) + + def exists(self, container, relpath) -> bool: + return os.path.exists(self._path(container, relpath)) + + def stream(self, container, relpath, chunk=8192): + path = self._path(container, relpath) + if not os.path.exists(path): + raise ArtifactNotFound(relpath) + return _iter_file(path, chunk), os.path.getsize(path) + + def read_text(self, container, relpath, max_bytes): + path = self._path(container, relpath) + if not os.path.exists(path): + return "" + with open(path, "r", errors="replace") as f: + return f.read(max_bytes + 1) + + def materialize(self, container, relpath): + path = self._path(container, relpath) + return (path, False) if os.path.exists(path) else (None, False) + + def iter_relpaths(self, container): + root = self._path(container, "") + if not os.path.isdir(root): + return + for dirpath, _dirs, files in os.walk(root): + for fn in files: + yield os.path.relpath(os.path.join(dirpath, fn), root) + + def download(self, container, relpath, dest_abspath): + src = self._path(container, relpath) + os.makedirs(os.path.dirname(dest_abspath), exist_ok=True) + if os.path.realpath(src) != os.path.realpath(dest_abspath): + shutil.copyfile(src, dest_abspath) + + def put_file(self, local_path, container, relpath): + dest = self._path(container, relpath) + dest_dir = os.path.dirname(dest) + os.makedirs(dest_dir, exist_ok=True) + if os.path.realpath(local_path) == os.path.realpath(dest): + return + # Write to a temp file in the SAME dir (same filesystem) then atomically rename, so a + # concurrent reader (the central read seam staging from a shared/NFS mount) never observes + # a half-written object at the final key. A direct copyfile would expose partial content. + fd, tmp = tempfile.mkstemp(dir=dest_dir, prefix=".part-") + os.close(fd) + try: + shutil.copyfile(local_path, tmp) + os.replace(tmp, dest) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +class S3Store(ArtifactStore): + """Any S3-compatible object store via boto3. endpoint_url/creds are optional: empty + endpoint_url -> AWS's default endpoint; empty creds -> boto3's default chain (IAM role + on AWS). Set them to point at MinIO/Ceph/etc.""" + + def __init__(self, bucket, region="us-east-1", endpoint_url="", access_key="", secret_key=""): + self.bucket = bucket + self.region = region + self.endpoint_url = endpoint_url or None + self.access_key = access_key or None + self.secret_key = secret_key or None + self._cli = None + + def _client(self): + if self._cli is None: + import boto3 + + kwargs = {"region_name": self.region} + if self.endpoint_url: + kwargs["endpoint_url"] = self.endpoint_url + if self.access_key and self.secret_key: + kwargs["aws_access_key_id"] = self.access_key + kwargs["aws_secret_access_key"] = self.secret_key + self._cli = boto3.client("s3", **kwargs) + return self._cli + + def _key(self, container, relpath): + return f"{container}/{relpath}" + + def exists(self, container, relpath) -> bool: + try: + self._client().head_object(Bucket=self.bucket, Key=self._key(container, relpath)) + return True + except Exception: + return False + + def stream(self, container, relpath, chunk=8192): + try: + obj = self._client().get_object(Bucket=self.bucket, Key=self._key(container, relpath)) + except Exception: + raise ArtifactNotFound(relpath) + return obj["Body"].iter_chunks(chunk), obj.get("ContentLength") + + def read_text(self, container, relpath, max_bytes): + try: + obj = self._client().get_object( + Bucket=self.bucket, Key=self._key(container, relpath), Range=f"bytes=0-{max_bytes}" + ) + return obj["Body"].read().decode("utf-8", errors="replace") + except Exception: + return "" + + def materialize(self, container, relpath): + try: + obj = self._client().get_object(Bucket=self.bucket, Key=self._key(container, relpath)) + except Exception: + return (None, False) + fd, tmp = tempfile.mkstemp(prefix="cape_central_") + try: + with os.fdopen(fd, "wb") as f: + for c in obj["Body"].iter_chunks(65536): + f.write(c) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + return (None, False) + return (tmp, True) + + def iter_relpaths(self, container): + prefix = f"{container}/" + paginator = self._client().get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for obj in page.get("Contents", []): + rel = obj["Key"][len(prefix):] + if rel and not rel.endswith("/"): + yield rel + + def download(self, container, relpath, dest_abspath): + os.makedirs(os.path.dirname(dest_abspath), exist_ok=True) + self._client().download_file(self.bucket, self._key(container, relpath), dest_abspath) + + def put_file(self, local_path, container, relpath): + self._client().upload_file(local_path, self.bucket, self._key(container, relpath)) + + +def get_artifact_store(cfg): + """Return (store, is_central) for the central-mode config. Single-node (cfg.enabled + False) -> LocalFSStore over storage/analyses. Central -> S3Store (default) or LocalFSStore + when storage_backend='local' + a central_local_root is set.""" + from lib.cuckoo.common.constants import CUCKOO_ROOT + + if not cfg.enabled: + return LocalFSStore(os.path.join(CUCKOO_ROOT, "storage", "analyses")), False + if cfg.storage_backend == "local" and cfg.central_local_root: + return LocalFSStore(cfg.central_local_root), True + return S3Store(cfg.s3_bucket, cfg.s3_region, cfg.s3_endpoint_url, cfg.s3_access_key, cfg.s3_secret_key), True diff --git a/lib/cuckoo/common/tenancy_optional.py b/lib/cuckoo/common/tenancy_optional.py new file mode 100644 index 00000000000..5c952684874 --- /dev/null +++ b/lib/cuckoo/common/tenancy_optional.py @@ -0,0 +1,100 @@ +"""Import-optional facade for the lib-level MT symbols (lib.cuckoo.common.tenancy). + +Delegates to the real MT module when it's importable; when it ISN'T (the MT layer is not +deployed — e.g. an upstream central-only build), returns values IDENTICAL to the MT-disabled +code path (which the real functions already implement: viewer_for -> is_local_admin=True -> +every can_* gate True; scope_match -> None). FAIL-CLOSED: catch ImportError ONLY; a runtime +error from a deployed MT layer propagates rather than silently degrading to see-all. +""" +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Viewer: + """FALLBACK viewer, returned by viewer_for() ONLY when the MT layer is absent — see-all + (is_local_admin=True). When MT IS deployed, viewer_for() returns the REAL + lib.cuckoo.common.tenancy.Viewer, a DIFFERENT class. So do NOT + `isinstance(viewer_for(u), Viewer)` against this type — it's False in production. Treat + viewer_for()'s result structurally (.is_local_admin / .tenant_id), never by type.""" + user_id: int = 0 + tenant_id: int = 0 + is_tenant_admin: bool = False + is_local_admin: bool = True + + +@dataclass(frozen=True) +class MTConfig: + """FALLBACK config, returned by multitenancy_config() ONLY when the MT layer is absent. + When MT is deployed the REAL lib.cuckoo.common.tenancy.MTConfig (a different class) is + returned — don't isinstance-check against this; read .enabled / .mode structurally.""" + enabled: bool = False + mode: str = "shared" + default_visibility: str = "" + local_admins_manage_all_tenants: bool = True + + +def multitenancy_config(): + try: + from lib.cuckoo.common.tenancy import multitenancy_config as real + except ImportError: + return MTConfig() + return real() + + +def viewer_for(user): + # viewer_for lives in the web `users` MT app (needs the Django ORM), NOT the pure predicate + # lib.cuckoo.common.tenancy -- importing it from there always ImportError'd -> the facade + # silently degraded to see-all even when MT WAS deployed (cross-tenant leak). Resolve it from + # users.tenancy (present => real viewer; absent/non-Django => fall back to see-all). + try: + from users.tenancy import viewer_for as real + except ImportError: + return Viewer() + return real(user) + + +def scope_match(scope, viewer): + try: + from lib.cuckoo.common.tenancy import scope_match as real + except ImportError: + return None + return real(scope, viewer) + + +# Visibility constants — stable contract values. Present -> the real module's values; +# absent -> the same literals so callers comparing/storing visibility still work. +try: + from lib.cuckoo.common.tenancy import PUBLIC, PRIVATE, TENANT, VISIBILITIES # noqa: F401 +except ImportError: + PUBLIC, TENANT, PRIVATE = "public", "tenant", "private" + VISIBILITIES = (PUBLIC, TENANT, PRIVATE) + + +def default_visibility(cfg): + """Submit-time default visibility for the configured mode (see-all path -> PUBLIC).""" + try: + from lib.cuckoo.common.tenancy import default_visibility as real + except ImportError: + if getattr(cfg, "default_visibility", "") in VISIBILITIES: + return cfg.default_visibility + return PUBLIC if getattr(cfg, "mode", "shared") == "shared" else TENANT + return real(cfg) + + +def viewer_scope_match(viewer): + """Mongo $match restricting a query to the viewer's entitled scopes, or None (no + filter / see-all) when MT is disabled or the layer is absent.""" + try: + from lib.cuckoo.common.tenancy import viewer_scope_match as real + except ImportError: + return None + return real(viewer) + + +def viewer_scope_es_filter(viewer): + """Elasticsearch bool-filter analogue of viewer_scope_match, or None (see-all).""" + try: + from lib.cuckoo.common.tenancy import viewer_scope_es_filter as real + except ImportError: + return None + return real(viewer) diff --git a/modules/reporting/centralstore.py b/modules/reporting/centralstore.py new file mode 100644 index 00000000000..f5dca6cc656 --- /dev/null +++ b/modules/reporting/centralstore.py @@ -0,0 +1,271 @@ +"""centralstore — worker-side ingest seam for central mode. + +central_mode OFF (default): no-op. The worker behaves byte-for-byte single-node. + +central_mode ON: stamp the broker-supplied global job_id into results.info.job_id +and push the analysis artifact tree to S3 at //. Runs at +order 9998 — BEFORE the native mongodb reporting module (order 9999) — so the +report doc that mongodb.py writes to the central DocumentDB already carries +info.job_id. The DocumentDB write itself is the NATIVE mongodb.py path pointed at +DocumentDB via [mongodb] (tls=yes, retrywrites=no) — validated against live DocumentDB +(loop_saver/$set, calls chunking, files $addToSet, tenant_scope_idx). This module only adds the FS->S3 half +plus the job_id keying the read seam (artifact_storage.artifact_response) resolves. +""" +import logging +import os +import re + +from lib.cuckoo.common.abstracts import Report +from lib.cuckoo.common.central_mode import central_mode_config, upload_target_realpath +from lib.cuckoo.common.exceptions import CuckooReportError +from lib.cuckoo.common.storage_backend import get_artifact_store + +log = logging.getLogger(__name__) + +# job_id becomes an S3 key segment, so it must not contain path separators or +# traversal. The broker should stamp an authenticated job_id; this is the last +# line of defence against a tenant-supplied `custom` poisoning another job's +# prefix (audit CRITICAL-1). local- fallback satisfies the allowlist. +# Must start with an alnum (no leading '.'/'-'/'_') AND contain no '..' run, so a +# value like '.', '..', '.foo' or 'a..b' can never collapse 'results//' to a +# parent ref ('results/../') in an S3 key or the local staging path. _is_safe_job_id +# applies both rules (the regex alone permitted '.'/'..'). +_JOB_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") + + +def _is_safe_job_id(job_id): + return bool(job_id) and _JOB_ID_RE.match(job_id) is not None and ".." not in job_id + +# Upload the whole analysis tree to S3 (the "heavy detail" tier): shots, dropped +# files, pcap, procdump, AND reports/ (report.json/html/pdf are downloadable +# artifacts the filereport view serves). The DocumentDB report doc written by the +# native mongodb.py module is ADDITIVE (it powers the queryable UI tabs) — it does +# not replace the report files, so nothing is excluded here. +_EXCLUDE_DIRS = set() + + +def resolve_job_id(custom, analysis_id): + """The broker passes the global job_id through the task `custom` field, carried + all the way to reporting. Accept 'job_id=' (optionally among other + comma-separated k=v pairs) or a bare token. Fall back to 'local-' so central + mode also works when an analysis was submitted directly (no broker).""" + if custom: + text = str(custom) + for part in text.split(","): + part = part.strip() + if part.startswith("job_id="): + v = part.split("=", 1)[1].strip() + if v: + return v + token = text.strip() + if token and "=" not in token and "," not in token: + return token + return f"local-{analysis_id}" + + +class CentralStore(Report): + """Ship analysis artifacts to S3 + stamp the central job_id (central mode only).""" + + order = 9998 # before mongodb (9999): stamp info.job_id before the central DocumentDB write + + def run(self, results): + cfg = central_mode_config() + if not cfg.enabled: + return # single-node: no-op, behavior byte-for-byte unchanged + + store, _is_central = get_artifact_store(cfg) + # Validate the configured backend's prerequisites up front so a misconfig fails + # with one clean CuckooReportError, not N per-file upload failures (which would + # leave the job retained-but-never-done). The S3 path (default storage_backend) + # needs a bucket + boto3; the shared-mount path (storage_backend=local + + # central_local_root) needs neither. + using_local_mount = cfg.storage_backend == "local" and bool(cfg.central_local_root) + if not using_local_mount: + if not cfg.s3_bucket: + raise CuckooReportError("centralstore: central_mode enabled but [central_mode] s3_bucket unset") + try: + import boto3 # noqa: F401 + except ImportError as e: + raise CuckooReportError("centralstore: central mode requires boto3") from e + + info = results.setdefault("info", {}) + analysis_id = info.get("id") + job_id = info.get("job_id") or resolve_job_id(info.get("custom"), analysis_id) + if not _is_safe_job_id(job_id): + # Refuse a job_id that could escape/poison another job's S3 prefix. + raise CuckooReportError( + "centralstore: refusing unsafe job_id %r (must match %s and contain no '..')" + % (job_id, _JOB_ID_RE.pattern)) + info["job_id"] = job_id # carried into the DocumentDB doc; read seam keys S3 by it + + # Align info.id to the CENTRAL task id when the broker assigned a + # "ui-" job_id (the central-submit-bridge does this). The + # worker's local info.id is a per-worker sequence that collides across workers + # and is meaningless to the central UI, which addresses every analysis by its + # OWN task id. Rewriting info.id here (before mongodb.py at 9999 writes the doc) + # makes the central DocumentDB doc resolvable by the central task id natively — + # so the report view, all report-tab lookups, and the artifact seam work without + # touching ~25 info.id call sites in the upstream-synced views.py. + _m = re.match(r"^ui-(\d+)$", job_id) + if _m: + info["id"] = int(_m.group(1)) + + # The per-analysis container the read seam resolves to (artifact_storage. + # _store_and_container builds the same "/"). The raw object I/O + # is the pluggable store (S3-compatible or shared mount); the symlink-exfil + # guard + job_id keying stay here. + container = f"{cfg.s3_prefix}/{job_id}" + uploaded, failed = self._upload_tree(store, container) + # Guac recordings live outside the analysis tree (top-level storage/ + # guacrecordings/), so they need their own pass. Fold their counts in so the + # done marker — the cleanup purge gate — only fires once EVERYTHING this job + # produced (tree + binaries + any recording) is confirmed in the central store. + rec_up, rec_failed = self._upload_guacrecordings(store, container, analysis_id) + uploaded += rec_up + failed += rec_failed + log.info("centralstore: uploaded %d artifacts (%d recordings) to %s/ (%d failed)", + uploaded, rec_up, container, failed) + + # Stamp a local marker ONLY when the whole tree is confirmed in S3. The worker's + # cape-nvme-cleanup gate keys off this file: present => artifacts are durable in + # the central stores => the local (ephemeral-NVMe) copy is safe to purge. On any + # upload failure we leave no marker, so the analysis is retained until it is + # re-confirmed or the worker recycles (24h) — never purged unconfirmed. + if failed == 0: + _emit_done_marker(store, container, self.analysis_path, cfg, job_id, uploaded) + else: + log.warning("centralstore: %d upload(s) failed for job_id=%s; NOT marking done " + "(local copy retained for cleanup safety)", failed, job_id) + + def _trusted_roots(self, base_real): + """Content roots a sample-influenced symlink may legitimately resolve into: + storage/binaries (the content-addressed sample/dropped-file store the analysis + dir's `binary` symlink targets) and storage/guacrecordings. base_real is + .../storage/analyses/, so the storage root is two levels up.""" + storage_root = os.path.dirname(os.path.dirname(base_real)) + return [ + os.path.realpath(os.path.join(storage_root, "binaries")), + os.path.realpath(os.path.join(storage_root, "guacrecordings")), + ] + + def _upload_tree(self, store, container): + base = self.analysis_path + if not base or not os.path.isdir(base): + log.warning("centralstore: analysis_path missing or not a dir: %s", base) + return 0, 0 + base_real = os.path.realpath(base) + trusted_roots = self._trusted_roots(base_real) + count = 0 + failed = 0 + for root, dirs, files in os.walk(base): + # don't descend symlinked dirs; drop excluded dirs + dirs[:] = [d for d in dirs if d not in _EXCLUDE_DIRS and not os.path.islink(os.path.join(root, d))] + for fn in files: + full = os.path.join(root, fn) + # A regular in-tree file uploads itself; a symlink into a trusted + # content root (binary -> storage/binaries/, a recording -> + # storage/guacrecordings/) uploads its RESOLVED content under the + # analysis-relative key. Anything resolving elsewhere (a planted + # symlink to e.g. ~/.aws/credentials) returns None and is skipped — + # the artifact-exfil guard stays intact (audit CRITICAL). + src = upload_target_realpath(full, base_real, trusted_roots) + if src is None: + log.warning("centralstore: skipping out-of-tree/untrusted artifact %s", full) + continue + rel = os.path.relpath(full, base) + try: + store.put_file(src, container, rel) + count += 1 + except Exception as e: + failed += 1 + log.warning("centralstore: failed to upload %s -> %s/%s: %s", rel, container, rel, e) + return count, failed + + def _upload_guacrecordings(self, store, container, task_id): + """Ship Guacamole session recordings for this task. Recordings live in the + top-level storage/guacrecordings/ (NOT inside the analysis tree, so the walk + above never sees them) and are named '_' by the guac + consumer. They only exist when an analyst live-viewed the VM mid-detonation, + so this is usually a no-op; when present they upload under / + guacrecordings/ so the central store has them before the worker purges. + """ + base_real = os.path.realpath(self.analysis_path) + rec_root = os.path.join(os.path.dirname(os.path.dirname(base_real)), "guacrecordings") + if not task_id or not os.path.isdir(rec_root): + return 0, 0 + count = 0 + failed = 0 + prefix_match = f"{task_id}_" + for fn in os.listdir(rec_root): + # exact '' or '_' — never a different task that + # merely shares a leading digit (e.g. task 1 vs 15): require '_' boundary. + if fn != str(task_id) and not fn.startswith(prefix_match): + continue + full = os.path.join(rec_root, fn) + if os.path.islink(full) or not os.path.isfile(full): + continue + rel = f"guacrecordings/{fn}" + try: + store.put_file(full, container, rel) + count += 1 + except Exception as e: + failed += 1 + log.warning("centralstore: failed to upload recording %s -> %s/%s: %s", fn, container, rel, e) + return count, failed + + +def _emit_done_marker(store, container, analysis_path, cfg, job_id, uploaded): + """Emit the analysis completion marker, in TWO places, once the whole tree is confirmed: + + 1. LOCAL — storage/analyses//.centralstore.done. The worker's cape-nvme-cleanup purges + only analyses carrying this file, so it can never delete a job that didn't reach the + central store. + 2. CENTRAL STORE — uploaded as the FINAL object at /.centralstore.done. The READ + seam (artifact_storage._stage_tree) treats this key as the completion signal and only then + caches .central_staged; WITHOUT the store copy `complete` is never True, so every central + report view re-stages the entire tree from the store on every request. Uploaded last (and + only when failed==0 upstream) so a listing taken mid-upload never shows the marker before + the tree is complete. + + Best-effort: a marker failure never breaks reporting — the artifacts are already durable. If + the local write fails there's nothing to upload; if the store upload fails the local gate + still stands (cleanup stays safe) and the read side falls back to the per-file download seam.""" + import json + import time + + if not analysis_path or not os.path.isdir(analysis_path): + return + marker = os.path.join(analysis_path, ".centralstore.done") + # Backend-aware location string: s3://bucket/... for the S3 path, or the shared + # mount's /// for the local path. Informational only. + if cfg.storage_backend == "local" and cfg.central_local_root: + location = os.path.join(cfg.central_local_root, cfg.s3_prefix, job_id) + os.sep + else: + location = "s3://%s/%s/%s/" % (cfg.s3_bucket, cfg.s3_prefix, job_id) + try: + with open(marker, "w") as f: + json.dump({ + "job_id": job_id, + "location": location, + "artifacts": uploaded, + "ts": time.time(), + }, f) + except Exception as e: + log.warning("centralstore: could not write local done marker %s: %s", marker, e) + return # nothing to upload if the local marker couldn't even be written + # This is the SINGLE object that gates the read-side staging cache (artifact_storage. + # _stage_tree). Reporting runs once and won't re-emit it, and the worker's cleanup gate (the + # local marker) will let the ephemeral NVMe copy be purged — so a transient store hiccup here + # would permanently disable that job's read cache with no self-heal. Retry a few times before + # giving up; the tree itself is already durable, so this stays best-effort even on total failure. + for attempt in range(3): + try: + store.put_file(marker, container, ".centralstore.done") + return + except Exception as e: + log.warning("centralstore: marker upload attempt %d/3 to %s/.centralstore.done failed: %s", + attempt + 1, container, e) + if attempt < 2: + time.sleep(0.25 * (attempt + 1)) + log.warning("centralstore: gave up uploading %s/.centralstore.done after 3 attempts " + "(read-side staging cache disabled for this job; artifacts still served per-file)", container) diff --git a/tests/test_central_mode.py b/tests/test_central_mode.py new file mode 100644 index 00000000000..d4da0cc55d8 --- /dev/null +++ b/tests/test_central_mode.py @@ -0,0 +1,416 @@ +"""Pure-logic unit tests for central mode — no Django / pymongo / CAPE web imports +(the web layer is iterated live on a CAPE box). Run: pytest tests/test_central_mode.py""" +import os + +import pytest + +from lib.cuckoo.common.central_mode import _parse, _as_bool, _as_int, _as_port, upload_target_realpath +from lib.cuckoo.common.central_guac import ( + _libvirt_ssh_dsn, + _worker_api_token, + _worker_task_view_url, +) +from lib.cuckoo.common.hunt_query import build_hunt_facets +from lib.cuckoo.common.storage_backend import ( + ArtifactNotFound, + LocalFSStore, + S3Store, + get_artifact_store, +) +from lib.cuckoo.common.job_directory import ( + BrokerHttpJobDirectory, + DynamoJobDirectory, + JobLocation, + get_job_directory, + loc_from_item, +) + + +def test_as_bool(): + assert _as_bool("yes") is True + assert _as_bool("on") is True + assert _as_bool("no") is False + assert _as_bool(None, False) is False + assert _as_bool(True) is True + + +def test_as_int(): + assert _as_int("42", 0) == 42 + assert _as_int(7, 0) == 7 + assert _as_int(" 9443 ", 0) == 9443 # stripped + assert _as_int(None, 8000) == 8000 + assert _as_int("notaport", 8000) == 8000 # garbage -> default, no crash + + +def test_as_port(): + assert _as_port("9443", 8000) == 9443 + assert _as_port(443, 8000) == 443 + # int() accepts these but they're not real ports -> fall back to the default (not a broken URL) + assert _as_port("0", 8000) == 8000 + assert _as_port("-1", 8000) == 8000 + assert _as_port("99999", 8000) == 8000 + assert _as_port("nope", 8000) == 8000 + + +def test_central_mode_defaults_off(): + c = _parse({}) + assert c.enabled is False + assert c.s3_bucket == "" + assert c.s3_prefix == "results" + + +def test_central_mode_on(): + c = _parse({"enabled": "yes", "s3_bucket": "b", "s3_region": "us-east-1", "s3_prefix": "results"}) + assert c.enabled is True + assert c.s3_bucket == "b" + + +def test_central_mode_backend_fields_parse(): + # The new backend-selection fields (the only thing that made the store AWS-locked) + # parse from [central_mode], with sane defaults when absent. + d = _parse({}) + assert d.storage_backend == "s3" + assert d.s3_endpoint_url == "" and d.s3_access_key == "" and d.s3_secret_key == "" + assert d.central_local_root == "" + c = _parse({ + "enabled": "yes", + "storage_backend": "LOCAL", # normalized to lower + "s3_endpoint_url": "https://minio.local:9000", + "s3_access_key": "ak", + "s3_secret_key": "sk", + "central_local_root": "/srv/cape-central", + }) + assert c.storage_backend == "local" + assert c.s3_endpoint_url == "https://minio.local:9000" + assert c.s3_access_key == "ak" and c.s3_secret_key == "sk" + assert c.central_local_root == "/srv/cape-central" + + +def test_get_artifact_store_single_node_is_local_fs(): + store, is_central = get_artifact_store(_parse({})) + assert is_central is False + assert isinstance(store, LocalFSStore) + # single-node always reads the local storage/analyses tree + assert store.base_dir.endswith(os.path.join("storage", "analyses")) + + +def test_get_artifact_store_central_s3(): + store, is_central = get_artifact_store( + _parse({"enabled": "yes", "s3_bucket": "bkt", "s3_region": "eu-west-1"}) + ) + assert is_central is True + assert isinstance(store, S3Store) + assert store.bucket == "bkt" and store.region == "eu-west-1" + # endpoint/creds default to None so boto3 uses AWS's endpoint + default cred chain + assert store.endpoint_url is None and store.access_key is None and store.secret_key is None + + +def test_get_artifact_store_central_minio_endpoint(): + store, _ = get_artifact_store( + _parse({ + "enabled": "yes", "s3_bucket": "bkt", + "s3_endpoint_url": "https://minio.local:9000", + "s3_access_key": "ak", "s3_secret_key": "sk", + }) + ) + assert isinstance(store, S3Store) + assert store.endpoint_url == "https://minio.local:9000" + assert store.access_key == "ak" and store.secret_key == "sk" + + +def test_get_artifact_store_central_local_mount(tmp_path): + root = str(tmp_path / "central") + store, is_central = get_artifact_store( + _parse({"enabled": "yes", "storage_backend": "local", "central_local_root": root}) + ) + assert is_central is True + assert isinstance(store, LocalFSStore) + assert store.base_dir == root + + +def test_get_artifact_store_local_backend_without_root_falls_back_to_s3(): + # storage_backend=local but no central_local_root is a misconfig; it must NOT + # silently read the single-node tree — it falls through to the S3 store (whose + # missing-bucket prereq centralstore validates up front). + store, is_central = get_artifact_store( + _parse({"enabled": "yes", "storage_backend": "local", "s3_bucket": "bkt"}) + ) + assert is_central is True + assert isinstance(store, S3Store) + + +def test_localfsstore_roundtrip(tmp_path): + # LocalFSStore backs BOTH single-node and central-on-a-shared-mount, so its + # round-trip is the contract the read+write seams depend on. + base = str(tmp_path / "base") + store = LocalFSStore(base) + container = "results/job-1" + + src = tmp_path / "src.txt" + src.write_text("hello world") + store.put_file(str(src), container, "reports/report.json") + + assert store.exists(container, "reports/report.json") is True + assert store.exists(container, "missing") is False + + body_iter, length = store.stream(container, "reports/report.json") + assert b"".join(body_iter) == b"hello world" + assert length == len("hello world") + + assert store.read_text(container, "reports/report.json", 1000) == "hello world" + assert store.read_text(container, "missing", 1000) == "" + + path, is_temp = store.materialize(container, "reports/report.json") + assert is_temp is False and os.path.exists(path) + assert store.materialize(container, "missing") == (None, False) + + assert list(store.iter_relpaths(container)) == ["reports/report.json"] + + dest = tmp_path / "out" / "copy.json" + store.download(container, "reports/report.json", str(dest)) + assert dest.read_text() == "hello world" + + +def test_localfsstore_stream_missing_raises(tmp_path): + store = LocalFSStore(str(tmp_path)) + with pytest.raises(ArtifactNotFound): + store.stream("results/job-x", "nope") + + +def test_s3store_is_lazy_no_client_on_construct(): + # Constructing the store must NOT build a boto3 client (so import/config never + # touches the network); the client is built on first op. + store = S3Store("bkt", "us-east-1") + assert store._cli is None + + +# ---- Phase 2: job->worker directory (interactive guac routing) ---- + +def test_central_mode_job_directory_fields_parse(): + d = _parse({}) + assert d.job_directory == "broker_http" # default backend (vendor-neutral) + assert d.broker_url == "" and d.broker_api_token == "" + c = _parse({ + "enabled": "yes", + "job_directory": "BROKER_HTTP", # normalized to lower + "broker_url": "https://broker.local", + "broker_api_token": "tok", + }) + assert c.job_directory == "broker_http" + assert c.broker_url == "https://broker.local" and c.broker_api_token == "tok" + + +def test_loc_from_item_maps_and_normalizes(): + # the broker record (DynamoDB item OR broker HTTP status body) -> JobLocation + loc = loc_from_item({"sandbox_worker_ip": "10.0.0.5", "cape_task_id": 42, "status": "running"}) + assert loc == JobLocation("10.0.0.5", 42) + # cape_task_id may legitimately be 0 (distinct from absent); worker_ip "" -> None + assert loc_from_item({"sandbox_worker_ip": "", "cape_task_id": 0}) == JobLocation(None, 0) + assert loc_from_item({}) == JobLocation(None, None) + assert loc_from_item(None) == JobLocation(None, None) + # IPv6 is a valid IP too (kept as-is) + assert loc_from_item({"sandbox_worker_ip": "fd00::1", "cape_task_id": 5}) == JobLocation("fd00::1", 5) + + +def test_loc_from_item_rejects_non_ip_worker_ip(): + # sandbox_worker_ip becomes the netloc of a libvirt DSN + an authenticated apiv2 URL. A poisoned + # broker record must NOT flow through: a non-IP value (injection payload, hostname, garbage) is + # dropped to None so the caller keeps the localhost path — closes the DSN/URL-injection vector. + payload = "1.2.3.4/system?keyfile=/attacker/key&no_verify=1&x=" + assert loc_from_item({"sandbox_worker_ip": payload, "cape_task_id": 7}) == JobLocation(None, 7) + assert loc_from_item({"sandbox_worker_ip": "attacker.host:9999/x?a=", "cape_task_id": 1}) == JobLocation(None, 1) + assert loc_from_item({"sandbox_worker_ip": "not-an-ip", "cape_task_id": 2}) == JobLocation(None, 2) + # cape_task_id survives even when the IP is rejected (the job exists; it's just unroutable here) + assert loc_from_item({"sandbox_worker_ip": "999.999.999.999", "cape_task_id": 0}) == JobLocation(None, 0) + + +def test_get_job_directory_off_returns_none(): + assert get_job_directory(_parse({})) is None # central mode off + # enabled but default backend (broker_http) with no broker_url -> None (caller keeps localhost) + assert get_job_directory(_parse({"enabled": "yes"})) is None + + +def test_get_job_directory_default_is_broker_http(): + # default backend is now broker_http (vendor-neutral); with a broker_url it resolves to BrokerHttp + d = get_job_directory(_parse({"enabled": "yes", "broker_url": "https://broker.local"})) + assert isinstance(d, BrokerHttpJobDirectory) + + +def test_get_job_directory_dynamodb_explicit(): + # dynamodb is opt-in (AWS); must be selected explicitly now + d = get_job_directory(_parse({"enabled": "yes", "job_directory": "dynamodb", "broker_table": "tbl", "s3_region": "eu-west-1"})) + assert isinstance(d, DynamoJobDirectory) + assert d.table == "tbl" and d.region == "eu-west-1" + + +def test_get_job_directory_broker_http(): + d = get_job_directory(_parse({ + "enabled": "yes", "job_directory": "broker_http", + "broker_url": "https://broker.local/", "broker_api_token": "tok", + })) + assert isinstance(d, BrokerHttpJobDirectory) + assert d.broker_url == "https://broker.local" # trailing slash stripped + assert d.token == "tok" + + +def test_get_job_directory_broker_http_without_url_returns_none(): + # broker_http selected but no broker_url -> None (caller keeps localhost path) + assert get_job_directory(_parse({"enabled": "yes", "job_directory": "broker_http"})) is None + + +def test_broker_http_directory_no_url_lookup_none(): + assert BrokerHttpJobDirectory("").lookup("job-1") is None + + +def test_upload_target_realpath(tmp_path): + # Layout: storage/{analyses/, binaries, guacrecordings}; a sibling secret outside storage. + storage = tmp_path / "storage" + analysis = storage / "analyses" / "42" + binaries = storage / "binaries" + guac = storage / "guacrecordings" + for d in (analysis, binaries, guac): + d.mkdir(parents=True) + secret = tmp_path / "secret.txt" + secret.write_text("AWS_SECRET") + blob = binaries / "deadbeef" + blob.write_text("sample bytes") + rec = guac / "42_sess" + rec.write_text("guac dump") + + base_real = os.path.realpath(str(analysis)) + trusted = [os.path.realpath(str(binaries)), os.path.realpath(str(guac))] + + # a regular file inside the analysis tree -> uploaded (its own realpath) + plain = analysis / "report.json" + plain.write_text("{}") + assert upload_target_realpath(str(plain), base_real, trusted) == os.path.realpath(str(plain)) + + # the `binary` symlink -> resolves into storage/binaries (trusted) -> uploaded as the blob + binlink = analysis / "binary" + binlink.symlink_to(blob) + assert upload_target_realpath(str(binlink), base_real, trusted) == os.path.realpath(str(blob)) + + # a recording referenced via symlink into storage/guacrecordings (trusted) -> uploaded + reclink = analysis / "guac.rec" + reclink.symlink_to(rec) + assert upload_target_realpath(str(reclink), base_real, trusted) == os.path.realpath(str(rec)) + + # a sample-planted symlink to a host secret OUTSIDE storage roots -> skipped (None) + evil = analysis / "evil" + evil.symlink_to(secret) + assert upload_target_realpath(str(evil), base_real, trusted) is None + + # prefix-collision guard: a sibling dir sharing a name prefix must NOT count as inside + sibling = storage / "binaries-evil" + sibling.mkdir() + sneaky = sibling / "x" + sneaky.write_text("nope") + link2 = analysis / "sneaky" + link2.symlink_to(sneaky) + assert upload_target_realpath(str(link2), base_real, trusted) is None + + +def test_central_mode_worker_access_fields_parse(): + # Interactive-guac worker access: config-driven (was hardcoded deb paths in central_guac). + d = _parse({}) + assert d.worker_api_token_file == "/etc/cape/api-token" + assert d.worker_api_port == 8000 and isinstance(d.worker_api_port, int) + assert d.worker_ssh_user == "cape" + assert d.worker_ssh_keyfile == "/home/cape/.ssh/id_ed25519" + c = _parse({ + "worker_api_token_file": "/opt/secrets/tok", + "worker_api_port": "9443", # string in conf -> int + "worker_ssh_user": "sandbox", + "worker_ssh_keyfile": "/home/sandbox/.ssh/id_rsa", + }) + assert c.worker_api_token_file == "/opt/secrets/tok" + assert c.worker_api_port == 9443 and isinstance(c.worker_api_port, int) + assert c.worker_ssh_user == "sandbox" + assert c.worker_ssh_keyfile == "/home/sandbox/.ssh/id_rsa" + # a bad/out-of-range port degrades to the default, not a startup crash or a broken URL + assert _parse({"worker_api_port": "nope"}).worker_api_port == 8000 + assert _parse({"worker_api_port": "0"}).worker_api_port == 8000 + assert _parse({"worker_api_port": "99999"}).worker_api_port == 8000 + + +def test_central_guac_worker_url_and_dsn(): + # port + task id substitute; the deb defaults no longer live in the code path + assert _worker_task_view_url("10.0.0.5", 8000, 42) == "http://10.0.0.5:8000/apiv2/tasks/view/42/" + assert _worker_task_view_url("10.0.0.5", "9443", "7") == "http://10.0.0.5:9443/apiv2/tasks/view/7/" + # DSN carries the CONFIGURED user + keyfile (not hardcoded cape / id_ed25519) + assert _libvirt_ssh_dsn("10.0.0.9", "cape", "/home/cape/.ssh/id_ed25519") == \ + "qemu+ssh://cape@10.0.0.9/system?keyfile=/home/cape/.ssh/id_ed25519&no_verify=1" + assert _libvirt_ssh_dsn("10.0.0.9", "sandbox", "/home/sandbox/.ssh/id_rsa") == \ + "qemu+ssh://sandbox@10.0.0.9/system?keyfile=/home/sandbox/.ssh/id_rsa&no_verify=1" + # a keyfile with a URI metachar is quoted so it can't corrupt the query string + dsn = _libvirt_ssh_dsn("10.0.0.9", "cape", "/home/cape/my key&x") + assert "my%20key%26x" in dsn and dsn.endswith("&no_verify=1") + + +def test_central_guac_worker_api_token(tmp_path): + tok = tmp_path / "api-token" + tok.write_text(" s3cr3t\n") + assert _worker_api_token(str(tok)) == "s3cr3t" # stripped + # missing/unreadable -> "" (=> no auth header downstream), never raises + assert _worker_api_token(str(tmp_path / "nope")) == "" + + +def test_centralstore_done_marker_local_and_uploaded(tmp_path): + # The completion marker must be BOTH written locally (the worker's NVMe-cleanup gate) AND + # uploaded to the central store (the read seam's .central_staged completion signal). Regression: + # it was local-only, so artifact_storage._stage_tree never saw it and re-staged every view. + from modules.reporting.centralstore import _emit_done_marker + + analysis = tmp_path / "analyses" / "77" + analysis.mkdir(parents=True) + store = LocalFSStore(str(tmp_path / "central")) + cfg = _parse({"enabled": "yes", "s3_bucket": "bkt", "s3_prefix": "results"}) + container = "results/ui-77" + + _emit_done_marker(store, container, str(analysis), cfg, "ui-77", 12) + + # local marker present (cleanup gate) with the expected metadata ... + local_marker = analysis / ".centralstore.done" + assert local_marker.exists() + import json + meta = json.loads(local_marker.read_text()) + assert meta["job_id"] == "ui-77" and meta["artifacts"] == 12 + assert meta["location"] == "s3://bkt/results/ui-77/" + # ... AND uploaded to the store under the exact key the read seam checks for + assert store.exists(container, ".centralstore.done") is True + + +def test_centralstore_done_marker_upload_failure_keeps_local(tmp_path): + # A store put_file failure must NOT raise (the artifacts are already durable) and must leave + # the local marker intact so the worker's cleanup gate still fires. + from modules.reporting.centralstore import _emit_done_marker + + analysis = tmp_path / "analyses" / "88" + analysis.mkdir(parents=True) + + class BoomStore: + def put_file(self, *a, **k): + raise RuntimeError("s3 down") + + cfg = _parse({"enabled": "yes", "s3_bucket": "bkt"}) + _emit_done_marker(BoomStore(), "results/ui-88", str(analysis), cfg, "ui-88", 3) # must not raise + assert (analysis / ".centralstore.done").exists() + + +def test_hunt_facets_per_category_no_facet(): + sent = [] + + def fake_agg(coll, pipeline): + sent.append(pipeline) + return [{"_id": "evil.com", "count": 5, "task_ids": [1, 2]}] + + facets = build_hunt_facets( + fake_agg, + match={"$and": [{}, {"info.visibility": "public"}]}, + hunt_map={"domains": {"db_unwind": "$network.domains", "db_group": "$network.domains.domain", "db_match": {"count": {"$gte": 1}}}}, + categories={"domains": True}, + min_count=1, + ) + assert not any(any("$facet" in stage for stage in p) for p in sent) + assert sent[0][0] == {"$match": {"$and": [{}, {"info.visibility": "public"}]}} + assert facets["domains"][0]["_id"] == "evil.com" diff --git a/tests/test_mt_absent_smoke.py b/tests/test_mt_absent_smoke.py new file mode 100644 index 00000000000..589e36d0a77 --- /dev/null +++ b/tests/test_mt_absent_smoke.py @@ -0,0 +1,85 @@ +"""MT-absent integration smoke — the inverse of the MT-present faithful suite. + +With the multi-tenant layer (users.tenancy + lib.cuckoo.common.tenancy) made unimportable, +the import-optional facades must fall back to see-all AND the base view modules must still +import. This proves central mode runs WITHOUT our multi-tenant fork (the Phase 4 un-weave +goal). Box-run: needs the Django/CAPE environment (pytest-django sets up Django). The test +restores the real facade bindings in a finally so it doesn't pollute later tests. +""" +import builtins +import importlib + + +def test_mt_absent_facades_see_all_and_base_views_import(): + import lib.cuckoo.common.tenancy_optional as L + import web.tenancy_optional as W + + real_import = builtins.__import__ + + def hide(name, *args, **kwargs): + if name in ("users.tenancy", "lib.cuckoo.common.tenancy") or \ + name.startswith("users.tenancy.") or name.startswith("lib.cuckoo.common.tenancy."): + raise ImportError("simulated: MT layer absent") + return real_import(name, *args, **kwargs) + + builtins.__import__ = hide + try: + # re-run the facades' module-level (constant) imports under the hidden layer + importlib.reload(L) + importlib.reload(W) + + # lib facade -> MT-disabled-equivalent fallbacks + assert L.multitenancy_config().enabled is False + assert L.viewer_for(object()).is_local_admin is True + assert L.PUBLIC == "public" and L.VISIBILITIES == ("public", "tenant", "private") + assert L.scope_match("public", object()) is None + assert L.viewer_scope_match(object()) is None + + # web facade -> see-all fallbacks (can_* True, scopes None) + assert W.can_view_task(object(), object()) is True + assert W.can_view_sample(object(), sha256="x") is True + assert W.viewer_for(object()).is_local_admin is True + assert W.viewer_scope_filter(object()) is None + assert W.submission_scope(object()) == (None, W.PUBLIC) # 2-tuple, not None (callers unpack) + + # the base view modules import with the MT layer absent (the core un-weave claim) + for mod in ("analysis.views", "apiv2.views", "submission.views", + "dashboard.views", "guac.views", "guac.consumers", "compare.views"): + importlib.import_module(mod) + finally: + builtins.__import__ = real_import + importlib.reload(L) # restore real bindings for any later tests in the session + importlib.reload(W) + + +def test_no_unguarded_mt_app_imports_in_gate_sites(): + """Regression guard for the class the module-import smoke above MISSES: a raw, unguarded + `from users.tenancy import ...` in a FUNCTION-LOCAL import (fires only at call time) crashes a + single-node/upstream build where the `users` MT app is absent. The MT app must be reached ONLY + through the import-optional facades (tenancy_optional) or a local try/except ImportError. + Whitelist = the two facades + central_scope's guarded fallback.""" + import os + import re + + root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + allowed = { + "web/web/tenancy_optional.py", + "lib/cuckoo/common/tenancy_optional.py", + "web/analysis/central_scope.py", # raw import is inside try/except ImportError (guarded) + } + offenders = [] + for sub in ("lib", "web", "modules", "utils"): + for dirpath, _dirs, files in os.walk(os.path.join(root, sub)): + for fn in files: + if not fn.endswith(".py"): + continue + rel = os.path.relpath(os.path.join(dirpath, fn), root) + if rel in allowed or "test" in rel or rel.endswith("conftest.py"): + continue + try: + src = open(os.path.join(dirpath, fn), encoding="utf-8").read() + except OSError: + continue + if re.search(r"^\s*from users\.tenancy import", src, re.M): + offenders.append(rel) + assert not offenders, f"unguarded MT-app imports (route through the tenancy_optional facade): {offenders}" diff --git a/tests/test_tenancy_optional.py b/tests/test_tenancy_optional.py new file mode 100644 index 00000000000..9f8e37365af --- /dev/null +++ b/tests/test_tenancy_optional.py @@ -0,0 +1,64 @@ +import builtins +import pytest +from lib.cuckoo.common import tenancy_optional as topt + + +def _hide(monkeypatch, modname): + real_import = builtins.__import__ + def fake(name, *a, **k): + if name == modname or name.startswith(modname + "."): + raise ImportError(f"simulated-absent: {modname}") + return real_import(name, *a, **k) + monkeypatch.setattr(builtins, "__import__", fake) + + +def test_lib_facade_present_delegates(monkeypatch): + # MT present: multitenancy_config() returns the real config object (has .enabled) + cfg = topt.multitenancy_config() + assert hasattr(cfg, "enabled") + + +def test_lib_facade_absent_is_see_all(monkeypatch): + _hide(monkeypatch, "lib.cuckoo.common.tenancy") + assert topt.multitenancy_config().enabled is False + assert topt.viewer_for(object()).is_local_admin is True + assert topt.scope_match("acme", topt.viewer_for(object())) is None + + +def test_web_facade_absent_is_see_all(monkeypatch): + _hide(monkeypatch, "users.tenancy") + from web.tenancy_optional import can_view_task, can_view_sample, submission_scope, can_ban_user, PUBLIC + assert can_view_task(object(), object()) is True + assert can_view_sample(object(), sha256="x") is True + # submission_scope MUST return a 2-tuple (callers unpack it); single-tenant -> (None, public) + assert submission_scope(object()) == (None, PUBLIC) + _tid, _vis = submission_scope(object()) # unpack like the real callers do + assert (_tid, _vis) == (None, "public") + # can_ban_user is the ONE facade that must NOT degrade to see-all: the ban_user / + # ban_all_user_tasks views gate SOLELY on it, so the MT-absent fallback preserves + # upstream's staff/superuser-only boundary (a see-all True would let any authenticated + # user ban accounts on a single-node build). Deny a plain user; allow staff/superuser. + class _Plain: is_staff = False; is_superuser = False + class _Staff: is_staff = True; is_superuser = False + class _Super: is_staff = False; is_superuser = True + assert can_ban_user(_Plain(), 1) is False + assert can_ban_user(_Staff(), 1) is True + assert can_ban_user(_Super(), 1) is True + + +def test_web_facade_entitled_scope_filter_absent(monkeypatch): + _hide(monkeypatch, "dashboard.views") + from web.tenancy_optional import viewer_scope_filter + assert viewer_scope_filter(object()) is None + + +def test_web_facade_fail_closed_on_runtime_error(monkeypatch): + # Fail-closed is an MT-present contract: with the MT layer deployed, a runtime error in the + # authz backend must PROPAGATE (never degrade to see-all). Skip on an MT-free upstream build. + ut = pytest.importorskip("users.tenancy") + def boom(*a, **k): + raise RuntimeError("authz backend down") + monkeypatch.setattr(ut, "can_view_task", boom) + from web.tenancy_optional import can_view_task + with pytest.raises(RuntimeError): + can_view_task(object(), object()) diff --git a/web/analysis/central_scope.py b/web/analysis/central_scope.py new file mode 100644 index 00000000000..c1bd50da57e --- /dev/null +++ b/web/analysis/central_scope.py @@ -0,0 +1,45 @@ +"""Tenancy-optional scope resolution for central mode. + +central_mode and multitenancy are ORTHOGONAL toggles. Central mode must work both +WITH the MT fork (tenant-scoped) and WITHOUT it (single-tenant central — everyone +sees all within the deployment, like single-node but centralized), so the central +core can be proposed upstream independently of our multi-tenant layer. + +These helpers are the single choke point through which the central-mode code resolves +the viewer's tenant scope / per-sample visibility. They degrade to see-all ONLY when +the MT layer is not deployed (ImportError). They are deliberately FAIL-CLOSED: when +the MT layer IS deployed, its functions are authoritative and any RUNTIME error +propagates rather than being swallowed into a see-all result (which would silently +bypass tenant isolation — exactly the failure mode the isolation audit warns about). +We catch ImportError ONLY — that precisely means "MT layer absent" — never a broad +Exception. So for our deployments (MT layer present) behavior is byte-for-byte the +direct call; the fallback exists purely for an MT-free upstream deployment. + +entitled_scope_filter already returns None when MT is present but disabled, so see-all +is handled correctly across all three states: + MT layer absent -> ImportError -> None / True (single-tenant central) + MT present, disabled -> entitled_scope_filter -> None ; can_view_sample -> True + MT present, enabled -> real tenant $match / real visibility check +""" + + +def viewer_scope(user): + """Mongo ``$match`` restricting central results to what ``user`` may read, or None + (no tenant filtering / see-all) when the MT layer isn't deployed. None is the + see-all sentinel the central seams already understand (``if scope:``).""" + try: + from dashboard.views import entitled_scope_filter + except ImportError: + return None # MT layer not deployed -> single-tenant central + return entitled_scope_filter(user) # authoritative; handles MT on/off; errors propagate + + +def viewer_can_view_sample(user, *, sha256=None, sha1=None, md5=None, sample_id=None): + """Whether ``user`` may view a sample by hash. True when the MT layer isn't deployed + (single-tenant — the surrounding view's base-CAPE decorator stack still gates access); + otherwise delegates to the authoritative MT visibility check (fail-closed).""" + try: + from users.tenancy import can_view_sample + except ImportError: + return True # MT layer not deployed -> single-tenant central + return can_view_sample(user, sha256=sha256, sha1=sha1, md5=md5, sample_id=sample_id) diff --git a/web/analysis/central_views.py b/web/analysis/central_views.py new file mode 100644 index 00000000000..8e1ec77d5fa --- /dev/null +++ b/web/analysis/central_views.py @@ -0,0 +1,360 @@ +"""Central-mode artifact serving — the S3-backed counterparts of the analysis +download/serve views in web/analysis/views.py. + +Kept in a SEPARATE module on purpose: views.py is heavily synced from upstream +CAPEv2 (cape/ subtree merges), so it carries only a thin +``if central_mode_config().enabled: return central_(...)`` dispatch at the top +of each affected view. All the central-mode logic lives here, in a file upstream does +not have — so it never participates in a merge conflict, and we can take upstream +advancements with minimal friction. + +These functions only run when central mode is ON, and only AFTER the upstream view's +decorator stack (require_task_visibility, ratelimit, auth, …) has already run — so the +relational task is authorized before we touch the central data plane. Single-node +behavior is entirely in views.py and is never reached through this module. +""" +import os + +from django.shortcuts import render + + +def central_job_id_for_task(task_id): + """Resolve the broker job_id for a central task from its RDS `custom` field + (the submit-bridge stamps custom='job_id=ui-'). In the distributed topology + the worker assigns its OWN local info.id (collides across workers), so the + universal key for DocumentDB/S3 is info.job_id — NOT info.id. Returns None for a + non-bridged task (caller falls back to info.id keying for seeded/single-node docs).""" + try: + from analysis.views import db + t = db.view_task(int(task_id)) + custom = getattr(t, "custom", None) if t else None + if custom: + # custom is comma-separated k=v pairs (matches centralstore.resolve_job_id); + # take ONLY the job_id= value, not everything to end-of-string — else a + # 'job_id=ui-5,foo=bar' custom yields 'ui-5,foo=bar' and never matches the + # S3 prefix / DocumentDB doc the artifacts were keyed under. + for part in str(custom).split(","): + part = part.strip() + if part.startswith("job_id="): + v = part.split("=", 1)[1].strip() + if v: + return v + except Exception: + pass + return None + + +def central_analysis_query(task_id, scope=None): + """Mongo filter to fetch a task's analysis doc in central mode. + + PRIMARY key is info.job_id (globally unique) for a bridged task. centralstore also + re-keys info.id to the unique central task id for every bridged job, so {info.id} is + likewise unique in a real central deployment (every task arrives via the bridge) — + which is why the report-tab loaders that query {info.id: task_id} resolve to exactly + the authorized task's doc, not a colliding worker-local one. The info.id FALLBACK + here (non-bridged / seeded / single-node docs, where a worker-local info.id can + collide across workers) is ANDed with the viewer's `scope` (entitled_scope_filter) + as defence-in-depth so a collision can't surface another tenant's doc (audit HIGH).""" + jid = central_job_id_for_task(task_id) + q = {"info.job_id": jid} if jid else {"info.id": int(task_id)} + if scope: + q = {"$and": [q, scope]} + return q + + +def _task_sample_sha256(request, task_id): + """The sha256 of THIS task's submitted sample (mongo target.file.sha256), scoped to + the viewer. central S3 stores only this binary (key /binary), not a by-hash + store, so callers serving sample bytes must confirm the requested hash IS this.""" + from dev_utils.mongodb import mongo_find_one + from analysis.central_scope import viewer_scope + + doc = mongo_find_one("analysis", central_analysis_query(task_id, scope=viewer_scope(request.user)), + {"target.file.sha256": 1, "_id": 0}) + return ((((doc or {}).get("target") or {}).get("file") or {}).get("sha256") or "").lower() + + +def central_stage_one(request, task_id, s3_relpath, dest_abspath): + """Materialize ONE S3 artifact (/) to an exact local path so an + upstream zip path that reads that specific path works centrally. Used for memdumpzip + (memory/ is excluded from the bulk stage) and staticzip (reads the global binaries + store, OUTSIDE the analysis tree). Best-effort; never raises into the view.""" + import shutil + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import materialize_artifact + + if os.path.exists(dest_abspath): + return + src, is_temp = materialize_artifact(task_id, s3_relpath, scope=viewer_scope(request.user)) + if not src: + return + try: + os.makedirs(os.path.dirname(dest_abspath), exist_ok=True) + (shutil.move if is_temp else shutil.copy)(src, dest_abspath) + except Exception: + if is_temp: + try: + os.unlink(src) + except OSError: + pass + + +def central_stage_local(request, task_id): + """Stage the S3 analysis tree into the local FS so an upstream view that reads + storage/analyses// directly works centrally without a per-file rewrite. + Used for the zip-on-the-fly download bundles (zip_categories) — re-implementing + CAPE's pyzipper/password/download_all archiving in the per-file S3 seam would be a + lot of duplicated fork code, so instead we materialize the tree and let the + upstream zip path below run byte-for-byte unchanged. Cached via the .central_staged + marker (cheap on repeat); best-effort (never raises into the view).""" + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import ensure_local_analysis + + ensure_local_analysis(task_id, scope=viewer_scope(request.user)) + + +def central_file_nl(request, category, task_id, dlfile): + """Inline report assets: screenshots, bingraph, vba2graph (file_nl).""" + from django.http import Http404 + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import artifact_response + + # tenant-scope the S3/DocumentDB lookup as defence-in-depth against task_id + # collisions across workers (audit HIGH). + scope = viewer_scope(request.user) + if category == "screenshot": + cands = [(os.path.join("shots", dlfile + ext), dlfile + ext, cd) for ext, cd in ((".jpg", "image/jpeg"), (".png", "image/png"))] + elif category == "bingraph": + cands = [(os.path.join("bingraph", dlfile + "-ent.svg"), dlfile + "-ent.svg", "image/svg+xml")] + elif category == "vba2graph": + cands = [(os.path.join("vba2graph", "svg", f"{dlfile}.svg"), f"{dlfile}.svg", "image/svg+xml")] + else: + return render(request, "error.html", {"error": "Category not defined"}) + for relpath, fn, cd in cands: + try: + return artifact_response(task_id, relpath, cd, fn, scope=scope) + except Http404: + continue + return render(request, "error.html", {"error": f"Could not find {category} {dlfile}"}) + + +def central_filereport(request, task_id, fname): + """Full analysis report download (filereport); fname is the resolved report file.""" + from django.http import Http404 + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import artifact_response + + scope = viewer_scope(request.user) + try: + return artifact_response(task_id, f"reports/{fname}", "application/octet-stream", f"{task_id}_{fname}", scope=scope) + except Http404: + return render(request, "error.html", {"error": f"File not found: {fname}"}) + + +def central_full_memory_dump(request, analysis_number, names): + """Full memory dump / its strings (whole-file); names = candidate relpaths.""" + from django.http import Http404 + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import artifact_response + + scope = viewer_scope(request.user) + for name in names: + try: + return artifact_response(analysis_number, name, "application/octet-stream", name, scope=scope) + except Http404: + continue + return render(request, "error.html", {"error": "File not found"}) + + +def central_file(request, category, task_id, dlfile): + """Main multi-category artifact download (file). Maps each category to its + analysis-relative S3 relpath — the layout centralstore uploads to + s3:///results//. On-the-fly bundles (zip_categories, + pcapng) and search-driven *zipall sets aren't materialized in S3, so they return a + clear central-mode error rather than a silent 404.""" + from django.http import Http404 + + from analysis.central_scope import viewer_scope, viewer_can_view_sample + from lib.cuckoo.common.artifact_storage import artifact_response + from analysis.views import zip_categories + + OCTET = "application/octet-stream" + PCAP = "application/vnd.tcpdump.pcap" + + if category in ("sample", "static"): + # by-hash sample download: enforce the SAME visible-task-referencing-the-sample + # boundary as single-node (audit CRITICAL). In central S3 the ONLY binary stored + # for an analysis is THIS task's submitted sample (key /binary) — it is + # NOT a content-addressed by-hash store like single-node's storage/binaries/. + # So serve /binary only when the requested hash IS this task's sample; + # otherwise return not-found rather than streaming the WRONG file's bytes under + # the requested-hash name (review: wrong-artifact). dropped/related-by-hash from + # S3 is a documented follow-on. + if not viewer_can_view_sample(request.user, sha256=dlfile): + return render(request, "error.html", {"error": "File not found"}) + if _task_sample_sha256(request, task_id) != dlfile.lower(): + return render(request, "error.html", {"error": "File not found"}) + spec = ("binary", dlfile, OCTET) + elif category == "dropped": + spec = (f"files/{dlfile}", dlfile, OCTET) + elif category.startswith("CAPE") and category not in zip_categories: + spec = (f"CAPE/{dlfile}", dlfile, OCTET) + elif category == "pcap": + spec = ("dump.pcap", f"{dlfile}.pcap", PCAP) + elif category == "decrypted_pcap": + spec = ("dump_decrypted.pcap", f"{dlfile}.pcap", PCAP) + elif category == "mixed_pcap": + spec = ("dump_mixed.pcap", f"{dlfile}.pcap", PCAP) + elif category == "debugger_log": + spec = (f"debugger/{dlfile}.log", f"{dlfile}.log", "text/plain") + elif category.startswith("procdump") and category not in zip_categories: + spec = (f"procdump/{dlfile}", dlfile, OCTET) + elif category in ("memdump", "memdumpstrings"): + ext = ".dmp" if category == "memdump" else ".dmp.strings" + spec = (f"memory/{dlfile}{ext}", f"{dlfile}{ext}", OCTET) + elif category == "rtf": + spec = (f"rtf_objects/{dlfile}", dlfile, OCTET) + elif category == "usage": + spec = ("aux/usage.svg", "usage.svg", "image/svg+xml") + elif category == "suricata": + spec = (f"logs/files/{dlfile}", dlfile, OCTET) + elif category == "zip": # suricata dropped files bundle (pre-existing in tree) + spec = ("logs/files.zip", "files.zip", "application/zip") + elif category == "tlskeys": + spec = ("tlsdump/tlsdump.log", "tlsdump.log", "text/plain") + elif category == "sysmon": + spec = ("sysmon/sysmon.data", "sysmon.data", OCTET) + elif category == "evtx": + # fn is wrapped as f"{task_id}_{fn}" below, so don't prefix task_id here (else + # the download name doubles to "__evtx.zip"). + spec = ("evtx/evtx.zip", "evtx.zip", "application/zip") + elif category == "mitmdump": + spec = ("mitmdump/dump.har", "dump.har", "text/plain") + else: + return render(request, "error.html", { + "error": f"'{category}' is not yet available in central mode (server-side bundle/generated artifact)"}) + + relpath, fn, cd = spec + scope = viewer_scope(request.user) + try: + return artifact_response(task_id, relpath, cd, f"{task_id}_{fn}", scope=scope) + except Http404: + return render(request, "error.html", {"error": f"Could not find {category} {dlfile}"}) + + +def central_open_procdump(request, task_id, origname): + """Acquire the proc memory dump as a LOCAL file for the slicing logic in + procdump(): stream memory/ (or its .zip) from S3 to a temp. Returns + (dumpfile_path, tmp_file_path, tmpdir) matching procdump's existing cleanup + variables (it unlinks tmp_file_path + delete_folder(tmpdir)). (None, None, None) + if the dump is absent.""" + import tempfile + import zipfile + + from django.conf import settings + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import materialize_artifact + + scope = viewer_scope(request.user) + dumpfile, is_temp = materialize_artifact(task_id, f"memory/{origname}", scope=scope) + if dumpfile: + return dumpfile, (dumpfile if is_temp else None), None + + zpath, zt = materialize_artifact(task_id, f"memory/{origname}.zip", scope=scope) + if not zpath: + return None, None, None + tmpdir = tempfile.mkdtemp(prefix="capeprocdump_", dir=settings.TEMP_PATH) + try: + with zipfile.ZipFile(zpath, "r") as f: + extracted = f.extract(origname, path=tmpdir) + except Exception: + # bad/corrupt zip or origname not a member: don't leak tmpdir (and the temp zip) + import shutil + + shutil.rmtree(tmpdir, ignore_errors=True) + if zt: + try: + os.unlink(zpath) + except OSError: + pass + return None, None, None + if zt: + try: + os.unlink(zpath) + except OSError: + pass + return extracted, extracted, tmpdir + + +def central_vtupload(request, category, task_id, filename, dlfile): + """Upload a stored artifact to VirusTotal (vtupload): stream it from S3 to a temp, + POST, clean up.""" + import base64 + + import requests + + from analysis.central_scope import viewer_scope, viewer_can_view_sample + from lib.cuckoo.common.artifact_storage import materialize_artifact + from analysis.views import enabledconf, integrations_cfg + + if not (enabledconf["vtupload"] and integrations_cfg.virustotal.apikey): + return render(request, "error.html", {"error": "VirusTotal upload is not enabled"}) + + if category in ("sample", "static"): + if not viewer_can_view_sample(request.user, sha256=dlfile): + return render(request, "error.html", {"error": "File not found"}) + relpath = "binary" + elif category == "dropped": + relpath = f"files/{filename}" + elif category in ("CAPE", "procdump"): + relpath = f"{category}/{filename}" + else: + return render(request, "error.html", {"error": "Category not defined"}) + + scope = viewer_scope(request.user) + path, is_temp = materialize_artifact(task_id, relpath, scope=scope) + if not path: + return render(request, "error.html", {"error": "File not found"}) + try: + headers = {"x-apikey": integrations_cfg.virustotal.apikey} + with open(path, "rb") as fh: + response = requests.post( + "https://www.virustotal.com/api/v3/files", files={"file": (filename, fh)}, headers=headers, + timeout=120, + ) + if response.ok: + vid = response.json().get("data", {}).get("id") + if vid: + hashbytes, _ = base64.b64decode(vid).split(b":") + return render( + request, "success_vtup.html", + {"permalink": "https://www.virustotal.com/gui/file/{id}".format(id=hashbytes.decode())}, + ) + return render(request, "error.html", {"error": "Response code: {} - {}".format(response.status_code, response.reason)}) + except Exception as e: + # network error / non-JSON VT response / malformed id: single-node vtupload wraps + # the whole flow in try/except; mirror that so it renders an error, not a 500. + return render(request, "error.html", {"error": "VirusTotal upload failed: {}".format(e)}) + finally: + if is_temp: + try: + os.unlink(path) + except OSError: + pass + + +def central_pcapstream(request): + """Per-connection pcap regeneration isn't materialized in S3 (the full pcap is).""" + return render(request, "error.html", { + "error": "Per-connection pcap stream is not yet available in central mode — download the full pcap instead"}) + + +def central_on_demand(request): + """On-demand re-processing is a worker/broker concern in central mode, not a UI action.""" + return render(request, "error.html", { + "error": "On-demand detail generation is not available in central mode (re-processing is handled by workers)"}) diff --git a/web/analysis/test_central_scope.py b/web/analysis/test_central_scope.py new file mode 100644 index 00000000000..a5eb93457c9 --- /dev/null +++ b/web/analysis/test_central_scope.py @@ -0,0 +1,85 @@ +"""Tenancy-optional central scope shim (web/analysis/central_scope.py). + +Verifies the three deployment states (MT layer absent / present+disabled / present+ +enabled) collapse to the right scope, AND the security-critical FAIL-CLOSED contract: +when the MT layer IS deployed, a RUNTIME error in the scope/visibility resolution must +PROPAGATE — never be swallowed into a see-all result (which would silently bypass tenant +isolation). The shim catches ImportError ONLY (= MT layer not deployed); these tests lock +that down so a later broad-`except` regression is caught. +""" +import builtins + +import pytest + +from analysis.central_scope import viewer_scope, viewer_can_view_sample + + +def _hide(monkeypatch, modname): + """Force `import ` to raise ImportError, simulating an MT-free deployment + REGARDLESS of whether the module physically exists — so the MT-absent behavior can be + exercised on both an MT-present (fork) build and an MT-absent (upstream) build.""" + real_import = builtins.__import__ + + def fake(name, *a, **k): + if name == modname or name.startswith(modname + "."): + raise ImportError(f"simulated-absent: {modname}") + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake) + + +def test_viewer_scope_mt_layer_absent_is_see_all(monkeypatch): + # MT layer not deployed: `from dashboard.views import entitled_scope_filter` raises + # ImportError -> see-all (None). Hide the module so this holds on any build. + _hide(monkeypatch, "dashboard.views") + assert viewer_scope(object()) is None + + +def test_viewer_scope_delegates_when_mt_present(monkeypatch): + # Delegation is only meaningful when the MT layer is deployed (entitled_scope_filter lives + # in the MT-only dashboard.views surface); skip on an MT-free upstream build. + pytest.importorskip("users.tenancy") + sentinel = {"$or": [{"info.tenant_slug": "acme"}]} + monkeypatch.setattr("dashboard.views.entitled_scope_filter", lambda user: sentinel, raising=False) + assert viewer_scope(object()) is sentinel + + +def test_viewer_scope_fail_closed_on_runtime_error(monkeypatch): + # MT deployed but resolution blows up at runtime -> MUST propagate, not see-all. + pytest.importorskip("users.tenancy") + + def boom(user): + raise RuntimeError("scope resolution failed") + + monkeypatch.setattr("dashboard.views.entitled_scope_filter", boom, raising=False) + with pytest.raises(RuntimeError): + viewer_scope(object()) + + +def test_viewer_can_view_sample_mt_layer_absent_is_true(monkeypatch): + _hide(monkeypatch, "users.tenancy") + assert viewer_can_view_sample(object(), sha256="abc") is True + + +def test_viewer_can_view_sample_delegates_when_mt_present(monkeypatch): + pytest.importorskip("users.tenancy") + seen = {} + + def fake(user, *, sha256=None, sha1=None, md5=None, sample_id=None): + seen.update(sha256=sha256, sha1=sha1, md5=md5, sample_id=sample_id) + return False + + monkeypatch.setattr("users.tenancy.can_view_sample", fake) + assert viewer_can_view_sample(object(), sha256="deadbeef") is False + assert seen == {"sha256": "deadbeef", "sha1": None, "md5": None, "sample_id": None} + + +def test_viewer_can_view_sample_fail_closed_on_runtime_error(monkeypatch): + pytest.importorskip("users.tenancy") + + def boom(user, **kw): + raise RuntimeError("visibility check failed") + + monkeypatch.setattr("users.tenancy.can_view_sample", boom) + with pytest.raises(RuntimeError): + viewer_can_view_sample(object(), sha256="abc") diff --git a/web/analysis/views.py b/web/analysis/views.py index d16e2e5870c..eb430a372a4 100644 --- a/web/analysis/views.py +++ b/web/analysis/views.py @@ -1726,6 +1726,14 @@ def load_files(request, task_id, category): @param task_id: cuckoo task id """ is_ajax = request.headers.get("x-requested-with") == "XMLHttpRequest" + # Central mode: several tab loaders below read the local analysis tree (bingraph / + # vba2graph svgs, evtx.zip, ETW aux/*.json). report() stages the S3 tree on first + # view, but a deep-link straight to a tab can arrive before any report view — stage + # here too so those tabs aren't blank (cheap no-op once .central_staged exists). + from lib.cuckoo.common.central_mode import central_mode_config + if central_mode_config().enabled: + from analysis.central_views import central_stage_local + central_stage_local(request, task_id) if is_ajax and category in ( "CAPE", "dropped", @@ -1928,18 +1936,30 @@ def load_files(request, task_id, category): ajax_response["suricata"] = data.get("suricata", {}) ajax_response["cif"] = data.get("cif", []) ajax_response["pcapng"] = data.get("pcapng", {}) - tls_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "tlsdump", "tlsdump.log") - if _path_safe(tls_path): - ajax_response["tlskeys_exists"] = _path_safe(tls_path) - mitmdump_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "mitmdump", "dump.har") - if _path_safe(mitmdump_path): - ajax_response["mitmdump_exists"] = _path_safe(mitmdump_path) - decrypted_pcap_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "dump_decrypted.pcap") - if _path_safe(decrypted_pcap_path): - ajax_response["decrypted_pcap_exists"] = True - mixed_pcap_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "dump_mixed.pcap") - if _path_safe(mixed_pcap_path): - ajax_response["mixed_pcap_exists"] = True + from lib.cuckoo.common.central_mode import central_mode_config + if central_mode_config().enabled: + # Central: artifacts live in S3, not the local FS — check existence there + # (a local check hides links for files the worker actually produced). + from lib.cuckoo.common.artifact_storage import artifact_exists + from analysis.central_scope import viewer_scope + _sc = viewer_scope(request.user) + ajax_response["tlskeys_exists"] = artifact_exists(task_id, "tlsdump/tlsdump.log", scope=_sc) + ajax_response["mitmdump_exists"] = artifact_exists(task_id, "mitmdump/dump.har", scope=_sc) + ajax_response["decrypted_pcap_exists"] = artifact_exists(task_id, "dump_decrypted.pcap", scope=_sc) + ajax_response["mixed_pcap_exists"] = artifact_exists(task_id, "dump_mixed.pcap", scope=_sc) + else: + tls_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "tlsdump", "tlsdump.log") + if _path_safe(tls_path): + ajax_response["tlskeys_exists"] = _path_safe(tls_path) + mitmdump_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "mitmdump", "dump.har") + if _path_safe(mitmdump_path): + ajax_response["mitmdump_exists"] = _path_safe(mitmdump_path) + decrypted_pcap_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "dump_decrypted.pcap") + if _path_safe(decrypted_pcap_path): + ajax_response["decrypted_pcap_exists"] = True + mixed_pcap_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", str(task_id), "dump_mixed.pcap") + if _path_safe(mixed_pcap_path): + ajax_response["mixed_pcap_exists"] = True elif category == "behavior": ajax_response["detections2pid"] = data.get("detections2pid", {}) return render(request, page, ajax_response) @@ -2657,6 +2677,15 @@ def split_signature_calls(report): @require_safe @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def report(request, task_id): + # Central mode: the analysis tree lives in S3, not on this node's disk. Stage it + # locally (once, cached, excluding huge memory dumps) so EVERY report feature that + # reads the local filesystem renders against the original UI without porting each + # reader to S3. Loaded before the tab AJAX (load_files/load_evtx) fires. + from lib.cuckoo.common.central_mode import central_mode_config + if central_mode_config().enabled: + from lib.cuckoo.common.artifact_storage import ensure_local_analysis + from analysis.central_scope import viewer_scope + ensure_local_analysis(task_id, scope=viewer_scope(request.user)) network_report = {} report = {} if enabledconf["mongodb"]: @@ -2695,9 +2724,17 @@ def report(request, task_id): for service in CUSTOM_SERVICES: projection[service] = 1 + from lib.cuckoo.common.central_mode import central_mode_config + if central_mode_config().enabled: + from analysis.central_views import central_analysis_query + from analysis.central_scope import viewer_scope + _analysis_q = central_analysis_query(task_id, scope=viewer_scope(request.user)) + else: + _analysis_q = {"info.id": int(task_id)} + report = mongo_find_one( "analysis", - {"info.id": int(task_id)}, + _analysis_q, projection, sort=[("_id", -1)], max_time_ms=10000, @@ -2708,7 +2745,7 @@ def report(request, task_id): # Bypass hooks here too existence = mongo_find_one( "analysis", - {"info.id": int(task_id)}, + _analysis_q, {"sigma": 1, "sysmon": 1, "misp": 1, "classification": 1, "_id": 0}, no_hooks=True, ) @@ -3114,6 +3151,13 @@ def load_evtx_channel_count(request, task_id): # under SSO deployments) doesn't 401 the in-browser fetches. @authentication_classes([SessionAuthentication]) def file_nl(request, category, task_id, dlfile): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_file_nl + + return central_file_nl(request, category, task_id, dlfile) + base_path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id)) path = False if category == "screenshot": @@ -3223,6 +3267,33 @@ def _file_search_all_files(search_category: str, search_term: str) -> list: # of dropped files, payloads, etc. via session cookie auth. @authentication_classes([SessionAuthentication]) def file(request, category, task_id, dlfile): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_file, central_stage_local, central_stage_one, _task_sample_sha256 + + # Zip-on-the-fly bundles read the local analysis tree and archive it (pyzipper, + # optional password, download_all). Rather than duplicate that in the per-file S3 + # seam, stage the S3 tree locally and fall through to the upstream zip path below + # unchanged. Single-file downloads keep the efficient per-file seam. + if category in zip_categories: + central_stage_local(request, task_id) + # Two zip categories read paths the bulk stage does NOT populate: memdumpzip + # reads memory/ (excluded — large), staticzip reads the global binaries store + # (outside the analysis tree). Stage the one file each needs so the upstream + # zip path finds it instead of silently returning "File not found". + if category.startswith("memdumpzip"): + central_stage_one(request, task_id, f"memory/{dlfile}.dmp", + os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "memory", f"{dlfile}.dmp")) + elif category == "staticzip" and _task_sample_sha256(request, task_id) == str(dlfile).lower(): + # only the task's OWN sample is in central S3 (/binary); stage it + # to the binaries path upstream reads, gated to the task's sample hash so + # a non-matching hash can't be written under the wrong name (see S2). + central_stage_one(request, task_id, "binary", + os.path.join(CUCKOO_ROOT, "storage", "binaries", str(dlfile))) + else: + return central_file(request, category, task_id, dlfile) + file_name = dlfile cd = "application/octet-stream" path = "" @@ -3436,20 +3507,29 @@ def procdump(request, task_id, process_id, start, end, zipped=False): if es_as_db: analysis = es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id))["hits"]["hits"][0]["_source"] - dumpfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "memory", origname) + from lib.cuckoo.common.central_mode import central_mode_config - if not _path_safe(dumpfile): - return render(request, "error.html", {"error": f"File not found: {os.path.basename(dumpfile)}"}) + if central_mode_config().enabled: + from analysis.central_views import central_open_procdump - if not path_exists(dumpfile): - dumpfile += ".zip" - if not path_exists(dumpfile): + dumpfile, tmp_file_path, tmpdir = central_open_procdump(request, task_id, origname) + if not dumpfile: return render(request, "error.html", {"error": "File not found"}) - f = zipfile.ZipFile(dumpfile, "r") - tmpdir = tempfile.mkdtemp(prefix="capeprocdump_", dir=settings.TEMP_PATH) - tmp_file_path = f.extract(origname, path=tmpdir) - f.close() - dumpfile = tmp_file_path + else: + dumpfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "memory", origname) + + if not _path_safe(dumpfile): + return render(request, "error.html", {"error": f"File not found: {os.path.basename(dumpfile)}"}) + + if not path_exists(dumpfile): + dumpfile += ".zip" + if not path_exists(dumpfile): + return render(request, "error.html", {"error": "File not found"}) + f = zipfile.ZipFile(dumpfile, "r") + tmpdir = tempfile.mkdtemp(prefix="capeprocdump_", dir=settings.TEMP_PATH) + tmp_file_path = f.extract(origname, path=tmpdir) + f.close() + dumpfile = tmp_file_path content_type = "application/octet-stream" @@ -3527,13 +3607,31 @@ def filereport(request, task_id, category): } if category in formats: - path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "reports", formats[category]) + fname = formats[category] + + # Central mode: serve the report file from S3 via the FS->S3 seam. Single-node + # path below is unchanged (the seam returns the local file when central is off). + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from django.http import Http404 + + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.artifact_storage import artifact_response + + scope = viewer_scope(request.user) # tenant-scope the central lookup (audit HIGH) + try: + return artifact_response(task_id, f"reports/{fname}", "application/octet-stream", f"{task_id}_{fname}", scope=scope) + except Http404: + return render(request, "error.html", {"error": f"File not found: {fname}"}) + + path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "reports", fname) if not _path_safe(path) or not path_exists(path): - return render(request, "error.html", {"error": f"File not found: {formats[category]}"}) + return render(request, "error.html", {"error": f"File not found: {fname}"}) response = HttpResponse(Path(path).read_bytes(), content_type="application/octet-stream") - response["Content-Disposition"] = f"attachment; filename={task_id}_{formats[category]}" + response["Content-Disposition"] = f"attachment; filename={task_id}_{fname}" return response return render(request, "error.html", {"error": "File not found"}, status=404) @@ -3542,6 +3640,13 @@ def filereport(request, task_id, category): @require_safe @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def full_memory_dump_file(request, analysis_number): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_full_memory_dump + + return central_full_memory_dump(request, analysis_number, ("memory.dmp", "memory.dmp.zip")) + filename = False for name in ("memory.dmp", "memory.dmp.zip"): path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(analysis_number), name) @@ -3562,6 +3667,13 @@ def full_memory_dump_file(request, analysis_number): @require_safe @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def full_memory_dump_strings(request, analysis_number): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_full_memory_dump + + return central_full_memory_dump(request, analysis_number, ("memory.dmp.strings", "memory.dmp.strings.zip")) + filename = None for name in ("memory.dmp.strings", "memory.dmp.strings.zip"): path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(analysis_number), name) @@ -3755,6 +3867,13 @@ def remove(request, task_id): @require_safe @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def pcapstream(request, task_id, conntuple): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_pcapstream + + return central_pcapstream(request) + src, sport, dst, dport, proto = conntuple.split(",") sport, dport = int(sport), int(dport) @@ -3844,6 +3963,13 @@ def comments(request, task_id): @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def vtupload(request, category, task_id, filename, dlfile): + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_vtupload + + return central_vtupload(request, category, task_id, filename, dlfile) + if enabledconf["vtupload"] and integrations_cfg.virustotal.apikey: try: folder_name = False @@ -3926,6 +4052,12 @@ def on_demand(request, service: str, task_id: str, category: str, sha256): # 3. store results # 4. reload page """ + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from analysis.central_views import central_on_demand + + return central_on_demand(request) if service in CUSTOM_SERVICES: pass @@ -4196,34 +4328,40 @@ def hunt(request): start_date = (datetime.datetime.utcnow() - delta).strftime("%Y-%m-%d %H:%M:%S") match_query["info.started"] = {"$gte": start_date} - # Dynamic multi-category aggregation - facet_stages = {} - for cat_id, cat_config in HUNT_MAP.items(): - if categories[cat_id]: - stages = [] - if cat_config["db_unwind"]: - stages.append({"$unwind": cat_config["db_unwind"]}) - stages.extend([ - {"$group": {"_id": cat_config["db_group"], "count": {"$sum": 1}, "task_ids": {"$addToSet": "$info.id"}}}, - {"$match": cat_config.get("db_match", {"count": {"$gte": min_count}})}, - {"$sort": {"count": -1}}, - {"$limit": 100} - ]) - facet_stages[cat_id] = stages - - # MongoDB Pipeline using $facet for multi-category aggregation - pipeline = [ - {"$match": match_query} - ] - if facet_stages: - pipeline.append({"$facet": facet_stages}) + # Tenant isolation: restrict the docs the aggregation sees to the viewer's + # entitled scopes (no-op for break-glass / shared / multitenancy-disabled). + from analysis.central_scope import viewer_scope + from lib.cuckoo.common.central_mode import central_mode_config + from lib.cuckoo.common.hunt_query import build_hunt_facets + + _scope = viewer_scope(request.user) + _match = {"$and": [match_query, _scope]} if _scope else match_query try: - if facet_stages: - res = list(mongo_aggregate("analysis", pipeline)) - facets = res[0] if res else {} + if central_mode_config().enabled: + # Amazon DocumentDB rejects $facet -> per-category $group loop. Same + # result shape as the single-node $facet path below. + facets = build_hunt_facets(mongo_aggregate, _match, HUNT_MAP, categories, min_count) else: + # Single-node: preserve the original single-$facet pipeline byte-for-byte + # (only the toggle changes single-node behavior). + facet_stages = {} + for cat_id, cat_config in HUNT_MAP.items(): + if categories[cat_id]: + stages = [] + if cat_config["db_unwind"]: + stages.append({"$unwind": cat_config["db_unwind"]}) + stages.extend([ + {"$group": {"_id": cat_config["db_group"], "count": {"$sum": 1}, "task_ids": {"$addToSet": "$info.id"}}}, + {"$match": cat_config.get("db_match", {"count": {"$gte": min_count}})}, + {"$sort": {"count": -1}}, + {"$limit": 100}, + ]) + facet_stages[cat_id] = stages facets = {} + if facet_stages: + res = list(mongo_aggregate("analysis", [{"$match": _match}, {"$facet": facet_stages}])) + facets = res[0] if res else {} except Exception as e: return render(request, "error.html", {"error": f"Threat hunting aggregation failed: {e}"}) diff --git a/web/apiv2/views.py b/web/apiv2/views.py index 3802ad8d06e..e024bbf2fd0 100644 --- a/web/apiv2/views.py +++ b/web/apiv2/views.py @@ -1196,6 +1196,8 @@ def tasks_report(request, task_id, report_format="json", make_zip=False): if rtid: task_id = rtid + _central_stage(request, task_id) + resp = {} srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "reports") @@ -1380,6 +1382,8 @@ def tasks_iocs(request, task_id, detail=None): if rtid: task_id = rtid + _central_stage(request, task_id) + buf = {} if repconf.mongodb.get("enabled") and not buf: buf = mongo_find_one("analysis", {"info.id": int(task_id)}, {"behavior.calls": 0}) @@ -1614,6 +1618,8 @@ def tasks_screenshot(request, task_id, screenshot="all"): if rtid: task_id = rtid + _central_stage(request, task_id) + srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "shots") if not os.path.normpath(srcdir).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(srcdir)}"}) @@ -1667,6 +1673,8 @@ def tasks_pcap(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + srcfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "dump.pcap") if not os.path.normpath(srcfile).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(srcfile)}"}) @@ -1704,6 +1712,31 @@ def _resolve_task_id(task_id, enabled_key, check_tlp=True): return task_id, None +def _central_stage(request, task_id, include_memory=False): + """Central mode: stage the S3 results// tree to the local + storage/analyses// dir so the local-FS artifact reads in the apiv2 + download endpoints below work (same generic seam the web report view uses — + avoids rewriting each endpoint's FS reads). MUST be called AFTER the endpoint's + per-task authorization so an unauthorized task_id is never staged. The large + memory dumps are excluded from the bulk stage; the fullmemory/procmemory + endpoints pass include_memory=True to stage them on explicit demand. No-op + single-node; best-effort (never raises).""" + try: + from lib.cuckoo.common.central_mode import central_mode_config + + if not central_mode_config().enabled: + return + from lib.cuckoo.common.artifact_storage import ensure_local_analysis, ensure_local_memory + from analysis.central_scope import viewer_scope + + scope = viewer_scope(request.user) + ensure_local_analysis(task_id, scope=scope) + if include_memory: + ensure_local_memory(task_id, scope=scope) + except Exception: + pass + + def _serve_analysis_file(task_id, rel_path, download_name, content_type="application/octet-stream"): """Stream `/` back as an attachment. Returns a Response object (either a StreamingHttpResponse for success, or a JSON error).""" @@ -1877,6 +1910,7 @@ def tasks_pcap_variant(request, task_id, variant): task_id, err = _resolve_task_id(task_id, "taskpcap") if err: return err + _central_stage(request, task_id) v = (variant or "").lower() if v in _PCAP_VARIANTS: rel_path, fname = _PCAP_VARIANTS[v] @@ -1957,6 +1991,8 @@ def tasks_evtx(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + evtxfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "evtx", "evtx.zip") if not os.path.normpath(evtxfile).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(evtxfile)}"}) @@ -1984,6 +2020,7 @@ def tasks_mitmdump(request, task_id): rtid = check.get("rtid", 0) if rtid: task_id = rtid + _central_stage(request, task_id) harfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "mitmdump", "dump.har") if not os.path.normpath(harfile).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(harfile)}"}) @@ -2017,6 +2054,8 @@ def tasks_dropped(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "files") if not os.path.normpath(srcdir).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(srcdir)}"}) @@ -2067,6 +2106,8 @@ def tasks_selfextracted(request, task_id, tool="all"): if rtid: task_id = rtid + _central_stage(request, task_id) + srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "selfextracted") if not os.path.normpath(srcdir).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(srcdir)}"}) @@ -2166,6 +2207,8 @@ def tasks_surifile(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + srcfile = os.path.join(CUCKOO_ROOT, "storage", "analyses", "%s" % task_id, "logs", "files.zip") if not os.path.normpath(srcfile).startswith(ANALYSIS_BASE_PATH): return render(request, "error.html", {"error": f"File not found: {os.path.basename(srcfile)}"}) @@ -2231,6 +2274,7 @@ def tasks_procmemory(request, task_id, pid="all"): if rtid: task_id = rtid + _central_stage(request, task_id, include_memory=True) # Check if any process memory dumps exist srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", f"{task_id}", "memory") if not path_exists(srcdir): @@ -2309,6 +2353,7 @@ def tasks_fullmemory(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id, include_memory=True) filename = "" file_path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "memory.dmp") if path_exists(file_path): @@ -2557,6 +2602,8 @@ def tasks_payloadfiles(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "CAPE") if not os.path.normpath(srcdir).startswith(ANALYSIS_BASE_PATH): @@ -2594,6 +2641,8 @@ def tasks_procdumpfiles(request, task_id): if rtid: task_id = rtid + _central_stage(request, task_id) + # ToDo add all/one srcdir = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "procdump") @@ -2631,6 +2680,8 @@ def tasks_config(request, task_id, cape_name=False): if rtid: task_id = rtid + _central_stage(request, task_id) + buf = {} if repconf.mongodb.get("enabled"): buf = mongo_find_one("analysis", {"info.id": int(task_id)}, {"CAPE.configs": 1}, sort=[("_id", -1)]) diff --git a/web/guac/consumers.py b/web/guac/consumers.py index b5f41d7986f..d62175388bb 100644 --- a/web/guac/consumers.py +++ b/web/guac/consumers.py @@ -31,14 +31,16 @@ ACTIVE_GUAC_TASK_STATUSES = ("pending", "running") -def _get_vnc_port(vm_label): - """Look up VNC port for a VM from libvirt. Must be called from sync context.""" +def _get_vnc_port(vm_label, dsn=machinery_dsn): + """Look up VNC port for a VM from libvirt. Must be called from sync context. + `dsn` is the local hypervisor for single-node, or a worker's libvirt-over-SSH + in central mode (the VM lives on the worker hosting the job).""" if not LIBVIRT_AVAILABLE: return None conn = None try: - conn = libvirt.open(machinery_dsn) + conn = libvirt.open(dsn) if not conn: return None dom = conn.lookupByName(vm_label) @@ -167,6 +169,7 @@ async def connect(self): vm_label = self.vm_label vnc_port = None + worker_ip = None # central mode: set to the worker's IP when the task's VM is remote if self.guac_task_id > 0: # 3. Verify task can still host an interactive session task = await sync_to_async(db.view_task)(self.guac_task_id) @@ -178,8 +181,15 @@ async def connect(self): await self.close() return - # 4. Look up VNC port server-side from libvirt - vnc_port = await sync_to_async(_get_vnc_port)(vm_label) + # 4. Central mode: a broker-dispatched job's VM lives on a worker, so + # resolve that worker's libvirt DSN + IP and look up the VNC port from ITS + # libvirt; the tunnel then targets the worker's guacd. None => single-node. + from lib.cuckoo.common.central_guac import libvirt_dsn_for_task + + vnc_dsn, worker_ip = await sync_to_async(libvirt_dsn_for_task)(self.guac_task_id, machinery_dsn) + + # 4b. Look up VNC port server-side from libvirt (local or worker) + vnc_port = await sync_to_async(_get_vnc_port)(vm_label, vnc_dsn) if not vnc_port: logger.warning( "WebSocket rejected: no VNC port for VM %s", vm_label @@ -208,8 +218,9 @@ async def connect(self): await self.close() return - # 5. Parse config - guacd_hostname = web_cfg.guacamole.guacd_host or "localhost" + # 5. Parse config. Central mode: target the WORKER's guacd (which + # reaches the VM's VNC on its own localhost); single-node uses configured guacd. + guacd_hostname = worker_ip or web_cfg.guacamole.guacd_host or "localhost" guacd_port = int(web_cfg.guacamole.guacd_port) or 4822 guacd_recording_path = web_cfg.guacamole.guacd_recording_path or "" guest_protocol = web_cfg.guacamole.guest_protocol or "vnc" @@ -240,7 +251,10 @@ async def connect(self): "disable-theming": "true", } else: - guest_host = web_cfg.guacamole.vnc_host or "localhost" + # Central: the task's VM is on a worker, whose guacd (guacd_hostname= + # worker_ip) reaches the VM's VNC on its OWN localhost; single-node uses + # the configured vnc_host. + guest_host = "localhost" if worker_ip else (web_cfg.guacamole.vnc_host or "localhost") guest_port = vnc_port ignore_cert = "false" vnc_color_depth = str( diff --git a/web/guac/views.py b/web/guac/views.py index 56d4198d9d0..48fff9ca415 100644 --- a/web/guac/views.py +++ b/web/guac/views.py @@ -59,19 +59,51 @@ def index(request, task_id, session_data): if machinery not in machinery_available: return _error(request, task_id, f"Machinery type '{machinery}' is not supported") + # The VM label + guest IP are derived authoritatively from the task below (never from the + # attacker-controlled session_data path segment), so the task must exist. + _task = db.view_task(int(task_id)) + if not _task: + return _error(request, task_id, "The specified task doesn't seem to exist") + + # Central mode: a broker-dispatched job's VM is on a worker — check that + # worker's libvirt. None => local (single-node), DSN unchanged. + from lib.cuckoo.common.central_guac import libvirt_dsn_for_task + + dsn, _worker_ip = libvirt_dsn_for_task(int(task_id), machinery_dsn) + conn = None try: - conn = libvirt.open(machinery_dsn) + conn = libvirt.open(dsn) if not conn: return _error(request, task_id, "Could not connect to hypervisor") try: - session_id, label, guest_ip = ( + session_id, _claimed_label, _claimed_ip = ( urlsafe_b64decode(session_data).decode("utf8").split("|") ) except Exception as e: return _error(request, task_id, str(e)) + # SECURITY: BOTH the VM label AND the guest IP MUST come from the authorized task, + # never from the attacker-controlled session_data path segment. Trusting the label + # lets a caller who can reach ONE running task tunnel into another VM by name; + # trusting guest_ip lets them point the guacd RDP tunnel at an arbitrary host:3389 + # (SSRF — guest_host is built from this value in consumers.py). Derive both from the + # task: _task.machine + its machine record (single-node) or the worker's VM (central + # mode). Ignore _claimed_label / _claimed_ip entirely. + label = _task.machine + guest_ip = "" + if label: + _m = db.view_machine_by_label(label) + guest_ip = getattr(_m, "ip", "") or "" + else: + from lib.cuckoo.common.central_guac import worker_vm_for_task + + label, guest_ip = worker_vm_for_task(int(task_id)) + guest_ip = guest_ip or "" + if not label: + return _error(request, task_id, "No VM is associated with this task") + try: dom = conn.lookupByName(label) except Exception as e: diff --git a/web/submission/views.py b/web/submission/views.py index 7888d98d4f4..247eb9038bf 100644 --- a/web/submission/views.py +++ b/web/submission/views.py @@ -788,11 +788,24 @@ def status(request, task_id): "target": task.sample.sha256 if getattr(task, "sample") else task.target, } if web_conf.guacamole.enabled and get_options(task.options).get("interactive") == "1": - machine = db.view_machine_by_label(task.machine) - if machine: - guest_ip = machine.ip + machine = db.view_machine_by_label(task.machine) if task.machine else None + vm_label, guest_ip = (task.machine, machine.ip) if machine else (None, None) + if not machine: + # Central mode ONLY: the VM lives on a worker, so it's not in the central machines + # table — resolve the worker's VM label via the broker record + worker API. Gated on + # central mode so single-node behaves exactly as upstream (no session_data unless a + # real machine record resolved — never a degenerate empty-guest_ip session). + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from lib.cuckoo.common.central_guac import worker_vm_for_task + + w_label, w_ip = worker_vm_for_task(task_id) + if w_label: + vm_label, guest_ip = w_label, (w_ip or "") + if vm_label: session_id = uuid3(NAMESPACE_DNS, task_id).hex[:16] - session_data = urlsafe_b64encode(f"{session_id}|{task.machine}|{guest_ip}".encode("utf8")).decode("utf8") + session_data = urlsafe_b64encode(f"{session_id}|{vm_label}|{guest_ip or ''}".encode("utf8")).decode("utf8") response["session_data"] = session_data return render(request, "submission/status.html", response) @@ -808,13 +821,25 @@ def remote_session(request, task_id): session_data = "" if task.status == "running": - machine = db.view_machine_by_label(task.machine) + machine = db.view_machine_by_label(task.machine) if task.machine else None + vm_label, guest_ip = (machine.label, machine.ip) if machine else (None, None) if not machine: + # Central mode ONLY: the VM lives on a worker (not in the central machines table) — + # resolve it via the broker record + worker API. Gated on central mode so single-node + # is byte-for-byte upstream: no machine record -> the "Machine is not set" error below. + from lib.cuckoo.common.central_mode import central_mode_config + + if central_mode_config().enabled: + from lib.cuckoo.common.central_guac import worker_vm_for_task + + w_label, w_ip = worker_vm_for_task(task_id) + if w_label: + vm_label, guest_ip = w_label, (w_ip or "") + if not vm_label: return render(request, "error.html", {"error": "Machine is not set for this task."}) - guest_ip = machine.ip machine_status = True session_id = uuid3(NAMESPACE_DNS, task_id).hex[:16] - session_data = urlsafe_b64encode(f"{session_id}|{machine.label}|{guest_ip}".encode("utf8")).decode("utf8") + session_data = urlsafe_b64encode(f"{session_id}|{vm_label}|{guest_ip or ''}".encode("utf8")).decode("utf8") return render( request, diff --git a/web/web/settings.py b/web/web/settings.py index d646fa9079c..e4fa75660b7 100644 --- a/web/web/settings.py +++ b/web/web/settings.py @@ -314,6 +314,25 @@ ], } + +def _users_app_present() -> bool: + """True iff the multi-tenant `users` app is deployed (its package dir exists). + + The MT layer is import-optional: an upstream / central-only build simply omits + web/users/. This (drop the app from INSTALLED_APPS) and the tenancy_optional facades + (ImportError -> see-all) key off the SAME presence signal, so there is ONE source of + truth — no separate env flag that could diverge from the import-availability the facades + actually see (which previously left a flag-set-but-files-present build hitting non-migrated + tables). To run single-tenant WITH the files present, use `[multitenancy] enabled = no` + (the existing runtime toggle), not app removal. + """ + return (BASE_DIR / "users").is_dir() + + +# Drop the optional MT `users` app (and its migrations) when its package isn't deployed. +if not _users_app_present(): + INSTALLED_APPS = [a for a in INSTALLED_APPS if a != "users"] + AUDIT_FRAMEWORK = web_cfg.audit_framework.get("enabled", False) if api_cfg.api.token_auth_enabled: diff --git a/web/web/tenancy_optional.py b/web/web/tenancy_optional.py new file mode 100644 index 00000000000..f58309d1f70 --- /dev/null +++ b/web/web/tenancy_optional.py @@ -0,0 +1,113 @@ +"""Import-optional facade for the web-level MT symbols (users.tenancy + the +entitled_scope_filter/entitled_scopes defined in dashboard.views). + +Same contract as the lib facade: delegate when the MT layer is importable, fall back to the +MT-disabled-equivalent value when it raises ImportError, FAIL-CLOSED on runtime errors. Re- +exports the lib-level facade symbols so a view needs a single import. +""" +from lib.cuckoo.common.tenancy_optional import ( # noqa: F401 + MTConfig, + PRIVATE, + PUBLIC, + TENANT, + VISIBILITIES, + Viewer, + default_visibility, + scope_match, + viewer_scope_es_filter, + viewer_scope_match, +) + + +# viewer_for and multitenancy_config are delegated to users.tenancy (NOT the lib facade): +# the web layer historically imported these from users.tenancy, whose own viewer_for resolves +# multitenancy_config in the users.tenancy namespace. Routing them through the lib module would +# read a different binding (e.g. test fixtures patch users.tenancy.multitenancy_config), silently +# changing scoping. In production both bindings point at the same object; this preserves the +# web layer's exact resolution. +def viewer_for(user): + try: + from users.tenancy import viewer_for as real + except ImportError: + return Viewer() + return real(user) + + +def multitenancy_config(): + try: + from users.tenancy import multitenancy_config as real + except ImportError: + return MTConfig() + return real() + + +def can_view_task(user, task): + try: + from users.tenancy import can_view_task as real + except ImportError: + return True + return real(user, task) + + +def can_toggle_task(user, task): + try: + from users.tenancy import can_toggle_task as real + except ImportError: + return True + return real(user, task) + + +def can_manage_task(user, task): + try: + from users.tenancy import can_manage_task as real + except ImportError: + return True + return real(user, task) + + +def can_view_sample(user, *, sha256=None, sha1=None, md5=None, sample_id=None): + try: + from users.tenancy import can_view_sample as real + except ImportError: + return True + return real(user, sha256=sha256, sha1=sha1, md5=md5, sample_id=sample_id) + + +def can_ban_user(actor, target_user_id): + # The ban_user / ban_all_user_tasks VIEWS gate SOLELY on this call (they no longer carry their + # own is_staff check), so the MT-absent fallback MUST preserve upstream's staff/superuser-only + # boundary. Returning True would let ANY authenticated user (or anonymous, if WEB_AUTHENTICATION + # is off) ban/disable accounts on a single-node build. + try: + from users.tenancy import can_ban_user as real + except ImportError: + return bool(getattr(actor, "is_staff", False) or getattr(actor, "is_superuser", False)) + return real(actor, target_user_id) + + +def submission_scope(request): + """Resolve (tenant_id, visibility) for a new submission. The real function returns a + 2-tuple and every caller unpacks it (`_tid, _vis = submission_scope(request)`), so the + MT-absent fallback must ALSO be a 2-tuple — single-tenant: no tenant, public default.""" + try: + from users.tenancy import submission_scope as real + except ImportError: + return (None, PUBLIC) + return real(request) + + +def viewer_scope_filter(user): + """Facade for dashboard.views.entitled_scope_filter (None = see-all).""" + try: + from dashboard.views import entitled_scope_filter as real + except ImportError: + return None + return real(user) + + +def entitled_scopes(user): + try: + from dashboard.views import entitled_scopes as real + except ImportError: + return ("global",) + return real(user) diff --git a/web/web/test_settings_mt_optional.py b/web/web/test_settings_mt_optional.py new file mode 100644 index 00000000000..87756bd562b --- /dev/null +++ b/web/web/test_settings_mt_optional.py @@ -0,0 +1,36 @@ +"""Unit-tests for the _users_app_present() helper in web.settings. + +The MT `users` app is dropped from INSTALLED_APPS iff its package dir is absent — the SAME +import-availability signal the tenancy_optional facades use, so there is one source of truth +(no env flag that could diverge). These call the helper directly (monkeypatching the dir +check) so they never need to reload Django settings. +""" + + +def _helper(): + import web.settings as s + return s._users_app_present + + +def test_users_app_present_when_dir_exists(monkeypatch): + # Real deployment: web/users/ is on disk -> app stays installed. + assert _helper()() is True # the running checkout HAS web/users/ + + +def test_users_app_absent_when_dir_missing(monkeypatch): + # Upstream / central-only build: web/users/ omitted -> helper reports absent. + import web.settings as s + monkeypatch.setattr(s, "_users_app_present", lambda: False) + assert s._users_app_present() is False + + +def test_users_dropped_from_installed_apps_when_absent(monkeypatch): + # The conditional that consumes the helper: when absent, `users` is filtered out; when + # present, it stays. (Logic check, no settings reload.) + apps = ["analysis", "users", "dashboard"] + present = False + result = [a for a in apps if a != "users"] if not present else apps + assert "users" not in result + present = True + result = apps if present else [a for a in apps if a != "users"] + assert "users" in result From ce49dba406fd558544bd95982fe8ae43c919a79f Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Fri, 10 Jul 2026 10:26:32 -0500 Subject: [PATCH 48/53] central-mode: address review (CI ruff + gemini) - ruff E701/E702: expand the one-line helper classes in test_tenancy_optional (was failing CI). - artifact_storage._job_id_for_task: coalesce a null info doc ({"info": None}) before .get (was AttributeError), and cache task_id->job_id (immutable) so a view's repeated artifact_exists/stream calls don't each re-query mongo. Bounded, scope-keyed. - ensure_local_analysis / ensure_local_memory: let a clean Http404 propagate (so a direct view caller returns 404 instead of a broken page) while still swallowing transient errors best-effort. - central_guac._job_id_for_task + central_views.central_job_id_for_task: also accept the bare-token custom form, matching centralstore.resolve_job_id; coalesce null info doc in the mongo fallback. - S3Store.materialize: move tempfile.mkstemp inside the try so an OSError returns (None, False) instead of escaping and breaking the caller contract. --- lib/cuckoo/common/artifact_storage.py | 46 ++++++++++++++++++++++----- lib/cuckoo/common/central_guac.py | 11 +++++-- lib/cuckoo/common/storage_backend.py | 19 +++++------ tests/test_tenancy_optional.py | 15 +++++++-- web/analysis/central_views.py | 8 ++++- 5 files changed, 76 insertions(+), 23 deletions(-) diff --git a/lib/cuckoo/common/artifact_storage.py b/lib/cuckoo/common/artifact_storage.py index 43702a3d3fb..ebcbac70c26 100644 --- a/lib/cuckoo/common/artifact_storage.py +++ b/lib/cuckoo/common/artifact_storage.py @@ -28,6 +28,14 @@ def _safe_relpath(relpath): return relpath +# task_id -> job_id is immutable once a task is dispatched, and a single report/tab view resolves it +# several times (each artifact_exists / stream call). Cache the resolution to avoid N identical mongo +# lookups. Keyed by (task_id, scope) so a viewer's tenant filter can't leak another tenant's mapping. +# Only successful resolutions are cached (failures re-query); bounded so it can't grow unbounded. +_JOB_ID_CACHE = {} +_JOB_ID_CACHE_MAX = 1024 + + def _job_id_for_task(task_id, scope=None): """Central mode keys the store by the global job_id (the broker passes it in custom, stamped into info.job_id at reporting; centralstore re-keys info.id to the unique @@ -41,6 +49,11 @@ def _job_id_for_task(task_id, scope=None): from dev_utils.mongodb import mongo_find_one from django.http import Http404 + cache_key = (str(task_id), str(scope)) + cached = _JOB_ID_CACHE.get(cache_key) + if cached is not None: + return cached + # filereport/full_memory routes capture task_id/analysis_number as \w+ (not \d+), # so a non-numeric segment must raise Http404 (the views catch it -> clean error), # not an uncaught ValueError -> HTTP 500. @@ -52,9 +65,14 @@ def _job_id_for_task(task_id, scope=None): if scope: query = {"$and": [query, scope]} doc = mongo_find_one("analysis", query, {"info.job_id": 1}) - job_id = (doc or {}).get("info", {}).get("job_id") + # info may be missing OR explicitly None ({"info": None}); coalesce both to {} before .get. + job_id = ((doc or {}).get("info") or {}).get("job_id") if not job_id: raise Http404("no job_id mapping for task") + + if len(_JOB_ID_CACHE) >= _JOB_ID_CACHE_MAX: + _JOB_ID_CACHE.clear() + _JOB_ID_CACHE[cache_key] = job_id return job_id @@ -132,7 +150,9 @@ def ensure_local_analysis(task_id, scope=None, exclude_prefixes=("memory/", "mem seen, so a listing taken mid-upload isn't cached as complete. Excludes the large on-demand memory dumps (memory/ subtree + the root memory.dmp[.zip/.strings] full-RAM image) which would otherwise bloat every report view by GBs; they stage on demand via ensure_local_ - memory / stream via materialize_artifact. No-op single-node. Best-effort: never raises.""" + memory / stream via materialize_artifact. No-op single-node. Best-effort: swallows transient + errors (the per-file seam still serves downloads), but a clean Http404 propagates so a direct + view caller returns 404 rather than a broken page.""" cfg = central_mode_config() if not cfg.enabled: return @@ -146,23 +166,33 @@ def ensure_local_analysis(task_id, scope=None, exclude_prefixes=("memory/", "mem if complete: with open(marker, "w") as f: f.write("staged") - except Exception: - # leave whatever was staged; the per-file seam still covers downloads - pass + except Exception as e: + from django.http import Http404 + + # A clean not-found (bad task id / no job_id mapping / out-of-scope) must propagate so a + # direct view caller (report/load_files) returns a 404 instead of rendering a broken page. + # Everything else stays best-effort: leave whatever staged; the per-file seam still serves. + if isinstance(e, Http404): + raise def ensure_local_memory(task_id, scope=None): """Central mode: stage the memory dumps (the memory/ per-process subtree AND the root memory.dmp[.zip/.strings] full-RAM image) — which ensure_local_analysis EXCLUDES from the bulk stage because they are large — to the local analysis dir, on EXPLICIT demand (the - memory-download endpoints). Idempotent per-file; not marker-gated. Best-effort.""" + memory-download endpoints). Idempotent per-file; not marker-gated. Best-effort (a clean Http404 + propagates so the view 404s; other errors are swallowed).""" cfg = central_mode_config() if not cfg.enabled: return try: _stage_tree(task_id, scope, want=lambda rel: rel.startswith("memory/") or rel.startswith("memory.dmp")) - except Exception: - pass + except Exception as e: + from django.http import Http404 + + # Let a clean not-found propagate (view -> 404); swallow everything else (best-effort). + if isinstance(e, Http404): + raise def artifact_exists(task_id, relpath, scope=None): diff --git a/lib/cuckoo/common/central_guac.py b/lib/cuckoo/common/central_guac.py index 248918be675..9f50f841553 100644 --- a/lib/cuckoo/common/central_guac.py +++ b/lib/cuckoo/common/central_guac.py @@ -56,19 +56,26 @@ def _job_id_for_task(task_id): # comma-separated k=v pairs — take ONLY the job_id= value (not the rest of the # string), matching centralstore.resolve_job_id; a trailing ',foo=bar' would # otherwise corrupt the DynamoDB key / S3 prefix lookup. - for part in str(custom).split(","): + text = str(custom) + for part in text.split(","): part = part.strip() if part.startswith("job_id="): v = part.split("=", 1)[1].strip() if v: return v + # Bare-token form (custom is just the job id) — kept in sync with + # centralstore.resolve_job_id, which also accepts a bare token for non-bridged tasks. + token = text.strip() + if token and "=" not in token and "," not in token: + return token except Exception: pass try: from dev_utils.mongodb import mongo_find_one doc = mongo_find_one("analysis", {"info.id": int(task_id)}, {"info.job_id": 1}) - return (doc or {}).get("info", {}).get("job_id") + # info may be missing OR explicitly None ({"info": None}); coalesce both before .get. + return ((doc or {}).get("info") or {}).get("job_id") except Exception: return None diff --git a/lib/cuckoo/common/storage_backend.py b/lib/cuckoo/common/storage_backend.py index 79547e10a86..21f2cefd49e 100644 --- a/lib/cuckoo/common/storage_backend.py +++ b/lib/cuckoo/common/storage_backend.py @@ -185,22 +185,23 @@ def read_text(self, container, relpath, max_bytes): return "" def materialize(self, container, relpath): + # Whole body in one try so a mkstemp OSError (disk full / bad TEMP perms) can't escape and + # break the (None, False) contract the caller relies on. + tmp = None try: obj = self._client().get_object(Bucket=self.bucket, Key=self._key(container, relpath)) - except Exception: - return (None, False) - fd, tmp = tempfile.mkstemp(prefix="cape_central_") - try: + fd, tmp = tempfile.mkstemp(prefix="cape_central_") with os.fdopen(fd, "wb") as f: for c in obj["Body"].iter_chunks(65536): f.write(c) + return (tmp, True) except Exception: - try: - os.unlink(tmp) - except OSError: - pass + if tmp is not None: + try: + os.unlink(tmp) + except OSError: + pass return (None, False) - return (tmp, True) def iter_relpaths(self, container): prefix = f"{container}/" diff --git a/tests/test_tenancy_optional.py b/tests/test_tenancy_optional.py index 9f8e37365af..df2ca6e0fa3 100644 --- a/tests/test_tenancy_optional.py +++ b/tests/test_tenancy_optional.py @@ -38,9 +38,18 @@ def test_web_facade_absent_is_see_all(monkeypatch): # ban_all_user_tasks views gate SOLELY on it, so the MT-absent fallback preserves # upstream's staff/superuser-only boundary (a see-all True would let any authenticated # user ban accounts on a single-node build). Deny a plain user; allow staff/superuser. - class _Plain: is_staff = False; is_superuser = False - class _Staff: is_staff = True; is_superuser = False - class _Super: is_staff = False; is_superuser = True + class _Plain: + is_staff = False + is_superuser = False + + class _Staff: + is_staff = True + is_superuser = False + + class _Super: + is_staff = False + is_superuser = True + assert can_ban_user(_Plain(), 1) is False assert can_ban_user(_Staff(), 1) is True assert can_ban_user(_Super(), 1) is True diff --git a/web/analysis/central_views.py b/web/analysis/central_views.py index 8e1ec77d5fa..5988b6850d6 100644 --- a/web/analysis/central_views.py +++ b/web/analysis/central_views.py @@ -33,12 +33,18 @@ def central_job_id_for_task(task_id): # take ONLY the job_id= value, not everything to end-of-string — else a # 'job_id=ui-5,foo=bar' custom yields 'ui-5,foo=bar' and never matches the # S3 prefix / DocumentDB doc the artifacts were keyed under. - for part in str(custom).split(","): + text = str(custom) + for part in text.split(","): part = part.strip() if part.startswith("job_id="): v = part.split("=", 1)[1].strip() if v: return v + # Bare-token form (custom is just the job id) — kept in sync with + # centralstore.resolve_job_id, which also accepts a bare token for non-bridged tasks. + token = text.strip() + if token and "=" not in token and "," not in token: + return token except Exception: pass return None From b52ee161ad172a9de9e3774b9dfd49e1ec8ff874 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Fri, 10 Jul 2026 10:32:51 -0500 Subject: [PATCH 49/53] test(guac): stub _get_vnc_port with its new (vm_label, dsn) signature _get_vnc_port gained an optional dsn param (local hypervisor, or a worker's libvirt-over-SSH in central mode) and the consumer now passes it positionally; the app-factory stub was 1-arg and raised 'takes 1 positional argument but 2 were given', failing the 5 connect() tests. --- tests/web/test_guac_consumers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/web/test_guac_consumers.py b/tests/web/test_guac_consumers.py index 78de0ad917d..4093a2826b2 100644 --- a/tests/web/test_guac_consumers.py +++ b/tests/web/test_guac_consumers.py @@ -189,7 +189,9 @@ def _build(*, timeout_manager_cls=None, stub_monitor_timeout=True, monkeypatch.setattr(consumers, "GuacamoleClient", FakeGuacamoleClient) monkeypatch.setattr(consumers, "SessionTimeoutManager", timeout_manager_cls) monkeypatch.setattr(consumers, "Database", lambda: db) - monkeypatch.setattr(consumers, "_get_vnc_port", lambda vm_label: TEST_VNC_PORT) + # _get_vnc_port now takes an optional dsn (local hypervisor, or a worker's libvirt in + # central mode); the consumer passes it positionally, so the stub must accept it. + monkeypatch.setattr(consumers, "_get_vnc_port", lambda vm_label, dsn=None: TEST_VNC_PORT) monkeypatch.setattr(consumers.GuacamoleWebSocketConsumer, "read_guacd", read_guacd_impl) monkeypatch.setattr(consumers.GuacamoleWebSocketConsumer, "monitor_task_status", _background_task_stub) if stub_monitor_timeout: From 75e51cd9e8bddfd65325fa6c40ab4cfbf5c240a7 Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Fri, 10 Jul 2026 11:08:13 -0500 Subject: [PATCH 50/53] central-mode: address cloud codex review (cache authz-safety + docstring) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _JOB_ID_CACHE: only cache the UNSCOPED (see-all / MT-absent / break-glass) resolution, keyed by task_id alone. task_id->job_id is immutable so that is safe forever, but a tenant-SCOPED lookup is authorization-sensitive (a task's tenant/visibility can be reassigned) — a process-lifetime cache could serve a job_id the caller may no longer see. Scoped lookups now always re-query; the repeated-lookup win is kept for the common single-tenant central deployment. - _stage_tree docstring: drop the stale 'never raises' claim — it intentionally lets Http404 from _store_and_container propagate so callers can return a clean 404 (ensure_local_* catch the rest). --- lib/cuckoo/common/artifact_storage.py | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/cuckoo/common/artifact_storage.py b/lib/cuckoo/common/artifact_storage.py index ebcbac70c26..1205b3dd102 100644 --- a/lib/cuckoo/common/artifact_storage.py +++ b/lib/cuckoo/common/artifact_storage.py @@ -28,10 +28,12 @@ def _safe_relpath(relpath): return relpath -# task_id -> job_id is immutable once a task is dispatched, and a single report/tab view resolves it -# several times (each artifact_exists / stream call). Cache the resolution to avoid N identical mongo -# lookups. Keyed by (task_id, scope) so a viewer's tenant filter can't leak another tenant's mapping. -# Only successful resolutions are cached (failures re-query); bounded so it can't grow unbounded. +# Cache ONLY the unscoped resolution. task_id -> job_id is immutable once a task is dispatched, and a +# single report/tab view resolves it several times (each artifact_exists / stream call), so caching +# the see-all path avoids N identical mongo lookups. We deliberately do NOT cache a tenant-SCOPED +# resolution: that lookup is authorization-sensitive (a task's tenant/visibility can be reassigned), +# so a process-lifetime cache could keep serving a job_id the caller may no longer see. Scoped lookups +# always re-query. Only successful resolutions are cached; bounded so it can't grow unbounded. _JOB_ID_CACHE = {} _JOB_ID_CACHE_MAX = 1024 @@ -49,10 +51,13 @@ def _job_id_for_task(task_id, scope=None): from dev_utils.mongodb import mongo_find_one from django.http import Http404 - cache_key = (str(task_id), str(scope)) - cached = _JOB_ID_CACHE.get(cache_key) - if cached is not None: - return cached + # Only the unscoped (see-all / MT-absent / break-glass) path is cache-safe — see the note on + # _JOB_ID_CACHE. A present scope is authorization-sensitive, so never cache/serve it. + use_cache = not scope + if use_cache: + cached = _JOB_ID_CACHE.get(str(task_id)) + if cached is not None: + return cached # filereport/full_memory routes capture task_id/analysis_number as \w+ (not \d+), # so a non-numeric segment must raise Http404 (the views catch it -> clean error), @@ -70,9 +75,10 @@ def _job_id_for_task(task_id, scope=None): if not job_id: raise Http404("no job_id mapping for task") - if len(_JOB_ID_CACHE) >= _JOB_ID_CACHE_MAX: - _JOB_ID_CACHE.clear() - _JOB_ID_CACHE[cache_key] = job_id + if use_cache: + if len(_JOB_ID_CACHE) >= _JOB_ID_CACHE_MAX: + _JOB_ID_CACHE.clear() + _JOB_ID_CACHE[str(task_id)] = job_id return job_id @@ -113,7 +119,10 @@ def _stage_tree(task_id, scope, want): local storage/analyses// tree so the MANY report features that read the local filesystem work centrally without rewriting each reader. `want(rel) -> bool` selects which relpaths to stage. Returns True iff the .centralstore.done completion marker was seen. - Best-effort: never raises. No-op single-node (caller guards).""" + Per-file copy errors are swallowed (best-effort), BUT _store_and_container() may raise Http404 + (bad task id / no job_id mapping / out-of-scope) and that PROPAGATES so a caller can return a + clean 404; callers that want pure best-effort must catch it (ensure_local_* do). No-op + single-node (caller guards).""" store, container = _store_and_container(task_id, scope) local = _local_analysis_path(task_id, "") os.makedirs(local, exist_ok=True) From 7a2a8ff162e09ff90df6167047b52e950ff5fb3d Mon Sep 17 00:00:00 2001 From: Will Metcalf Date: Fri, 10 Jul 2026 12:33:32 -0500 Subject: [PATCH 51/53] central-mode: log silenced staging failures + LRU job_id cache (antigravity review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ensure_local_analysis / ensure_local_memory: after re-raising a clean Http404, log.warning the swallowed exception instead of dropping it silently — S3 creds/permission/network staging failures are now diagnosable (added a module logger). - _JOB_ID_CACHE: OrderedDict LRU (move_to_end on hit, popitem(last=False) at threshold) instead of clear-all, so hot recently-viewed tasks stay cached and eviction is one-at-a-time (no miss-storm). --- lib/cuckoo/common/artifact_storage.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/cuckoo/common/artifact_storage.py b/lib/cuckoo/common/artifact_storage.py index 1205b3dd102..a5fe59e92eb 100644 --- a/lib/cuckoo/common/artifact_storage.py +++ b/lib/cuckoo/common/artifact_storage.py @@ -6,12 +6,16 @@ mode runs on any S3-compatible store (AWS/MinIO/Ceph) or a shared mount, with AWS purely config. Validated live on a CAPE box; the branch/seam logic here is the unit-testable part. """ +import logging import os +from collections import OrderedDict from lib.cuckoo.common.constants import CUCKOO_ROOT from lib.cuckoo.common.central_mode import central_mode_config from lib.cuckoo.common.storage_backend import get_artifact_store, ArtifactNotFound +log = logging.getLogger(__name__) + def _local_analysis_path(task_id, relpath): return os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), relpath) @@ -33,8 +37,9 @@ def _safe_relpath(relpath): # the see-all path avoids N identical mongo lookups. We deliberately do NOT cache a tenant-SCOPED # resolution: that lookup is authorization-sensitive (a task's tenant/visibility can be reassigned), # so a process-lifetime cache could keep serving a job_id the caller may no longer see. Scoped lookups -# always re-query. Only successful resolutions are cached; bounded so it can't grow unbounded. -_JOB_ID_CACHE = {} +# always re-query. Only successful resolutions are cached; a bounded LRU so the hot (recently viewed) +# tasks stay cached and we evict one-at-a-time instead of dumping the whole cache at the threshold. +_JOB_ID_CACHE = OrderedDict() # most-recently-used at the end _JOB_ID_CACHE_MAX = 1024 @@ -57,6 +62,7 @@ def _job_id_for_task(task_id, scope=None): if use_cache: cached = _JOB_ID_CACHE.get(str(task_id)) if cached is not None: + _JOB_ID_CACHE.move_to_end(str(task_id)) # mark most-recently-used return cached # filereport/full_memory routes capture task_id/analysis_number as \w+ (not \d+), @@ -76,9 +82,10 @@ def _job_id_for_task(task_id, scope=None): raise Http404("no job_id mapping for task") if use_cache: - if len(_JOB_ID_CACHE) >= _JOB_ID_CACHE_MAX: - _JOB_ID_CACHE.clear() _JOB_ID_CACHE[str(task_id)] = job_id + _JOB_ID_CACHE.move_to_end(str(task_id)) + if len(_JOB_ID_CACHE) > _JOB_ID_CACHE_MAX: + _JOB_ID_CACHE.popitem(last=False) # evict least-recently-used return job_id @@ -183,6 +190,8 @@ def ensure_local_analysis(task_id, scope=None, exclude_prefixes=("memory/", "mem # Everything else stays best-effort: leave whatever staged; the per-file seam still serves. if isinstance(e, Http404): raise + # ...but don't stay SILENT — S3 creds/permission/network failures otherwise vanish. + log.warning("central mode: failed to stage analysis %s: %s", task_id, e) def ensure_local_memory(task_id, scope=None): @@ -199,9 +208,11 @@ def ensure_local_memory(task_id, scope=None): except Exception as e: from django.http import Http404 - # Let a clean not-found propagate (view -> 404); swallow everything else (best-effort). + # Let a clean not-found propagate (view -> 404); log-then-swallow everything else so a + # staging failure (S3 creds/permission/network) is diagnosable rather than silent. if isinstance(e, Http404): raise + log.warning("central mode: failed to stage memory for analysis %s: %s", task_id, e) def artifact_exists(task_id, relpath, scope=None): From c05fdbc31a97911d4ee85674190aecfcfbba3a50 Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 18:09:00 +0000 Subject: [PATCH 52/53] fix: build Suricata/network DNS passlist per-run instead of mutating a shared global suricata.py appended the passlist file into the imported module-global domain_passlist_re on every run(). In a long-lived worker process (e.g. the pebble engine with maxtasksperchild=0, which never recycles a worker) the shared list grew by the whole passlist every task, so the per-event re.search loop became O(tasks_processed x events) and eventually stalled Suricata processing for minutes -- presenting as a hung/"deadlocked" task. Engines that use a fresh process per task reset the global and hid it. Fix: build the passlist per run via a new _compile_passlist() helper (local task_passlist_re) starting from a base safelist compiled once at module import, matching with compiled.search(). network.py had the same append into the same shared global at import time, which also polluted Suricata's copy with duplicates -- switched to a private, pre-compiled dns_passlist_re. Invalid passlist entries are skipped instead of raising. Adds regression tests asserting the shared global is never mutated and that filtering behaviour is unchanged; the network test forces a clean reload so the module's top-level build is actually re-run and measured. --- modules/processing/network.py | 45 +++++++++++----------- modules/processing/suricata.py | 62 ++++++++++++++++++++++++------ tests/test_network_passlist.py | 37 ++++++++++++++++++ tests/test_suricata_passlist.py | 67 +++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 34 deletions(-) create mode 100644 tests/test_network_passlist.py create mode 100644 tests/test_suricata_passlist.py diff --git a/modules/processing/network.py b/modules/processing/network.py index 6c39f3bc8db..a28a08c70bb 100644 --- a/modules/processing/network.py +++ b/modules/processing/network.py @@ -17,7 +17,7 @@ import tempfile import traceback from base64 import b64encode -from collections import OrderedDict, namedtuple, defaultdict +from collections import OrderedDict, defaultdict, namedtuple from contextlib import suppress from hashlib import md5, sha1, sha256 from itertools import islice @@ -115,12 +115,21 @@ logging.getLogger("httpreplay").setLevel(logging.CRITICAL) comment_re = re.compile(r"\s*#.*") +# Build the DNS passlist once, pre-compiled. Do NOT append to the imported +# domain_passlist_re module-global: that list is shared with suricata.py, so +# mutating it here polluted that module's passlist with duplicates. Pre-compiling +# also removes the per-event recompile cost in the match loops below. +dns_passlist_re = [] +for pattern in domain_passlist_re: + with suppress(re.error): + dns_passlist_re.append(re.compile(pattern)) if enabled_passlist and passlist_file: f = path_read_file(os.path.join(CUCKOO_ROOT, passlist_file), mode="text") for domain in f.splitlines(): domain = comment_re.sub("", domain).strip() if domain: - domain_passlist_re.append(domain) + with suppress(re.error): + dns_passlist_re.append(re.compile(domain)) ip_passlist = set() network_passlist = [] @@ -527,8 +536,8 @@ def _add_dns(self, udpdata, ts): query["answers"].append(ans) if enabled_passlist: - for reject in domain_passlist_re: - if re.search(reject, query["request"]): + for reject in dns_passlist_re: + if reject.search(query["request"]): for addip in query["answers"]: if routing_cfg.inetsim.enabled and addip["data"] == routing_cfg.inetsim.server: continue @@ -609,8 +618,8 @@ def _add_http(self, conn, tcpdata, ts): entry["host"] = conn["dst"] if enabled_passlist: - for reject in domain_passlist_re: - if re.search(reject, entry["host"]): + for reject in dns_passlist_re: + if reject.search(entry["host"]): return False entry["port"] = conn["dport"] @@ -811,8 +820,8 @@ def run(self): self._tcp_dissect(connection, tcp.data, ts) src, sport, dst, dport = connection["src"], connection["sport"], connection["dst"], connection["dport"] if not ( - (dst, dport, src, sport) in self.tcp_connections_seen - or (src, sport, dst, dport) in self.tcp_connections_seen + (dst, dport, src, sport) in self.tcp_connections_seen + or (src, sport, dst, dport) in self.tcp_connections_seen ): self.tcp_connections.append((src, sport, dst, dport, offset, ts - first_ts)) self.tcp_connections_seen.add((src, sport, dst, dport)) @@ -843,8 +852,8 @@ def run(self): src, sport, dst, dport = connection["src"], connection["sport"], connection["dst"], connection["dport"] if not ( - (dst, dport, src, sport) in self.udp_connections_seen - or (src, sport, dst, dport) in self.udp_connections_seen + (dst, dport, src, sport) in self.udp_connections_seen + or (src, sport, dst, dport) in self.udp_connections_seen ): self.udp_connections.append((src, sport, dst, dport, offset, ts - first_ts)) self.udp_connections_seen.add((src, sport, dst, dport)) @@ -993,8 +1002,8 @@ def run(self): hostname = sent.headers.get("host") included_to_passlist = False - for reject in domain_passlist_re: - if hostname and re.search(reject, hostname): + for reject in dns_passlist_re: + if hostname and reject.search(hostname): included_to_passlist = True if included_to_passlist: @@ -1392,17 +1401,9 @@ def _merge_behavior_network(self, network): winhttp_sessions = net_map.get("winhttp_sessions") if winhttp_sessions: # Recompute current http host set (includes http/http_ex/https_ex) - http_events = ( - (network.get("http", []) or []) + - (network.get("http_ex", []) or []) + - (network.get("https_ex", []) or []) - ) + http_events = (network.get("http", []) or []) + (network.get("http_ex", []) or []) + (network.get("https_ex", []) or []) - existing_hosts = { - _norm_domain(h.get("host")) - for h in http_events - if h.get("host") - } + existing_hosts = {_norm_domain(h.get("host")) for h in http_events if h.get("host")} for p in winhttp_sessions: proc_sessions = (p or {}).get("sessions") or {} diff --git a/modules/processing/suricata.py b/modules/processing/suricata.py index 3fa82d249bd..e681fc52ddd 100644 --- a/modules/processing/suricata.py +++ b/modules/processing/suricata.py @@ -37,6 +37,15 @@ log = logging.getLogger(__name__) +# Pre-compile the static base safelist once at import (pure CPU on a constant list, +# no I/O or config) so _compile_passlist() doesn't recompile it on every run. +_BASE_PASSLIST_RE = [] +for _pattern in domain_passlist_re: + try: + _BASE_PASSLIST_RE.append(re.compile(_pattern)) + except re.error: + log.warning("Suricata: invalid base passlist regex %r; skipping", _pattern) + class Suricata(Processing): """Suricata processing.""" @@ -59,6 +68,33 @@ def json_default(self, obj): return obj.decode() raise TypeError + @staticmethod + def _compile_passlist(enabled_passlist, passlist_file): + """Return the DNS passlist as compiled regex patterns, built per run. + + Starts from the pre-compiled base safelist and appends the optional + passlist file, NEVER mutating the imported module-global + ``domain_passlist_re``: a reused worker process (pebble ``-mc0`` never + recycles a worker) would otherwise re-append the whole file every task, + growing the per-event match loop in run() without bound until Suricata + processing stalls for minutes and looks like a deadlock. Pre-compiling + also removes the per-event recompile cost that dominated that loop. + Invalid file entries are skipped rather than aborting the run. + """ + patterns = list(_BASE_PASSLIST_RE) + if enabled_passlist and passlist_file: + comment_re = re.compile(r"\s*#.*") + data = path_read_file(os.path.join(CUCKOO_ROOT, passlist_file), mode="text") + for domain in data.splitlines(): + domain = comment_re.sub("", domain).strip() + if not domain: + continue + try: + patterns.append(re.compile(domain)) + except re.error: + log.warning("Suricata: invalid passlist domain regex %r; skipping", domain) + return patterns + def run(self): """Run Suricata. @return: hash with alerts @@ -238,14 +274,11 @@ def run(self): enabled_passlist = processing_cfg.network.dnswhitelist passlist_file = processing_cfg.network.dnswhitelist_file - comment_re = re.compile(r"\s*#.*") - - if enabled_passlist and passlist_file: - f = path_read_file(os.path.join(CUCKOO_ROOT, passlist_file), mode="text") - for domain in f.splitlines(): - domain = comment_re.sub("", domain).strip() - if domain: - domain_passlist_re.append(domain) + # Build the DNS passlist fresh and pre-compiled per run. Never append to the + # imported module-global domain_passlist_re — in a reused worker (pebble + # -mc0) that accumulates the whole passlist every task and stalls the loop + # below. See _compile_passlist for the full rationale. + task_passlist_re = self._compile_passlist(enabled_passlist, passlist_file) filter_event_types = {"alert": "", "http": "hostname", "tls": "sni", "dns": "rrname", "ssh": "hostname", "fileinfo": ""} @@ -273,9 +306,10 @@ def run(self): search_value = h.get("value", "") break - for reject in domain_passlist_re: - if re.search(reject, search_value): + for reject in task_passlist_re: + if reject.search(search_value): skip_event = True + break if skip_event: continue @@ -321,11 +355,15 @@ def run(self): continue if http_data.get("version") == "2" and "request_headers" in http_data: # HTTP/2: extract fields from pseudo-headers and header arrays - req_headers = {h["name"]: h["value"] for h in http_data.get("request_headers", []) if "name" in h and "value" in h} + req_headers = { + h["name"]: h["value"] for h in http_data.get("request_headers", []) if "name" in h and "value" in h + } # Skip HTTP/2 control frames (SETTINGS, WINDOW_UPDATE, etc.) that have no :method if ":method" not in req_headers: continue - resp_headers = {h["name"]: h["value"] for h in http_data.get("response_headers", []) if "name" in h and "value" in h} + resp_headers = { + h["name"]: h["value"] for h in http_data.get("response_headers", []) if "name" in h and "value" in h + } hlog["hostname"] = req_headers.get(":authority", None) hlog["uri"] = req_headers.get(":path", None) hlog["http_method"] = req_headers.get(":method", None) diff --git a/tests/test_network_passlist.py b/tests/test_network_passlist.py new file mode 100644 index 00000000000..760bddb6686 --- /dev/null +++ b/tests/test_network_passlist.py @@ -0,0 +1,37 @@ +"""Regression test for the network.py twin of the Suricata passlist bug. + +`network.py` built its DNS passlist by appending the passlist file to the +imported module-global ``domain_passlist_re`` — the same shared list read by +``suricata.py``. That polluted suricata's copy with duplicates and is the same +mutate-a-shared-import anti-pattern that stalled reused workers. The fix builds +a private, pre-compiled ``dns_passlist_re`` and never touches the global. +""" + +import importlib +import os +import sys + +CUCKOO_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..") +sys.path.append(CUCKOO_ROOT) + + +def test_network_import_does_not_mutate_shared_passlist(): + # Force a clean (re)import so network.py's top-level passlist build actually + # re-runs and is measured. Without this, sys.modules caching can make the test + # pass trivially: if network was already imported, ``base_len`` would be read + # AFTER any mutation and the assert would compare base_len to itself. + sys.modules.pop("modules.processing.network", None) + import data.safelist.domains as safelist + + importlib.reload(safelist) + base_len = len(safelist.domain_passlist_re) + + import modules.processing.network as net + + net = importlib.reload(net) + + assert len(safelist.domain_passlist_re) == base_len, "network.py must not mutate the shared domain_passlist_re global" + # network keeps its own pre-compiled list that includes the base patterns + assert len(net.dns_passlist_re) >= base_len + assert all(hasattr(p, "search") for p in net.dns_passlist_re), "passlist entries must be pre-compiled patterns" + assert any(p.search("x.windowsupdate.com") for p in net.dns_passlist_re) diff --git a/tests/test_suricata_passlist.py b/tests/test_suricata_passlist.py new file mode 100644 index 00000000000..a311cdf61dd --- /dev/null +++ b/tests/test_suricata_passlist.py @@ -0,0 +1,67 @@ +"""Regression test for the Suricata DNS-passlist accumulation bug. + +`suricata.py` used to `domain_passlist_re.append(domain)` into the imported +module-global on every run(). In a reused worker process (pebble -mc0 never +recycles a worker) the list grew by the whole passlist file every task, so the +per-event `re.search` loop became O(tasks_processed x events) until Suricata +processing stalled — a multi-minute hang that presents as a deadlock. prefork +forks a fresh process per task, which reset the global and hid the bug. + +The fix builds a fresh, pre-compiled passlist per run without mutating the +global. These tests lock in: (1) the global is never mutated, (2) repeated +builds do not accumulate, (3) filtering still matches/rejects correctly. +""" + +import os +import sys + +CUCKOO_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..") +sys.path.append(CUCKOO_ROOT) + +from data.safelist.domains import domain_passlist_re +from modules.processing.suricata import Suricata + + +def test_compile_passlist_does_not_mutate_global(): + base_len = len(domain_passlist_re) + # Build twice; neither call may grow the imported module-global, and neither + # build may accumulate on top of the previous one. + first = Suricata._compile_passlist(False, "") + second = Suricata._compile_passlist(False, "") + assert len(domain_passlist_re) == base_len, "run() must not mutate the shared domain_passlist_re global" + assert len(first) == len(second) == base_len, "passlist must not accumulate across builds" + + +def test_compile_passlist_appends_file_without_touching_global(tmp_path, monkeypatch): + import modules.processing.suricata as suri + + base_len = len(domain_passlist_re) + wl = tmp_path / "wl.txt" + wl.write_text("# a comment line\nfoo\\.example\\.com$\n\nbar\\.test$\n") + monkeypatch.setattr(suri, "CUCKOO_ROOT", str(tmp_path)) + + patterns = Suricata._compile_passlist(True, "wl.txt") + + # base + 2 file domains (comment line and blank line are skipped) + assert len(patterns) == base_len + 2 + # even the file-append path must leave the shared global untouched + assert len(domain_passlist_re) == base_len + # and the compiled patterns must still match / reject correctly + assert any(p.search("foo.example.com") for p in patterns) + assert not any(p.search("zzz-nomatch-1234.invalid") for p in patterns) + + +def test_compile_passlist_survives_bad_regex(tmp_path, monkeypatch): + """A malformed passlist entry must be skipped, not crash the whole run.""" + import modules.processing.suricata as suri + + base_len = len(domain_passlist_re) + wl = tmp_path / "wl.txt" + wl.write_text("good\\.test$\n(((unterminated\n") + monkeypatch.setattr(suri, "CUCKOO_ROOT", str(tmp_path)) + + patterns = Suricata._compile_passlist(True, "wl.txt") + + # only the valid entry is compiled; the bad one is dropped + assert len(patterns) == base_len + 1 + assert any(p.search("good.test") for p in patterns) From 0468148c11373a21c2297a458c497f40f353abbb Mon Sep 17 00:00:00 2001 From: wmetcalf Date: Fri, 10 Jul 2026 20:18:30 +0000 Subject: [PATCH 53/53] address review: log invalid passlist regexes in network.py, break on match, sys.path.insert in tests - network.py: log a warning when a base/file passlist regex fails to compile (was silently suppressed), matching suricata.py. - network.py: break out of the Pcap2 hostname passlist loop on first match. - tests: sys.path.insert(0, ...) so the local checkout takes import priority. --- modules/processing/network.py | 9 +++++++-- tests/test_network_passlist.py | 2 +- tests/test_suricata_passlist.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/processing/network.py b/modules/processing/network.py index a28a08c70bb..062de036d30 100644 --- a/modules/processing/network.py +++ b/modules/processing/network.py @@ -121,15 +121,19 @@ # also removes the per-event recompile cost in the match loops below. dns_passlist_re = [] for pattern in domain_passlist_re: - with suppress(re.error): + try: dns_passlist_re.append(re.compile(pattern)) + except re.error: + log.warning("Network: invalid base passlist regex %r; skipping", pattern) if enabled_passlist and passlist_file: f = path_read_file(os.path.join(CUCKOO_ROOT, passlist_file), mode="text") for domain in f.splitlines(): domain = comment_re.sub("", domain).strip() if domain: - with suppress(re.error): + try: dns_passlist_re.append(re.compile(domain)) + except re.error: + log.warning("Network: invalid passlist domain regex %r; skipping", domain) ip_passlist = set() network_passlist = [] @@ -1005,6 +1009,7 @@ def run(self): for reject in dns_passlist_re: if hostname and reject.search(hostname): included_to_passlist = True + break if included_to_passlist: continue diff --git a/tests/test_network_passlist.py b/tests/test_network_passlist.py index 760bddb6686..45f31beab87 100644 --- a/tests/test_network_passlist.py +++ b/tests/test_network_passlist.py @@ -12,7 +12,7 @@ import sys CUCKOO_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..") -sys.path.append(CUCKOO_ROOT) +sys.path.insert(0, CUCKOO_ROOT) def test_network_import_does_not_mutate_shared_passlist(): diff --git a/tests/test_suricata_passlist.py b/tests/test_suricata_passlist.py index a311cdf61dd..0d851d3c5dd 100644 --- a/tests/test_suricata_passlist.py +++ b/tests/test_suricata_passlist.py @@ -16,7 +16,7 @@ import sys CUCKOO_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..") -sys.path.append(CUCKOO_ROOT) +sys.path.insert(0, CUCKOO_ROOT) from data.safelist.domains import domain_passlist_re from modules.processing.suricata import Suricata