From 8985867a42f0151956b54ae1fe0029f9befa7bf6 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 2 Jul 2026 11:50:57 +0200 Subject: [PATCH 1/7] adjust mem --- src/datasets/loaders/tenx_atera/config.vsh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/loaders/tenx_atera/config.vsh.yaml b/src/datasets/loaders/tenx_atera/config.vsh.yaml index 6da3a43bf..ed74ffeac 100644 --- a/src/datasets/loaders/tenx_atera/config.vsh.yaml +++ b/src/datasets/loaders/tenx_atera/config.vsh.yaml @@ -68,4 +68,4 @@ runners: - type: executable - type: nextflow directives: - label: [midmem, midcpu, midtime] \ No newline at end of file + label: [highmem, midcpu, midtime] \ No newline at end of file From 76335233f7bf5aa333c72b3c551e0d1b4bd04183 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 2 Jul 2026 15:18:35 +0200 Subject: [PATCH 2/7] Publish atera loader output to local raw/ folder; update nebius labels Co-Authored-By: Claude Opus 4.8 --- .../spatial/process_10x_atera_nebius.sh | 4 +++- src/base/labels_nebius.config | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/create_resources/spatial/process_10x_atera_nebius.sh b/scripts/create_resources/spatial/process_10x_atera_nebius.sh index d3475e586..432122295 100644 --- a/scripts/create_resources/spatial/process_10x_atera_nebius.sh +++ b/scripts/create_resources/spatial/process_10x_atera_nebius.sh @@ -8,7 +8,9 @@ cd "$REPO_ROOT" set -e -publish_dir="s3://openproblems-data/resources/datasets" +# store the loader output locally, mirroring the process_datasets layout ($id/) +# but under a sibling raw/ folder +publish_dir="/scratch/task_ist_preprocessing/raw" cat > /tmp/params_atera.yaml << HERE param_list: diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index fd98f257d..de49246a0 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -47,20 +47,26 @@ process { withLabel: midcpu { cpus = 15 } withLabel: highcpu { cpus = 30 } withLabel: lowmem { - memory = { get_memory( 20.GB * task.attempt ) } + // Nebius 48vcpu-192gb nodes: 188 GB allocatable + memory = { get_memory( 25.GB * task.attempt ) } disk = { 50.GB * task.attempt } + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00p5wjvb7bc55b2js']] } withLabel: midmem { + // Nebius 64vcpu-256gb nodes: 251 GB allocatable memory = { get_memory( 50.GB * task.attempt ) } disk = { 100.GB * task.attempt } + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00wvqfyde1nfwezs0']] } withLabel: highmem { - memory = { task.attempt == 1 ? 200.GB : 500.GB } + // Nebius 128vcpu-512gb nodes: 503 GB allocatable — cap well below node max + memory = { task.attempt == 1 ? 200.GB : 450.GB } disk = 200.GB pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00hnqhyfdcsy9m09n']] } withLabel: veryhighmem { - memory = { get_memory( 200.GB * task.attempt ) } + // Nebius 128vcpu-512gb nodes: 503 GB allocatable — cap at 450 GB to leave headroom + memory = { [ 200.GB * task.attempt, 450.GB ].min() } disk = { 400.GB * task.attempt } pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00hnqhyfdcsy9m09n']] } From db60559daf70c62d5f18ace795afd770eb65163b Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 6 Jul 2026 15:22:50 +0200 Subject: [PATCH 3/7] GPU mem adjust --- src/base/labels_nebius.config | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index de49246a0..fa7cfe90e 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -121,6 +121,17 @@ process { containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } + withLabel: gpuhighmem { + // Nebius high-mem gpu-node-group: NVIDIA L40S, 1 GPU, 40 CPUs, 160 GiB RAM + // ~156 GiB allocatable — cap memory at 150 GB to leave headroom + cpus = 30 + accelerator = 1 + memory = { [ 100.GB * task.attempt, 150.GB ].min() } + disk = 200.GB + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00dhcgx1xqjskycvc']] + containerOptions = { workflow.containerEngine == "singularity" ? '--nv': + ( workflow.containerEngine == "docker" ? '--gpus all': null ) } + } withLabel: hightime { time = 12.h } withLabel: veryhightime { time = 24.h } From 4a147095cc3e21ba4080f80852206acb7701bbd9 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 6 Jul 2026 20:53:42 +0200 Subject: [PATCH 4/7] debug allen brai --- .../allen_brain_cell_atlas_merfish/script.py | 50 +++++++++++++++++-- .../cellpose/config.vsh.yaml | 2 +- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py index 014775282..f9e4fe979 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py +++ b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py @@ -1,6 +1,7 @@ import math import os import re +import time import urllib.request from datetime import datetime from pathlib import Path @@ -137,8 +138,38 @@ def close(self): self.fileptr.close() +def robust_urlretrieve(url, path, retries=5, backoff=5): + """urlretrieve that retries on transient network errors. + + Returns True on success, False if the file is genuinely absent (404) or + every retry was exhausted. Partial downloads are cleaned up so a later + ``.exists()`` check does not treat a truncated file as complete. + """ + for attempt in range(1, retries + 1): + try: + urllib.request.urlretrieve(url, path) + return True + except urllib.error.HTTPError as e: + if e.code == 404: + print(datetime.now() - t0, f"Not found (404): {url}", flush=True) + return False + print(datetime.now() - t0, + f"HTTP {e.code} (attempt {attempt}/{retries}) for {url}", flush=True) + except (urllib.error.URLError, ConnectionError, TimeoutError) as e: + print(datetime.now() - t0, + f"Download failed (attempt {attempt}/{retries}) for {url}: {e}", flush=True) + Path(path).unlink(missing_ok=True) + if attempt < retries: + time.sleep(backoff * attempt) + return False + + def download_fov_image(fov_idx, n_format, prefix, suffix, base_url, frames, stain): - """Download one FOV dax file and return a (n_z, px, py) uint16 array.""" + """Download one FOV dax file and return a (n_z, px, py) uint16 array. + + Returns None if the FOV image cannot be fetched (missing file or persistent + network failure) so the caller can skip it instead of crashing the run. + """ fmt = f"{fov_idx:03d}" if n_format == 3 else f"{fov_idx:04d}" stem = prefix + fmt + suffix dax_path = TMP_DIR / (stem + ".dax") @@ -149,8 +180,11 @@ def download_fov_image(fov_idx, n_format, prefix, suffix, base_url, frames, stai return tifffile.imread(tif_path) if not dax_path.exists(): - urllib.request.urlretrieve(base_url + stem + ".dax", dax_path) - urllib.request.urlretrieve(base_url + stem + ".inf", inf_path) + if not robust_urlretrieve(base_url + stem + ".dax", dax_path): + return None + if not robust_urlretrieve(base_url + stem + ".inf", inf_path): + dax_path.unlink(missing_ok=True) + return None reader = DaxReader(str(dax_path)) img = np.array([reader.load_frame(f) for f in frames]) @@ -288,6 +322,9 @@ def um_to_px_y(y): yp, yp_max = int(row["y_min"]), int(row["y_max"]) fov_img = download_fov_image(fov, n_format, dapi_file_prefix, download_suffix, image_base_url, dapi_frames, "DAPI") + if fov_img is None: + print(datetime.now() - t0, f"Skipping FOV {fov}: image unavailable", flush=True) + continue # Find neighbors for linear blending at overlaps fov_right = fov_df[ @@ -304,8 +341,11 @@ def um_to_px_y(y): right = len(fov_right) > 0 and fov_right[0] not in missing_fovs bottom = len(fov_bottom) > 0 and fov_bottom[0] not in missing_fovs + bot_img = right_img = None if bottom: bot_img = download_fov_image(int(fov_bottom[0]), n_format, dapi_file_prefix, download_suffix, image_base_url, dapi_frames, "DAPI") + bottom = bot_img is not None + if bottom: fov_img[:, FRAME_SIZE_PIXELS - OVERLAP_SIZE:, :] = ( np.multiply(fov_img[:, FRAME_SIZE_PIXELS - OVERLAP_SIZE:, :], w1) + np.multiply(bot_img[:, :OVERLAP_SIZE, :], w2) @@ -313,13 +353,17 @@ def um_to_px_y(y): if right: right_img = download_fov_image(int(fov_right[0]), n_format, dapi_file_prefix, download_suffix, image_base_url, dapi_frames, "DAPI") + right = right_img is not None + if right: fov_img[:, :, FRAME_SIZE_PIXELS - OVERLAP_SIZE:] = ( np.multiply(fov_img[:, :, FRAME_SIZE_PIXELS - OVERLAP_SIZE:], w1.T) + np.multiply(right_img[:, :, :OVERLAP_SIZE], w2.T) ) / (w1.T + w2.T) + br_img = None if len(fov_br) > 0 and fov_br[0] not in missing_fovs: br_img = download_fov_image(int(fov_br[0]), n_format, dapi_file_prefix, download_suffix, image_base_url, dapi_frames, "DAPI") + if br_img is not None: sl_x = slice(FRAME_SIZE_PIXELS - OVERLAP_SIZE, None) sl_y = slice(FRAME_SIZE_PIXELS - OVERLAP_SIZE, None) if bottom and right: diff --git a/src/methods_segmentation/cellpose/config.vsh.yaml b/src/methods_segmentation/cellpose/config.vsh.yaml index 1187c12a9..e0aad23bd 100644 --- a/src/methods_segmentation/cellpose/config.vsh.yaml +++ b/src/methods_segmentation/cellpose/config.vsh.yaml @@ -93,4 +93,4 @@ runners: - type: executable - type: nextflow directives: - label: [ midtime, midcpu, highmem, gpu ] + label: [ midtime, midcpu, highmem, gpuhighmem ] From 442e48aa2870f27d1ca7afb0f62cc9173d043420 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 6 Jul 2026 22:21:42 +0200 Subject: [PATCH 5/7] add bruker mirror script --- .../spatial/mirror_bruker_to_s3.sh | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 scripts/create_resources/spatial/mirror_bruker_to_s3.sh diff --git a/scripts/create_resources/spatial/mirror_bruker_to_s3.sh b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh new file mode 100644 index 000000000..4a02bf7db --- /dev/null +++ b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +# Mirror the Bruker CosMx raw zips from the (flaky) NanoString/liquidweb server to S3. +# +# The NanoString server frequently resets the connection mid-transfer, and these +# zips are enormous (HalfBrain ~175 GB, NormalLiver ~349 GB). Nextflow's foreign-file +# staging does NOT resume, so it dies on every run. Mirroring to S3 once makes future +# runs fast and deterministic. +# +# This script is resumable and idempotent: +# - curl -C - resumes a partial download after a connection reset +# - files already present in S3 (with matching size) are skipped +# - each file is deleted from local scratch after upload, so you only ever need +# free disk for the LARGEST single file (~350 GB), not the sum +# +# Just re-run it if it dies; it picks up where it left off. +# +# Usage: +# SCRATCH_DIR=/path/to/big/scratch ./mirror_bruker_to_s3.sh + +set -euo pipefail + +# --- Config ----------------------------------------------------------------- + +SRC_BASE="https://smi-public.objects.liquidweb.services" +S3_DEST="${S3_DEST:-s3://openproblems-data/resources_raw/bruker_cosmx}" +SCRATCH_DIR="${SCRATCH_DIR:-$PWD/bruker_mirror_scratch}" + +# Files to mirror. The URL-encoded names are what the server serves; the second +# column is the (decoded) name to store under on S3. +FILES=( + "HalfBrain.zip|HalfBrain.zip" + "Half%20%20Brain%20simple%20%20files%20.zip|Half Brain simple files.zip" + "NormalLiverFiles.zip|NormalLiverFiles.zip" +) + +# --- Preflight -------------------------------------------------------------- + +command -v curl >/dev/null || { echo "ERROR: curl not found" >&2; exit 1; } +command -v aws >/dev/null || { echo "ERROR: aws CLI not found" >&2; exit 1; } + +mkdir -p "$SCRATCH_DIR" +echo "Scratch dir : $SCRATCH_DIR" +echo "S3 dest : $S3_DEST" +echo "Free space :" +df -h "$SCRATCH_DIR" | sed 's/^/ /' +echo + +remote_size() { + # Content-Length of the remote file, or empty if unavailable + curl -sSL -I --max-time 60 "$1" \ + | tr -d '\r' \ + | awk 'tolower($1)=="content-length:"{print $2}' \ + | tail -n1 +} + +s3_size() { + # Size of the object in S3, or empty if it doesn't exist + aws s3api head-object --bucket "$1" --key "$2" --query 'ContentLength' --output text 2>/dev/null || true +} + +# --- Main loop -------------------------------------------------------------- + +for entry in "${FILES[@]}"; do + url_name="${entry%%|*}" + local_name="${entry##*|}" + url="$SRC_BASE/$url_name" + local_path="$SCRATCH_DIR/$local_name" + + # S3 key = everything after s3://bucket/ + bucket="$(echo "$S3_DEST" | sed -E 's#^s3://([^/]+)/.*#\1#')" + prefix="$(echo "$S3_DEST" | sed -E 's#^s3://[^/]+/(.*)#\1#')" + key="$prefix/$local_name" + + echo "================================================================" + echo "File: $local_name" + echo " URL: $url" + echo " S3 : s3://$bucket/$key" + + expected="$(remote_size "$url")" + if [[ -z "$expected" ]]; then + echo " WARN: could not read remote Content-Length; proceeding without size checks" + else + echo " Remote size: $expected bytes ($(awk -v b="$expected" 'BEGIN{printf "%.1f GB", b/1024/1024/1024}'))" + fi + + # Skip if already uploaded with the right size + existing="$(s3_size "$bucket" "$key")" + if [[ -n "$existing" && "$existing" != "None" ]]; then + if [[ -z "$expected" || "$existing" == "$expected" ]]; then + echo " SKIP: already in S3 (size $existing bytes)" + continue + else + echo " Present in S3 but size mismatch ($existing != $expected); re-uploading" + fi + fi + + # Download with resume + aggressive retry on connection resets + echo " Downloading (resumable)..." + curl -L -C - \ + --retry 100 --retry-all-errors --retry-delay 10 \ + --connect-timeout 60 \ + -o "$local_path" \ + "$url" + + # Verify size before uploading + if [[ -n "$expected" ]]; then + actual="$(stat -c%s "$local_path" 2>/dev/null || stat -f%z "$local_path")" + if [[ "$actual" != "$expected" ]]; then + echo " ERROR: downloaded size $actual != expected $expected. Re-run to resume." >&2 + exit 1 + fi + echo " Size verified: $actual bytes" + fi + + # Upload to S3 + echo " Uploading to s3://$bucket/$key ..." + aws s3 cp "$local_path" "s3://$bucket/$key" + + # Free scratch space before the next (much larger) file + echo " Removing local copy to free space" + rm -f "$local_path" + echo " Done: $local_name" +done + +echo "================================================================" +echo "All files mirrored to $S3_DEST" +echo +echo "Next: update the input_raw / input_flat_files URLs in" +echo " scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh" +echo "to point at the S3 paths, e.g.:" +echo " input_raw: $S3_DEST/HalfBrain.zip" +echo " input_flat_files: $S3_DEST/Half Brain simple files.zip" +echo " input_raw: $S3_DEST/NormalLiverFiles.zip" From 44bf8a523fc27c8623e435a5ed37f522a5c58599 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 7 Jul 2026 02:26:18 +0200 Subject: [PATCH 6/7] fix label issue --- src/methods_segmentation/cellpose/script.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/methods_segmentation/cellpose/script.py b/src/methods_segmentation/cellpose/script.py index 190f80da3..2f33dd2aa 100644 --- a/src/methods_segmentation/cellpose/script.py +++ b/src/methods_segmentation/cellpose/script.py @@ -47,9 +47,25 @@ def convert_to_lower_dtype(arr): parsed_data = Labels2DModel.parse(data_array, transformations=transformation) sd_output.labels['segmentation'] = parsed_data +metadata = sdata.tables["metadata"] +# cell_id is required downstream. Standard Xenium exports carry an explicit +# "cell_id" column, but some exports (e.g. the Xenium WTA preview used for the +# Atera dataset) don't — there the per-cell identifier lives in the table's +# instance_key column (falling back to the obs index). +instance_key = metadata.uns.get("spatialdata_attrs", {}).get("instance_key") +if "cell_id" in metadata.obs.columns: + cell_id = metadata.obs["cell_id"].values +elif instance_key and instance_key in metadata.obs.columns: + cell_id = metadata.obs[instance_key].values +else: + cell_id = metadata.obs.index.values +obs = metadata.obs[[]].copy() +obs["cell_id"] = cell_id +if "region" in metadata.obs.columns: + obs["region"] = metadata.obs["region"].values sd_output.tables['table'] = ad.AnnData( - obs=sdata.tables["metadata"].obs[["cell_id", "region"]], - var=sdata.tables["metadata"].var[[]] + obs=obs, + var=metadata.var[[]] ) print("Writing output", flush=True) From ea9b48a93f6caa32fd03ef2f31a45c3c344f8ce4 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 7 Jul 2026 02:26:50 +0200 Subject: [PATCH 7/7] adjust memory gpu --- src/methods_cell_type_annotation/moscot/config.vsh.yaml | 2 +- src/methods_cell_type_annotation/tangram/config.vsh.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/methods_cell_type_annotation/moscot/config.vsh.yaml b/src/methods_cell_type_annotation/moscot/config.vsh.yaml index 27a02cedc..57d041245 100644 --- a/src/methods_cell_type_annotation/moscot/config.vsh.yaml +++ b/src/methods_cell_type_annotation/moscot/config.vsh.yaml @@ -57,4 +57,4 @@ runners: - type: executable - type: nextflow directives: - label: [ hightime, midcpu, veryhighmem, gpu ] + label: [ hightime, midcpu, veryhighmem, gpuhighmem ] diff --git a/src/methods_cell_type_annotation/tangram/config.vsh.yaml b/src/methods_cell_type_annotation/tangram/config.vsh.yaml index 447a3617c..714f7a2d2 100644 --- a/src/methods_cell_type_annotation/tangram/config.vsh.yaml +++ b/src/methods_cell_type_annotation/tangram/config.vsh.yaml @@ -40,4 +40,4 @@ runners: - type: executable - type: nextflow directives: - label: [ midtime, midcpu, midmem, gpu ] + label: [ midtime, midcpu, midmem, gpuhighmem ]