From 8985867a42f0151956b54ae1fe0029f9befa7bf6 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 2 Jul 2026 11:50:57 +0200 Subject: [PATCH 1/2] adjust mem --- src/datasets/loaders/tenx_atera/config.vsh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/loaders/tenx_atera/config.vsh.yaml b/src/datasets/loaders/tenx_atera/config.vsh.yaml index 6da3a43bf..ed74ffeac 100644 --- a/src/datasets/loaders/tenx_atera/config.vsh.yaml +++ b/src/datasets/loaders/tenx_atera/config.vsh.yaml @@ -68,4 +68,4 @@ runners: - type: executable - type: nextflow directives: - label: [midmem, midcpu, midtime] \ No newline at end of file + label: [highmem, midcpu, midtime] \ No newline at end of file From 76335233f7bf5aa333c72b3c551e0d1b4bd04183 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 2 Jul 2026 15:18:35 +0200 Subject: [PATCH 2/2] Publish atera loader output to local raw/ folder; update nebius labels Co-Authored-By: Claude Opus 4.8 --- .../spatial/process_10x_atera_nebius.sh | 4 +++- src/base/labels_nebius.config | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/create_resources/spatial/process_10x_atera_nebius.sh b/scripts/create_resources/spatial/process_10x_atera_nebius.sh index d3475e586..432122295 100644 --- a/scripts/create_resources/spatial/process_10x_atera_nebius.sh +++ b/scripts/create_resources/spatial/process_10x_atera_nebius.sh @@ -8,7 +8,9 @@ cd "$REPO_ROOT" set -e -publish_dir="s3://openproblems-data/resources/datasets" +# store the loader output locally, mirroring the process_datasets layout ($id/) +# but under a sibling raw/ folder +publish_dir="/scratch/task_ist_preprocessing/raw" cat > /tmp/params_atera.yaml << HERE param_list: diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index fd98f257d..de49246a0 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -47,20 +47,26 @@ process { withLabel: midcpu { cpus = 15 } withLabel: highcpu { cpus = 30 } withLabel: lowmem { - memory = { get_memory( 20.GB * task.attempt ) } + // Nebius 48vcpu-192gb nodes: 188 GB allocatable + memory = { get_memory( 25.GB * task.attempt ) } disk = { 50.GB * task.attempt } + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00p5wjvb7bc55b2js']] } withLabel: midmem { + // Nebius 64vcpu-256gb nodes: 251 GB allocatable memory = { get_memory( 50.GB * task.attempt ) } disk = { 100.GB * task.attempt } + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00wvqfyde1nfwezs0']] } withLabel: highmem { - memory = { task.attempt == 1 ? 200.GB : 500.GB } + // Nebius 128vcpu-512gb nodes: 503 GB allocatable — cap well below node max + memory = { task.attempt == 1 ? 200.GB : 450.GB } disk = 200.GB pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00hnqhyfdcsy9m09n']] } withLabel: veryhighmem { - memory = { get_memory( 200.GB * task.attempt ) } + // Nebius 128vcpu-512gb nodes: 503 GB allocatable — cap at 450 GB to leave headroom + memory = { [ 200.GB * task.attempt, 450.GB ].min() } disk = { 400.GB * task.attempt } pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00hnqhyfdcsy9m09n']] }