From db57dbca604da4d8fbf677e9a4006902794c0e35 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sun, 11 Oct 2026 21:56:35 +0200 Subject: [PATCH 01/18] Add a launcher that runs FastSurfer in its container image --- doc/overview/CONTAINERS.md | 30 ++ test/lint/test_launcher_builds.py | 54 +++ test/utils/test_fastsurfer_container.py | 148 ++++++++ tools/fastsurfer_container.py | 445 ++++++++++++++++++++++++ 4 files changed, 677 insertions(+) create mode 100644 test/lint/test_launcher_builds.py create mode 100644 test/utils/test_fastsurfer_container.py create mode 100644 tools/fastsurfer_container.py diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index f70e7146b..de5283724 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -19,6 +19,36 @@ define access to data and hardware such as graphics cards, and the options of Fa the container passes on to `run_fastsurfer.sh`. It is useful to think of the image as a "hard drive" and the container as a "simulated computer inside the computer". +The launcher +------------ +`fastsurfer_container.py` writes and runs the container command for you. It takes the same flags as +`run_fastsurfer.sh` and needs only Python 3.8 or newer, not FastSurfer itself. It + +- uses Apptainer, Singularity or Docker, whichever is installed (choose one with `--runtime`), +- picks the image that runs on your NVIDIA GPU and driver, or the CPU image without one, see + [which image fits your GPU](install/LINUX.md#nvidia-gpus), +- downloads the image once (for Apptainer, into `$HOME/my_singularity_images`), +- shares the files and directories the flags name with the container, at the same paths, so relative + paths work too, +- passes the FreeSurfer license from `$FS_LICENSE` if `--fs_license` is not given, and runs the + container as you. + +Download it, and run FastSurfer: + +```bash +curl -fsSLO https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py +python3 fastsurfer_container.py --image_version {{ FASTSURFER_VERSION }} \ + --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ + --sid subjectX --sd $HOME/my_fastsurfer_analysis \ + --3T --threads 4 +``` + +`--image_version` selects the FastSurfer release; in a FastSurfer checkout, the launcher in `tools/` +defaults to the version of the checkout. With `--brun`, it runs `brun_fastsurfer.sh` for several +subjects; the paths in a subjects list have to be absolute. `--print_only` prints the commands +instead of running them, and `--help` lists all launcher options. The sections below explain the +commands it runs. + Using Apptainer (or Singularity) -------------------------------- [Apptainer](https://apptainer.org) is the open-source continuation of Singularity, and both run the same images. diff --git a/test/lint/test_launcher_builds.py b/test/lint/test_launcher_builds.py new file mode 100644 index 000000000..64c3912d5 --- /dev/null +++ b/test/lint/test_launcher_builds.py @@ -0,0 +1,54 @@ +# Copyright 2026 DeepMI Lab, German Center for Neurodegenerative Diseases (DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Guard the copy of the CUDA images in tools/fastsurfer_container.py. + +The launcher runs with any system Python and without FastSurfer installed, so it cannot import +FastSurferCNN/gpu_support.py and keeps its own copy of the CUDA builds and their minimum drivers. +A copy left behind when the CUDA versions change makes it pick an image that is not released. +""" + +import importlib.util +import sys +import tomllib +from pathlib import Path + +FASTSURFER_HOME = Path(__file__).parent.parent.parent + + +def _load(name: str, path: Path): + # from the path, because the lint job installs nothing but pytest; registered in sys.modules, + # which dataclasses need + spec = importlib.util.spec_from_file_location(name, path) + module = sys.modules.setdefault(name, importlib.util.module_from_spec(spec)) + spec.loader.exec_module(module) + return module + + +launcher = _load("fastsurfer_container", FASTSURFER_HOME / "tools" / "fastsurfer_container.py") +gpu_support = _load("gpu_support_for_lint", FASTSURFER_HOME / "FastSurferCNN" / "gpu_support.py") + + +def test_launcher_builds_match_gpu_support(): + assert launcher.NEWEST_BUILD == gpu_support.NEWEST_BUILD + assert launcher.LEGACY_BUILD == gpu_support.LEGACY_BUILD + for (major, _), _ in (launcher.NEWEST_BUILD, launcher.LEGACY_BUILD): + assert launcher.MIN_DRIVER[major] == gpu_support.MIN_DRIVER[major] + + +def test_launcher_default_build_matches_pyproject(): + with open(FASTSURFER_HOME / "pyproject.toml", "rb") as fp: + cuda = tomllib.load(fp)["tool"]["cuda"]["version"] + assert launcher.NEWEST_BUILD[0] == tuple(int(v) for v in cuda.split(".")) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py new file mode 100644 index 000000000..773871115 --- /dev/null +++ b/test/utils/test_fastsurfer_container.py @@ -0,0 +1,148 @@ +# Copyright 2026 DeepMI Lab, German Center for Neurodegenerative Diseases (DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the image choice and the path handling of tools/fastsurfer_container.py.""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +# a script rather than a module of FastSurfer, so it is loaded from its path; dataclasses need it +# registered in sys.modules +_SCRIPT = Path(__file__).parent.parent.parent / "tools" / "fastsurfer_container.py" +_spec = importlib.util.spec_from_file_location("fastsurfer_container", _SCRIPT) +launcher = sys.modules.setdefault(_spec.name, importlib.util.module_from_spec(_spec)) +_spec.loader.exec_module(launcher) + + +@pytest.mark.parametrize( + ("gpu", "prefix"), + [ + (((8, 6), 580), "cu132"), # Ampere, current driver + (((12, 0), 590), "cu132"), # Blackwell + (((8, 6), 550), "cu126"), # driver too old for CUDA 13 + (((6, 1), 580), "cu126"), # Pascal, dropped by CUDA 13 + (((7, 0), 600), "cu126"), # Volta + ((None, 580), "cu132"), # capability unknown, the driver decides + ((None, 550), "cu126"), + ], +) +def test_choose_build_gpu(gpu, prefix): + build = launcher.choose_build(gpu, "auto") + assert (build.prefix, build.gpu) == (prefix, True) + + +def test_choose_build_explains_the_older_image(): + build = launcher.choose_build(((8, 6), 550), "auto") + assert build.notes and "550" in build.notes[0] + assert launcher.choose_build(((6, 1), 580), "auto").notes == [] + + +@pytest.mark.parametrize( + "gpu", + [ + ((12, 0), 550), # Blackwell needs CUDA 13, whose driver is missing + ((3, 7), 580), # Kepler, no image supports it + ((8, 6), 470), # driver too old for every image + ], +) +def test_choose_build_falls_back_to_cpu(gpu): + build = launcher.choose_build(gpu, "auto") + assert (build.prefix, build.gpu) == ("cpu", False) + assert build.notes + with pytest.raises(launcher.LauncherError): + launcher.choose_build(gpu, "cuda") + + +def test_choose_build_cpu(): + assert launcher.choose_build(((8, 6), 580), "cpu").prefix == "cpu" + assert launcher.choose_build(None, "auto").prefix == "cpu" + with pytest.raises(launcher.LauncherError): + launcher.choose_build(None, "cuda:0") + + +def test_image_reference(): + assert launcher.image_reference("cu132", "2.6.0") == "deepmi/fastsurfer:cu132-v2.6.0" + for version in (None, "2.7.0-dev0"): + with pytest.raises(launcher.LauncherError): + launcher.image_reference("cpu", version) + + +def test_read_image_version(tmp_path): + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text("[build-system]\nversion = '9'\n\n[project]\nname = 'x'\nversion = '2.6.0'\n\n[tool.a]\n" + "version = '13.2'\n") + assert launcher.read_image_version(pyproject) == "2.6.0" + assert launcher.read_image_version(tmp_path / "missing.toml") is None + + +def test_plan_paths_run(tmp_path): + data = tmp_path / "data" + data.mkdir() + (data / "t1.mgz").write_bytes(b"") + license_file = tmp_path / "license.txt" + license_file.write_text("") + args = ["--t1", "data/t1.mgz", "--sid", "data", "--sd", "data/out", "--fs_license", str(license_file), + "--conformed_name", "elsewhere/orig.mgz", "--3T"] + out, mounts = launcher.plan_paths(args, tmp_path) + assert out == ["--t1", str(data / "t1.mgz"), "--sid", "data", "--sd", str(data / "out"), + "--fs_license", str(license_file), "--conformed_name", str(tmp_path / "elsewhere/orig.mgz"), + "--3T"] + # the output directories are created, because a mount needs them to exist + assert (data / "out").is_dir() and (tmp_path / "elsewhere").is_dir() + # data/out is inside data, and everything is inside tmp_path once the license is shared + assert launcher.minimal_mounts(mounts) == [tmp_path] + assert launcher.minimal_mounts([data / "out", data, tmp_path / "elsewhere"]) == [data, tmp_path / "elsewhere"] + + +def test_plan_paths_missing_input(tmp_path): + with pytest.raises(launcher.LauncherError): + launcher.plan_paths(["--t1", "missing.mgz", "--sd", "out"], tmp_path) + + +def test_plan_paths_brun(tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir() + b.mkdir() + (a / "t1.mgz").write_bytes(b"") + (b / "t2.nii.gz").write_bytes(b"") + subjects = tmp_path / "list" / "subjects.txt" + subjects.parent.mkdir() + subjects.write_text(f"# comment\n\nsub1={a / 't1.mgz'} --t2 '{b / 't2.nii.gz'}'\n") + _, mounts = launcher.plan_paths(["--subjects_list", str(subjects), "--sd", str(tmp_path / "out")], tmp_path) + assert set(launcher.minimal_mounts(mounts)) == {a, b, subjects.parent, tmp_path / "out"} + + out, mounts = launcher.plan_paths(["--subjects", "sub1=a/t1.mgz", "--sd", "out"], tmp_path) + assert out[:2] == ["--subjects", f"sub1={a / 't1.mgz'}"] + + # relative paths in a list cannot be resolved inside the container + subjects.write_text("sub1=a/t1.mgz\n") + with pytest.raises(launcher.LauncherError): + launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) + + +def test_commands(tmp_path): + args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] + docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") + assert docker == ["docker", "run", "--rm", "--user", "1000:1000", "--gpus", "all", + "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") + assert brun[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", + "/fastsurfer/brun_fastsurfer.sh"] + sif = tmp_path / "fs.sif" + apptainer = launcher.apptainer_command("apptainer", sif, False, [tmp_path], "run_fastsurfer.sh", args, []) + assert apptainer == ["apptainer", "exec", "--no-mount", "home,cwd", "-e", "-B", str(tmp_path), str(sif), + "/fastsurfer/run_fastsurfer.sh"] + args diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py new file mode 100644 index 000000000..a2913da68 --- /dev/null +++ b/tools/fastsurfer_container.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +# Copyright 2026 DeepMI Lab, German Center for Neurodegenerative Diseases (DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run FastSurfer in its container image, without writing the container command. + +Takes the flags of run_fastsurfer.sh (or, with --brun, of brun_fastsurfer.sh) and runs them in the +official image with Apptainer, Singularity or Docker, whichever is installed: + + python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 + +It picks the image that fits the NVIDIA GPU and driver (or the CPU image), downloads it once, +shares every file and directory the flags name with the container at the same path, passes the +FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the +calling user. + +Only the Python standard library is used, so this runs with the system Python of a cluster, +without FastSurfer installed. +""" + +from __future__ import annotations + +import argparse +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +DOCKER_REPO = "deepmi/fastsurfer" +DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" + +# The CUDA images, as in FastSurferCNN/gpu_support.py: (CUDA version, tag prefix), the oldest GPU +# each supports and the newest the legacy one does. The test in test/lint keeps them in sync. +NEWEST_BUILD = ((13, 2), "cu132") +LEGACY_BUILD = ((12, 6), "cu126") +MIN_DRIVER = {12: 525, 13: 580} +MIN_CAPABILITY = {"cu132": (7, 5), "cu126": (5, 0)} +LEGACY_MAX_CAPABILITY = (9, 0) + +# flags of run_fastsurfer.sh and brun_fastsurfer.sh whose value is a file that has to exist +INPUT_FLAGS = {"--t1", "--t2", "--lesion_mask", "--fs_license", "--subjects_list", "--subject_list"} +# flags whose value is a file or directory FastSurfer writes, so its directory is created if missing +OUTPUT_FLAGS = { + "--sd", "--seg_log", "--conformed_name", "--norm_name", "--norm_name_t2", "--seg", + "--asegdkt_segfile", "--aparc_aseg_segfile", "--asegdkt_statsfile", "--aseg_statsfile", + "--aseg_segfile", "--mask_name", "--cereb_segfile", "--cereb_statsfile", "--hypo_segfile", + "--hypo_statsfile", "--statusfile", +} + + +class LauncherError(Exception): + """A problem with the request, reported as one message without a traceback.""" + + +@dataclass +class Build: + """The image build to run, and whether the container gets the GPU.""" + + prefix: str + gpu: bool + notes: list[str] = field(default_factory=list) + + +def read_image_version(pyproject: Path) -> str | None: + """The FastSurfer version in the pyproject.toml of the checkout this file is in, if there is one.""" + try: + text = pyproject.read_text() + except OSError: + return None + project = re.search(r"^\[project\]\s*$(.*?)(?=^\[|\Z)", text, re.MULTILINE | re.DOTALL) + match = project and re.search(r"^version\s*=\s*['\"]([^'\"]+)['\"]", project.group(1), re.MULTILINE) + return match.group(1) if match else None + + +def query_nvidia_gpu(index: int = 0) -> tuple[tuple[int, int] | None, int] | None: + """ + The compute capability and the driver version of an NVIDIA GPU, from nvidia-smi. + + Returns None without an NVIDIA GPU. The capability is None if this driver cannot report it. + """ + if shutil.which("nvidia-smi") is None: + return None + for fields in ("compute_cap,driver_version", "driver_version"): + try: + out = subprocess.run( + ["nvidia-smi", f"--query-gpu={fields}", "--format=csv,noheader"], + capture_output=True, text=True, check=True, timeout=30, + ).stdout.strip().splitlines() + except (OSError, subprocess.SubprocessError): + continue + if index >= len(out): + return None + values = [v.strip() for v in out[index].split(",")] + driver = int(values[-1].split(".")[0]) + if len(values) == 1: + return None, driver + major, minor = values[0].split(".") + return (int(major), int(minor)), driver + return None + + +def choose_build(gpu: tuple[tuple[int, int] | None, int] | None, device: str) -> Build: + """ + The image that runs on this GPU and driver, preferring the default CUDA version. + + Parameters + ---------- + gpu : tuple, None + The compute capability (None if unknown) and the driver major version, or None without a GPU. + device : str + The value of FastSurfer's --device, "auto" if it was not given. + + Returns + ------- + Build + The tag prefix, whether to give the container the GPU, and notes for the user. + """ + if device == "cpu": + return Build("cpu", False) + if gpu is None: + if device.startswith("cuda"): + raise LauncherError(f"--device {device} needs an NVIDIA GPU, but nvidia-smi found none.") + return Build("cpu", False, ["No NVIDIA GPU found, so FastSurfer runs on the CPU."]) + capability, driver = gpu + + def runs(build: tuple[tuple[int, int], str]) -> bool: + (cuda_major, _), prefix = build + if driver < MIN_DRIVER[cuda_major]: + return False + if capability is None: + return True + if prefix == LEGACY_BUILD[1] and capability > LEGACY_MAX_CAPABILITY: + return False + return capability >= MIN_CAPABILITY[prefix] + + if runs(NEWEST_BUILD): + return Build(NEWEST_BUILD[1], True) + newest_min_driver = MIN_DRIVER[NEWEST_BUILD[0][0]] + if runs(LEGACY_BUILD): + notes = [] + if driver < newest_min_driver and (capability is None or capability >= MIN_CAPABILITY[NEWEST_BUILD[1]]): + notes.append( + f"The NVIDIA driver {driver} is older than {newest_min_driver}, so this uses the image for " + f"CUDA {LEGACY_BUILD[0][0]}.{LEGACY_BUILD[0][1]}. Update the driver to use the default image." + ) + return Build(LEGACY_BUILD[1], True, notes) + if capability is not None and capability < MIN_CAPABILITY[LEGACY_BUILD[1]]: + reason = f"The GPU (compute capability {capability[0]}.{capability[1]}) is too old for FastSurfer's images" + else: + reason = (f"The NVIDIA driver {driver} is too old for FastSurfer's images, update it to " + f"{newest_min_driver} or newer") + if device.startswith("cuda"): + raise LauncherError(f"{reason}.") + return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) + + +def image_reference(prefix: str, version: str | None) -> str: + """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" + if version is None: + raise LauncherError( + "Which FastSurfer version to run is unknown, because this file is not part of a FastSurfer " + "checkout. Pass it with --image_version, for example --image_version 2.6.0." + ) + if "dev" in version: + raise LauncherError( + f"There are no images of the development version {version}. Pass a released version with " + f"--image_version, or your own image with --image." + ) + return f"{DOCKER_REPO}:{prefix}-v{version}" + + +def _absolute(value: str, cwd: Path) -> Path: + return Path(os.path.normpath(cwd / os.path.expanduser(value))) + + +def _parent_of_file(path: Path) -> Path: + return path if path.is_dir() else path.parent + + +def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[str], list[Path]]: + """ + Make the paths in FastSurfer's arguments absolute and find the directories to share. + + Parameters + ---------- + args : list[str] + The arguments for run_fastsurfer.sh or brun_fastsurfer.sh. + cwd : Path + The directory relative paths are relative to. + create : bool, default=True + Whether to create missing output directories (they have to exist to be shared). + + Returns + ------- + list[str] + The arguments, with every path absolute. + list[Path] + The directories to share with the container, at the same path. + """ + out: list[str] = [] + mounts: list[Path] = [] + i = 0 + while i < len(args): + flag = args[i] + out.append(flag) + i += 1 + if flag in INPUT_FLAGS | OUTPUT_FLAGS and i < len(args): + path = _absolute(args[i], cwd) + if flag in INPUT_FLAGS: + if not path.exists(): + raise LauncherError(f"{flag} {args[i]}: no such file.") + mounts.append(_parent_of_file(path)) + if flag in ("--subjects_list", "--subject_list"): + mounts.extend(_subjects_list_mounts(path)) + else: + directory = path if flag == "--sd" else path.parent + if create: + directory.mkdir(parents=True, exist_ok=True) + mounts.append(directory) + out.append(str(path)) + i += 1 + elif flag == "--subjects": + # brun_fastsurfer.sh: = ... until the next flag + while i < len(args) and not args[i].startswith("--"): + subject, sep, t1 = args[i].partition("=") + if not sep: + raise LauncherError(f"--subjects {args[i]}: expected =.") + path = _absolute(t1, cwd) + if not path.exists(): + raise LauncherError(f"--subjects {args[i]}: no such file {t1}.") + mounts.append(path.parent) + out.append(f"{subject}={path}") + i += 1 + return out, mounts + + +def _subjects_list_mounts(subjects_list: Path) -> list[Path]: + """The directories of the files a brun_fastsurfer.sh subjects list names, which must be absolute.""" + mounts = [] + for number, line in enumerate(subjects_list.read_text().splitlines(), start=1): + tokens = shlex.split(line, comments=True) + if not tokens: + continue + _, _, t1 = tokens[0].partition("=") + paths = [t1] + [tokens[k + 1] for k, tok in enumerate(tokens[1:-1], start=1) if tok in INPUT_FLAGS] + for value in paths: + if not os.path.isabs(os.path.expanduser(value)): + raise LauncherError( + f"{subjects_list}, line {number}: {value} is a relative path. Use absolute paths in a " + f"subjects list, the container does not start in the current directory." + ) + mounts.append(_parent_of_file(Path(os.path.expanduser(value)))) + return mounts + + +def minimal_mounts(paths: list[Path]) -> list[Path]: + """The given directories without duplicates and without those inside another one.""" + kept: list[Path] = [] + for path in sorted(set(paths), key=lambda p: len(p.parts)): + if not any(path == k or k in path.parents for k in kept): + kept.append(path) + return sorted(kept) + + +def find_runtime(requested: str) -> str: + """The container runtime to use: Apptainer first, then Singularity, then Docker.""" + names = ["apptainer", "singularity", "docker"] if requested == "auto" else [requested] + for name in names: + if shutil.which(name): + return name + if requested == "auto": + raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") + raise LauncherError(f"{requested} is not installed.") + + +def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], script: str, + args: list[str], extra: list[str]) -> list[str]: + """The Apptainer/Singularity command; the home directory is not mounted, see the docs.""" + cmd = [runtime, "exec"] + (["--nv"] if gpu else []) + ["--no-mount", "home,cwd", "-e"] + for mount in mounts: + cmd += ["-B", str(mount)] + return cmd + extra + [str(sif), f"/fastsurfer/{script}"] + args + + +def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: list[str], + extra: list[str], user: str) -> list[str]: + """The Docker command, running as the calling user.""" + cmd = ["docker", "run", "--rm", "--user", user] + (["--gpus", "all"] if gpu else []) + for mount in mounts: + cmd += ["-v", f"{mount}:{mount}"] + cmd += extra + if script != "run_fastsurfer.sh": + # the image's entrypoint runs run_fastsurfer.sh; keep its environment setup for the other scripts + return cmd + ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", image, f"/fastsurfer/{script}"] + args + return cmd + [image] + args + + +def _fastsurfer_device(args: list[str]) -> str: + for i in range(len(args) - 1): + if args[i] == "--device": + return args[i + 1] + return "auto" + + +def _license_args(args: list[str], cwd: Path) -> list[str]: + """--fs_license from $FS_LICENSE or $FREESURFER_HOME, if the arguments have none.""" + if "--fs_license" in args: + return [] + candidates = [os.environ.get("FS_LICENSE", "")] + if os.environ.get("FREESURFER_HOME"): + candidates += [os.path.join(os.environ["FREESURFER_HOME"], name) for name in ("license.txt", ".license")] + for candidate in candidates: + if candidate and _absolute(candidate, cwd).is_file(): + return ["--fs_license", candidate] + return [] + + +def make_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="fastsurfer_container.py", + description=__doc__.split("\n\n")[0].strip(), + epilog=( + "All other arguments are passed to run_fastsurfer.sh (or brun_fastsurfer.sh), see " + "https://deep-mi.org/FastSurfer/stable/scripts/RUN_FASTSURFER.html. Relative paths are " + "relative to the current directory." + ), + allow_abbrev=False, + ) + parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", + help="the container runtime (default: auto, the first installed of these)") + parser.add_argument("--image_version", + help="the FastSurfer version to run, e.g. 2.6.0 (default: the version of this checkout)") + parser.add_argument("--image", + help="run this image instead: a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer " + ".sif file") + parser.add_argument("--image_dir", type=Path, + default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), + help="where Apptainer/Singularity images are stored (default: $FASTSURFER_IMAGE_DIR or " + "~/my_singularity_images)") + parser.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") + parser.add_argument("--runtime_args", default="", + help="additional arguments for the container runtime, e.g. --runtime_args '--group-add 1001'") + parser.add_argument("--print_only", action="store_true", help="print the commands instead of running them") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = make_parser() + options, fastsurfer_args = parser.parse_known_args(argv) + if not fastsurfer_args: + parser.print_help() + return 1 + + def info(message: str) -> None: + print(f"INFO: {message}", file=sys.stderr) + + try: + if os.name != "posix": + raise LauncherError("Run this in WSL on Windows, where Docker's paths are Linux paths.") + cwd = Path.cwd() + runtime = find_runtime(options.runtime) + fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) + args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) + mounts = minimal_mounts(mounts) + if Path("/") in mounts: + raise LauncherError("The paths would share the whole file system with the container, keep the " + "data in a directory below /.") + if Path.home() in mounts: + info("This shares your home directory with the container. Settings in it may affect " + "FastSurfer, so keep the data in a subdirectory.") + if "--fs_license" not in args and "--seg_only" not in args: + info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") + if platform.system() == "Darwin" and platform.machine() == "arm64": + info("On a Mac with Apple silicon, the macOS package runs natively and uses the GPU, see " + "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html.") + + device = _fastsurfer_device(args) + if options.image: + # an own image: give it the GPU unless the run is on the CPU + build = Build("custom", device != "cpu" and query_nvidia_gpu() is not None) + image = options.image + else: + index = int(device.split(":", 1)[1]) if device.startswith("cuda:") else 0 + build = choose_build(query_nvidia_gpu(index) if device != "cpu" else None, device) + pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" + image = image_reference(build.prefix, options.image_version or read_image_version(pyproject)) + for note in build.notes: + info(note) + + script = "brun_fastsurfer.sh" if options.brun else "run_fastsurfer.sh" + extra = shlex.split(options.runtime_args) + commands = [] + if runtime == "docker": + user = f"{os.getuid()}:{os.getgid()}" + commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user)) + else: + if image.endswith(".sif"): + sif = _absolute(image, cwd) + if not sif.is_file(): + raise LauncherError(f"--image {image}: no such file.") + else: + sif = options.image_dir / (image.rsplit("/", 1)[-1].replace(":", "-") + ".sif") + if not sif.is_file(): + commands.append([runtime, "build", str(sif), f"docker://{image}"]) + commands.append(apptainer_command(runtime, sif, build.gpu, mounts, script, args, extra)) + except LauncherError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + if options.print_only: + for command in commands: + print(shlex.join(command)) + return 0 + for command in commands[:-1]: + info(f"Building the image once: {shlex.join(command)}") + Path(command[2]).parent.mkdir(parents=True, exist_ok=True) + if subprocess.run(command).returncode != 0: + print("ERROR: Building the image failed.", file=sys.stderr) + return 1 + info(f"Running: {shlex.join(commands[-1])}") + sys.stdout.flush() + sys.stderr.flush() + os.execvp(commands[-1][0], commands[-1]) + return 0 # not reached + + +if __name__ == "__main__": + raise SystemExit(main()) From cf0e303472b1fb10f0f8a2e060d0008f0ef696f1 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sun, 11 Oct 2026 22:09:07 +0200 Subject: [PATCH 02/18] Point Apple silicon Macs to the macOS package and check Docker's memory in the launcher --- doc/overview/CONTAINERS.md | 5 ++ test/utils/test_fastsurfer_container.py | 2 + tools/fastsurfer_container.py | 64 +++++++++++++++++++++---- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index de5283724..720ae2b7f 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -49,6 +49,11 @@ subjects; the paths in a subjects list have to be absolute. `--print_only` print instead of running them, and `--help` lists all launcher options. The sections below explain the commands it runs. +On a Mac with Apple silicon, use the [macOS package](install/MACOS.md) instead: the images are built for +Intel and AMD processors, which such a Mac only emulates, much more slowly and without its GPU. The +launcher therefore stops there and points to the package, unless you pass `--runtime docker`. With +Docker, it also warns if Docker may use less memory than FastSurfer needs. + Using Apptainer (or Singularity) -------------------------------- [Apptainer](https://apptainer.org) is the open-source continuation of Singularity, and both run the same images. diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 773871115..707d60eae 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -139,6 +139,8 @@ def test_commands(tmp_path): docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") assert docker == ["docker", "run", "--rm", "--user", "1000:1000", "--gpus", "all", "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + emulated = launcher.docker_command("img:tag", False, [tmp_path], "run_fastsurfer.sh", args, [], "501:20", True) + assert emulated[:7] == ["docker", "run", "--rm", "--user", "501:20", "--platform", "linux/amd64"] brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") assert brun[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", "/fastsurfer/brun_fastsurfer.sh"] diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index a2913da68..411067a91 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -45,6 +45,10 @@ DOCKER_REPO = "deepmi/fastsurfer" DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" +MACOS_PACKAGE_URL = "https://github.com/Deep-MI/FastSurfer/releases/latest/download/FastSurfer-macos-darwin_arm64.pkg" +MACOS_DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html" +# the 15 GB the macOS page asks Docker to have, less what the kernel of Docker's VM keeps for itself +MIN_DOCKER_MEMORY_GIB = 14 # The CUDA images, as in FastSurferCNN/gpu_support.py: (CUDA version, tag prefix), the oldest GPU # each supports and the newest the legacy one does. The test in test/lint keeps them in sync. @@ -300,9 +304,11 @@ def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], sc def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: list[str], - extra: list[str], user: str) -> list[str]: - """The Docker command, running as the calling user.""" + extra: list[str], user: str, emulate: bool = False) -> list[str]: + """The Docker command, running as the calling user; `emulate` runs the Intel image on Apple silicon.""" cmd = ["docker", "run", "--rm", "--user", user] + (["--gpus", "all"] if gpu else []) + if emulate: + cmd += ["--platform", "linux/amd64"] for mount in mounts: cmd += ["-v", f"{mount}:{mount}"] cmd += extra @@ -312,6 +318,32 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: return cmd + [image] + args +def is_apple_silicon() -> bool: + """Whether this is a Mac with Apple silicon, also when Python itself runs under Rosetta.""" + if platform.system() != "Darwin": + return False + if platform.machine() == "arm64": + return True + try: + out = subprocess.run(["sysctl", "-n", "hw.optional.arm64"], capture_output=True, text=True, timeout=10) + except (OSError, subprocess.SubprocessError): + return False + return out.stdout.strip() == "1" + + +def docker_memory_gib() -> float: + """The memory Docker's containers can use; on macOS and Windows, that of Docker's VM.""" + try: + out = subprocess.run(["docker", "info", "--format", "{{.MemTotal}}"], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError) as error: + raise LauncherError(f"Could not ask Docker for its settings: {error}") from error + if out.returncode != 0: + reason = (out.stderr.strip().splitlines() or ["docker info failed"])[-1] + raise LauncherError(f"Docker is installed, but not running: {reason}") + return int(out.stdout.strip()) / 2**30 + + def _fastsurfer_device(args: list[str]) -> str: for i in range(len(args) - 1): if args[i] == "--device": @@ -368,12 +400,23 @@ def main(argv: list[str] | None = None) -> int: parser.print_help() return 1 - def info(message: str) -> None: - print(f"INFO: {message}", file=sys.stderr) + def info(message: str, level: str = "INFO") -> None: + print(f"{level}: {message}", file=sys.stderr) try: if os.name != "posix": raise LauncherError("Run this in WSL on Windows, where Docker's paths are Linux paths.") + # the images are built for Intel/AMD processors, which a Mac with Apple silicon only emulates + apple_silicon = is_apple_silicon() + package_hint = (f"install the FastSurfer package for macOS instead, {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}). " + f"It runs natively and uses the Mac's GPU: in the console it opens, call run_fastsurfer.sh " + f"with the same flags.") + if apple_silicon and options.runtime == "auto": + raise LauncherError(f"On a Mac with Apple silicon, {package_hint} To run the Intel image in Docker " + f"under emulation anyway, which is much slower, pass --runtime docker.") + if apple_silicon: + info(f"Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " + f"macOS package and cannot use the GPU. Better {package_hint}", level="WARNING") cwd = Path.cwd() runtime = find_runtime(options.runtime) fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) @@ -387,9 +430,6 @@ def info(message: str) -> None: "FastSurfer, so keep the data in a subdirectory.") if "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - if platform.system() == "Darwin" and platform.machine() == "arm64": - info("On a Mac with Apple silicon, the macOS package runs natively and uses the GPU, see " - "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html.") device = _fastsurfer_device(args) if options.image: @@ -409,7 +449,15 @@ def info(message: str) -> None: commands = [] if runtime == "docker": user = f"{os.getuid()}:{os.getgid()}" - commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user)) + emulate = apple_silicon and not options.image + commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user, emulate)) + if not options.print_only: + memory = docker_memory_gib() + if memory < MIN_DOCKER_MEMORY_GIB: + where = ("in Docker's settings (Docker Desktop: Settings > Resources; Colima: colima start " + "--memory 16)") if platform.system() == "Darwin" else "for Docker" + info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " + f"15 GB. Increase the memory {where}.", level="WARNING") else: if image.endswith(".sif"): sif = _absolute(image, cwd) From 5ca25e988111a15dcbeb99deb2d9ee8b9792c5a0 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sun, 11 Oct 2026 22:26:07 +0200 Subject: [PATCH 03/18] Run the newest release in the launcher unless --image_version names another --- doc/overview/CONTAINERS.md | 7 +- test/utils/test_fastsurfer_container.py | 28 ++++---- tools/fastsurfer_container.py | 89 ++++++++++++++++++------- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 720ae2b7f..7b420c68b 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -37,14 +37,15 @@ Download it, and run FastSurfer: ```bash curl -fsSLO https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py -python3 fastsurfer_container.py --image_version {{ FASTSURFER_VERSION }} \ +python3 fastsurfer_container.py \ --t1 $HOME/my_mri_data/subjectX/t1_weighted.nii.gz \ --sid subjectX --sd $HOME/my_fastsurfer_analysis \ --3T --threads 4 ``` -`--image_version` selects the FastSurfer release; in a FastSurfer checkout, the launcher in `tools/` -defaults to the version of the checkout. With `--brun`, it runs `brun_fastsurfer.sh` for several +It runs the newest FastSurfer release and says which one. For reproducible results, keep a study on +one version with `--image_version `. Without internet, it uses the newest FastSurfer image +already on the machine. With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have to be absolute. `--print_only` prints the commands instead of running them, and `--help` lists all launcher options. The sections below explain the commands it runs. diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 707d60eae..1ac07964b 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -74,19 +74,21 @@ def test_choose_build_cpu(): launcher.choose_build(None, "cuda:0") -def test_image_reference(): - assert launcher.image_reference("cu132", "2.6.0") == "deepmi/fastsurfer:cu132-v2.6.0" - for version in (None, "2.7.0-dev0"): - with pytest.raises(launcher.LauncherError): - launcher.image_reference("cpu", version) - - -def test_read_image_version(tmp_path): - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text("[build-system]\nversion = '9'\n\n[project]\nname = 'x'\nversion = '2.6.0'\n\n[tool.a]\n" - "version = '13.2'\n") - assert launcher.read_image_version(pyproject) == "2.6.0" - assert launcher.read_image_version(tmp_path / "missing.toml") is None +def test_newest_version(): + tags = ["cu132-v2.6.0", "cu132-v2.10.1", "cu132-v2.9.3", "cu126-v2.11.0", "cu132-v2.11.0-dev0", "latest", + "cpu-latest", "cu132-v2.10.1_abc123"] + # numeric, not alphabetic: 2.10 is newer than 2.9; other images and development builds do not count + assert launcher.newest_version(tags, "cu132") == "2.10.1" + assert launcher.newest_version(tags, "cu126") == "2.11.0" + assert launcher.newest_version(tags, "cpu") is None + + +def test_local_tags_apptainer(tmp_path): + for name in ("fastsurfer-cpu-v2.5.4.sif", "fastsurfer-cpu-v2.6.0.sif", "fastsurfer-myimage.sif", "other.sif"): + (tmp_path / name).write_bytes(b"") + tags = launcher.local_tags("apptainer", tmp_path) + assert sorted(tags) == ["cpu-v2.5.4", "cpu-v2.6.0", "myimage"] + assert launcher.newest_version(tags, "cpu") == "2.6.0" def test_plan_paths_run(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 411067a91..6a3757ae7 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -21,7 +21,8 @@ python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 -It picks the image that fits the NVIDIA GPU and driver (or the CPU image), downloads it once, +It runs the newest FastSurfer release unless --image_version names another, picks the image that +fits the NVIDIA GPU and driver (or the CPU image), downloads it once, shares every file and directory the flags name with the container at the same path, passes the FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the calling user. @@ -82,15 +83,45 @@ class Build: notes: list[str] = field(default_factory=list) -def read_image_version(pyproject: Path) -> str | None: - """The FastSurfer version in the pyproject.toml of the checkout this file is in, if there is one.""" +def newest_version(tags: list[str], prefix: str) -> str | None: + """The newest release among image tags such as cu132-v2.6.0, ignoring development builds.""" + versions = [] + for tag in tags: + match = re.fullmatch(re.escape(prefix) + r"-v(\d+)\.(\d+)\.(\d+)", tag) + if match: + versions.append(tuple(int(v) for v in match.groups())) + return ".".join(str(v) for v in max(versions)) if versions else None + + +def published_tags(prefix: str) -> list[str]: + """The tags of the image on Docker Hub that start with `prefix`; empty if Docker Hub cannot be reached.""" + import json + import urllib.request + + url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name={prefix}-v" + tags: list[str] = [] try: - text = pyproject.read_text() - except OSError: - return None - project = re.search(r"^\[project\]\s*$(.*?)(?=^\[|\Z)", text, re.MULTILINE | re.DOTALL) - match = project and re.search(r"^version\s*=\s*['\"]([^'\"]+)['\"]", project.group(1), re.MULTILINE) - return match.group(1) if match else None + while url and len(tags) < 1000: + with urllib.request.urlopen(url, timeout=20) as response: + page = json.load(response) + tags += [result["name"] for result in page.get("results", [])] + url = page.get("next") + except (OSError, ValueError, KeyError): + return [] + return tags + + +def local_tags(runtime: str, image_dir: Path) -> list[str]: + """The tags of the FastSurfer images already on this machine, for runs without internet.""" + if runtime != "docker": + prefix = DOCKER_REPO.rsplit("/", 1)[-1] + "-" + return [p.name[len(prefix):-len(".sif")] for p in image_dir.glob(f"{prefix}*.sif")] + try: + out = subprocess.run(["docker", "image", "ls", DOCKER_REPO, "--format", "{{.Tag}}"], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError): + return [] + return out.stdout.split() if out.returncode == 0 else [] def query_nvidia_gpu(index: int = 0) -> tuple[tuple[int, int] | None, int] | None: @@ -175,21 +206,29 @@ def runs(build: tuple[tuple[int, int], str]) -> bool: return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) -def image_reference(prefix: str, version: str | None) -> str: +def image_reference(prefix: str, version: str) -> str: """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" - if version is None: - raise LauncherError( - "Which FastSurfer version to run is unknown, because this file is not part of a FastSurfer " - "checkout. Pass it with --image_version, for example --image_version 2.6.0." - ) - if "dev" in version: - raise LauncherError( - f"There are no images of the development version {version}. Pass a released version with " - f"--image_version, or your own image with --image." - ) return f"{DOCKER_REPO}:{prefix}-v{version}" +def resolve_version(prefix: str, runtime: str, image_dir: Path) -> tuple[str, str]: + """ + The newest released FastSurfer version for this image, and where that was found. + + Asks Docker Hub, and without internet looks at the images already on this machine. + """ + version = newest_version(published_tags(prefix), prefix) + if version is not None: + return version, "the newest release" + version = newest_version(local_tags(runtime, image_dir), prefix) + if version is not None: + return version, "the newest image on this machine, Docker Hub could not be reached" + raise LauncherError( + "Could not find out the newest FastSurfer release, because Docker Hub cannot be reached and no " + "FastSurfer image is on this machine yet. Pass the version with --image_version, e.g. 2.6.0." + ) + + def _absolute(value: str, cwd: Path) -> Path: return Path(os.path.normpath(cwd / os.path.expanduser(value))) @@ -378,7 +417,7 @@ def make_parser() -> argparse.ArgumentParser: parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", help="the container runtime (default: auto, the first installed of these)") parser.add_argument("--image_version", - help="the FastSurfer version to run, e.g. 2.6.0 (default: the version of this checkout)") + help="the FastSurfer version to run, e.g. 2.6.0 (default: the newest release)") parser.add_argument("--image", help="run this image instead: a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer " ".sif file") @@ -439,8 +478,12 @@ def info(message: str, level: str = "INFO") -> None: else: index = int(device.split(":", 1)[1]) if device.startswith("cuda:") else 0 build = choose_build(query_nvidia_gpu(index) if device != "cpu" else None, device) - pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" - image = image_reference(build.prefix, options.image_version or read_image_version(pyproject)) + version = options.image_version + if version is None: + version, source = resolve_version(build.prefix, runtime, options.image_dir) + info(f"Running FastSurfer {version}, {source}. Pass --image_version {version} to keep " + f"using this version when newer ones are released.") + image = image_reference(build.prefix, version) for note in build.notes: info(note) From 0739b85cbdbf5a0947f656a8901dc78dd97b6eab Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Sun, 11 Oct 2026 22:32:51 +0200 Subject: [PATCH 04/18] Tell launcher users how far an older version is behind, and recommend the CPU over an old GPU image --- doc/overview/CONTAINERS.md | 5 +- test/utils/test_fastsurfer_container.py | 20 +++++ tools/fastsurfer_container.py | 111 ++++++++++++++++++++---- 3 files changed, 118 insertions(+), 18 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 7b420c68b..45725ae9a 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -44,7 +44,10 @@ python3 fastsurfer_container.py \ ``` It runs the newest FastSurfer release and says which one. For reproducible results, keep a study on -one version with `--image_version `. Without internet, it uses the newest FastSurfer image +one version with `--image_version `; for an older version, it says how many releases it is +behind. A GPU that the newest images no longer support runs them on the CPU. The image of an older +release may still support that GPU, but the launcher recommends the newest release on the CPU, or a +machine with a newer GPU, over an old version. Without internet, it uses the newest FastSurfer image already on the machine. With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have to be absolute. `--print_only` prints the commands instead of running them, and `--help` lists all launcher options. The sections below explain the diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 1ac07964b..688595e69 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -83,6 +83,26 @@ def test_newest_version(): assert launcher.newest_version(tags, "cpu") is None +def test_official_image_version(): + assert launcher.official_image_version("deepmi/fastsurfer:cu118-v2.2.0") == "2.2.0" + assert launcher.official_image_version("docker.io/deepmi/fastsurfer:cpu-v2.6.0") == "2.6.0" + assert launcher.official_image_version("/images/fastsurfer-cu118-v2.2.0.sif") == "2.2.0" + for image in ("deepmi/fastsurfer:latest", "my_fastsurfer:cu132-v2.6.0", "fastsurfer-myimage.sif", + "deepmi/fastsurfer:cu132-v2.7.0-dev0"): + assert launcher.official_image_version(image) is None + + +def test_age_note(): + releases = [(2, 2, 0), (2, 3, 3), (2, 4, 2), (2, 5, 4), (2, 6, 0)] + assert launcher.age_note("2.6.0", releases, gpu_unsupported=True) is None + assert launcher.age_note("2.6.0", [], gpu_unsupported=True) is None # Docker Hub unreachable + level, message = launcher.age_note("2.5.4", releases, gpu_unsupported=False) + assert level == "INFO" and "1 release older" in message + # an old GPU image instead of the newest release on the CPU: recommend against it + level, message = launcher.age_note("2.2.0", releases, gpu_unsupported=True, flag="--image") + assert level == "WARNING" and "4 releases older than the newest, 2.6.0" in message and "CPU" in message + + def test_local_tags_apptainer(tmp_path): for name in ("fastsurfer-cpu-v2.5.4.sif", "fastsurfer-cpu-v2.6.0.sif", "fastsurfer-myimage.sif", "other.sif"): (tmp_path / name).write_bytes(b"") diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 6a3757ae7..2c41cd6e8 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -83,22 +83,36 @@ class Build: notes: list[str] = field(default_factory=list) +def parse_version(version: str) -> tuple[int, ...] | None: + """A release version such as 2.6.0 as numbers, None for anything else (e.g. a development build).""" + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", version) + return tuple(int(v) for v in match.groups()) if match else None + + +def release_versions(tags: list[str], prefix: str) -> list[tuple[int, ...]]: + """The releases among image tags such as cu132-v2.6.0, oldest first, ignoring development builds.""" + versions = set() + for tag in tags: + if tag.startswith(f"{prefix}-v"): + version = parse_version(tag[len(prefix) + 2:]) + if version is not None: + versions.add(version) + return sorted(versions) + + def newest_version(tags: list[str], prefix: str) -> str | None: """The newest release among image tags such as cu132-v2.6.0, ignoring development builds.""" - versions = [] - for tag in tags: - match = re.fullmatch(re.escape(prefix) + r"-v(\d+)\.(\d+)\.(\d+)", tag) - if match: - versions.append(tuple(int(v) for v in match.groups())) - return ".".join(str(v) for v in max(versions)) if versions else None + versions = release_versions(tags, prefix) + return ".".join(str(v) for v in versions[-1]) if versions else None -def published_tags(prefix: str) -> list[str]: - """The tags of the image on Docker Hub that start with `prefix`; empty if Docker Hub cannot be reached.""" +def published_tags(name: str) -> list[str] | None: + """The tags of the image on Docker Hub that contain `name`; None if Docker Hub cannot be reached.""" import json + import urllib.parse import urllib.request - url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name={prefix}-v" + url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name={urllib.parse.quote(name)}" tags: list[str] = [] try: while url and len(tags) < 1000: @@ -107,10 +121,51 @@ def published_tags(prefix: str) -> list[str]: tags += [result["name"] for result in page.get("results", [])] url = page.get("next") except (OSError, ValueError, KeyError): - return [] + return None return tags +def official_image_version(image: str) -> str | None: + """The FastSurfer version of an official image, e.g. 2.2.0 for deepmi/fastsurfer:cu118-v2.2.0, else None.""" + name = Path(image).name if image.endswith(".sif") else image + pattern = (rf"{re.escape(DOCKER_REPO.rsplit('/', 1)[-1])}-[a-z0-9.]+-v(.+)\.sif" if image.endswith(".sif") + else rf"(?:docker\.io/)?{re.escape(DOCKER_REPO)}:[a-z0-9.]+-v(.+)") + match = re.fullmatch(pattern, name) + return match.group(1) if match and parse_version(match.group(1)) else None + + +def age_note(version: str, releases: list[tuple[int, ...]], gpu_unsupported: bool, + flag: str = "--image_version") -> tuple[str, str] | None: + """ + A note for a run of an older release, as (level, message), None for the newest or an unknown version. + + Parameters + ---------- + version : str + The requested version. + releases : list[tuple[int, ...]] + All releases, oldest first. + gpu_unsupported : bool + Whether the newest images do not support this GPU, so an old image is the only way to use it. + flag : str, default="--image_version" + The flag that requested the version. + """ + requested = parse_version(version) + if requested is None or not releases or requested >= releases[-1]: + return None + newest = ".".join(str(v) for v in releases[-1]) + behind = sum(release > requested for release in releases) + message = f"FastSurfer {version} is {behind} release{'s' if behind > 1 else ''} older than the newest, {newest}." + if gpu_unsupported: + return "WARNING", ( + f"{message} The images of newer versions do not support this GPU, so they run on the CPU here. " + f"Running the newest release on the CPU (leave out --image and --image_version), or on a machine " + f"with a newer GPU, is recommended over this old version, which lacks the improvements and fixes " + f"since." + ) + return "INFO", f"{message} Leave out {flag} to run the newest release." + + def local_tags(runtime: str, image_dir: Path) -> list[str]: """The tags of the FastSurfer images already on this machine, for runs without internet.""" if runtime != "docker": @@ -217,7 +272,7 @@ def resolve_version(prefix: str, runtime: str, image_dir: Path) -> tuple[str, st Asks Docker Hub, and without internet looks at the images already on this machine. """ - version = newest_version(published_tags(prefix), prefix) + version = newest_version(published_tags(f"{prefix}-v") or [], prefix) if version is not None: return version, "the newest release" version = newest_version(local_tags(runtime, image_dir), prefix) @@ -471,19 +526,40 @@ def info(message: str, level: str = "INFO") -> None: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") device = _fastsurfer_device(args) + index = int(device.split(":", 1)[1]) if device.startswith("cuda:") else 0 + gpu = query_nvidia_gpu(index) if device != "cpu" else None + # whether the newest images can use this GPU; a GPU they do not support runs them on the CPU + gpu_unsupported = gpu is not None and not choose_build(gpu, "auto").gpu if options.image: - # an own image: give it the GPU unless the run is on the CPU - build = Build("custom", device != "cpu" and query_nvidia_gpu() is not None) + # an own or an older official image: give it the GPU unless the run is on the CPU + build = Build("custom", gpu is not None) image = options.image + requested, flag = official_image_version(image), "--image" else: - index = int(device.split(":", 1)[1]) if device.startswith("cuda:") else 0 - build = choose_build(query_nvidia_gpu(index) if device != "cpu" else None, device) - version = options.image_version + build = choose_build(gpu, device) + requested, flag = options.image_version, "--image_version" + version = requested if version is None: version, source = resolve_version(build.prefix, runtime, options.image_dir) info(f"Running FastSurfer {version}, {source}. Pass --image_version {version} to keep " f"using this version when newer ones are released.") image = image_reference(build.prefix, version) + if requested is not None: + # an older version may have no image for this GPU, e.g. none for a newer CUDA version + available = published_tags(f"-v{requested}") + tag = image.rsplit(":", 1)[-1] + if available is not None and tag not in available: + images = sorted(t for t in available if t.endswith(f"-v{requested}")) + raise LauncherError( + f"There is no image {image}. " + + (f"The images of FastSurfer {requested} are {', '.join(images)}; pass one with " + f"--image {DOCKER_REPO}:." if images else f"FastSurfer {requested} is not released.") + ) + if requested is not None: + note = age_note(requested, release_versions(published_tags("cpu-v") or [], "cpu"), + gpu_unsupported and build.gpu, flag) + if note is not None: + info(note[1], level=note[0]) for note in build.notes: info(note) @@ -492,7 +568,8 @@ def info(message: str, level: str = "INFO") -> None: commands = [] if runtime == "docker": user = f"{os.getuid()}:{os.getgid()}" - emulate = apple_silicon and not options.image + # official images are built for Intel/AMD only; an own image may be built for this Mac + emulate = apple_silicon and (not options.image or official_image_version(options.image) is not None) commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user, emulate)) if not options.print_only: memory = docker_memory_gib() From 751088a59ec04acd54e3dc11c1289f00977ee74b Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Mon, 12 Oct 2026 16:19:25 +0200 Subject: [PATCH 05/18] Handle GPU lists and --viewagg_device in the launcher, and fix five review findings --- test/utils/test_fastsurfer_container.py | 61 +++++++-- tools/fastsurfer_container.py | 165 +++++++++++++++++------- 2 files changed, 165 insertions(+), 61 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 688595e69..cf04ff12c 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -41,14 +41,24 @@ ], ) def test_choose_build_gpu(gpu, prefix): - build = launcher.choose_build(gpu, "auto") + build = launcher.choose_build([gpu], "auto") assert (build.prefix, build.gpu) == (prefix, True) def test_choose_build_explains_the_older_image(): - build = launcher.choose_build(((8, 6), 550), "auto") + build = launcher.choose_build([((8, 6), 550)], "auto") assert build.notes and "550" in build.notes[0] - assert launcher.choose_build(((6, 1), 580), "auto").notes == [] + assert launcher.choose_build([((6, 1), 580)], "auto").notes == [] + + +def test_choose_build_several_gpus(): + # the image has to run on every GPU used: an Ampere and a Pascal need the CUDA 12.6 image + assert launcher.choose_build([((8, 6), 580), ((6, 1), 580)], "auto").prefix == "cu126" + # a Blackwell and a Pascal have no image in common + build = launcher.choose_build([((12, 0), 580), ((6, 1), 580)], "auto") + assert build.prefix == "cpu" and "both GPUs" in build.notes[0] + with pytest.raises(launcher.LauncherError): + launcher.choose_build([((12, 0), 580), ((6, 1), 580)], "cuda") @pytest.mark.parametrize( @@ -60,18 +70,43 @@ def test_choose_build_explains_the_older_image(): ], ) def test_choose_build_falls_back_to_cpu(gpu): - build = launcher.choose_build(gpu, "auto") + build = launcher.choose_build([gpu], "auto") assert (build.prefix, build.gpu) == ("cpu", False) assert build.notes with pytest.raises(launcher.LauncherError): - launcher.choose_build(gpu, "cuda") + launcher.choose_build([gpu], "cuda") def test_choose_build_cpu(): - assert launcher.choose_build(((8, 6), 580), "cpu").prefix == "cpu" - assert launcher.choose_build(None, "auto").prefix == "cpu" + assert launcher.choose_build([((8, 6), 580)], "cpu").prefix == "cpu" + assert launcher.choose_build([], "auto").prefix == "cpu" + with pytest.raises(launcher.LauncherError): + launcher.choose_build([], "cuda") + + +def test_cuda_indices(): + assert launcher.cuda_indices("cuda:1") == [1] + assert launcher.cuda_indices("cuda:0,1") == [0, 1] + assert launcher.cuda_indices("cuda:0-2,5") == [0, 1, 2, 5] + assert launcher.cuda_indices("cuda") is None and launcher.cuda_indices("auto") is None with pytest.raises(launcher.LauncherError): - launcher.choose_build(None, "cuda:0") + launcher.cuda_indices("cuda:first") + + +def test_plan_gpus(): + gpus = [((8, 6), 580), ((6, 1), 580), ((7, 5), 580)] + assert launcher.plan_gpus([], gpus, brun=False) == ("auto", [gpus[0]]) + # brun_fastsurfer.sh may spread "auto" over every GPU + assert launcher.plan_gpus([], gpus, brun=True) == ("auto", gpus) + assert launcher.plan_gpus(["--device", "cuda:1-2"], gpus, brun=True) == ("cuda", gpus[1:]) + assert launcher.plan_gpus(["--device", "cpu"], gpus, brun=False) == ("cpu", []) + # the view aggregation can need a GPU while the segmentation runs on the CPU + assert launcher.plan_gpus(["--device", "cpu", "--viewagg_device", "cuda:2"], gpus, False) == ("cuda", [gpus[2]]) + # the last of repeated flags counts, as in the scripts + assert launcher.plan_gpus(["--device", "cpu", "--device", "cuda:1"], gpus, False) == ("cuda", [gpus[1]]) + assert launcher.plan_gpus(["--device", "cuda:1", "--device", "cpu"], gpus, False) == ("cpu", []) + with pytest.raises(launcher.LauncherError): + launcher.plan_gpus(["--device", "cuda:3"], gpus, brun=False) def test_newest_version(): @@ -150,10 +185,12 @@ def test_plan_paths_brun(tmp_path): out, mounts = launcher.plan_paths(["--subjects", "sub1=a/t1.mgz", "--sd", "out"], tmp_path) assert out[:2] == ["--subjects", f"sub1={a / 't1.mgz'}"] - # relative paths in a list cannot be resolved inside the container - subjects.write_text("sub1=a/t1.mgz\n") - with pytest.raises(launcher.LauncherError): - launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) + # relative paths in a list cannot be resolved inside the container, and brun_fastsurfer.sh does not + # expand ~ in a list + for line in ("sub1=a/t1.mgz\n", "sub1=~/t1.mgz\n"): + subjects.write_text(line) + with pytest.raises(launcher.LauncherError): + launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) def test_commands(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 2c41cd6e8..ff685da92 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -179,14 +179,14 @@ def local_tags(runtime: str, image_dir: Path) -> list[str]: return out.stdout.split() if out.returncode == 0 else [] -def query_nvidia_gpu(index: int = 0) -> tuple[tuple[int, int] | None, int] | None: +def query_nvidia_gpus() -> list[tuple[tuple[int, int] | None, int]]: """ - The compute capability and the driver version of an NVIDIA GPU, from nvidia-smi. + The compute capability and the driver version of each NVIDIA GPU, from nvidia-smi, in index order. - Returns None without an NVIDIA GPU. The capability is None if this driver cannot report it. + Empty without an NVIDIA GPU. A capability is None if the driver cannot report it, e.g. as N/A. """ if shutil.which("nvidia-smi") is None: - return None + return [] for fields in ("compute_cap,driver_version", "driver_version"): try: out = subprocess.run( @@ -195,72 +195,135 @@ def query_nvidia_gpu(index: int = 0) -> tuple[tuple[int, int] | None, int] | Non ).stdout.strip().splitlines() except (OSError, subprocess.SubprocessError): continue - if index >= len(out): - return None - values = [v.strip() for v in out[index].split(",")] - driver = int(values[-1].split(".")[0]) - if len(values) == 1: - return None, driver - major, minor = values[0].split(".") - return (int(major), int(minor)), driver - return None + gpus = [] + try: + for line in out: + values = [v.strip() for v in line.split(",")] + driver = int(values[-1].split(".")[0]) + capability = None + if len(values) > 1 and re.fullmatch(r"\d+\.\d+", values[0]): + major, minor = values[0].split(".") + capability = (int(major), int(minor)) + gpus.append((capability, driver)) + except (ValueError, IndexError): + continue # an unreadable driver version: try the next query + return gpus + return [] -def choose_build(gpu: tuple[tuple[int, int] | None, int] | None, device: str) -> Build: +def choose_build(gpus: list[tuple[tuple[int, int] | None, int]], use: str) -> Build: """ - The image that runs on this GPU and driver, preferring the default CUDA version. + The image that runs on all of these GPUs and their driver, preferring the default CUDA version. Parameters ---------- - gpu : tuple, None - The compute capability (None if unknown) and the driver major version, or None without a GPU. - device : str - The value of FastSurfer's --device, "auto" if it was not given. + gpus : list[tuple] + The GPUs FastSurfer will use: compute capability (None if unknown) and driver major version. + use : str + "cuda" if a GPU was requested explicitly, "auto" if FastSurfer would use one if it can, and + "cpu" if it runs on the CPU only. Returns ------- Build The tag prefix, whether to give the container the GPU, and notes for the user. """ - if device == "cpu": + if use == "cpu": return Build("cpu", False) - if gpu is None: - if device.startswith("cuda"): - raise LauncherError(f"--device {device} needs an NVIDIA GPU, but nvidia-smi found none.") + if not gpus: + if use == "cuda": + raise LauncherError("A cuda device was requested, but nvidia-smi found no NVIDIA GPU.") return Build("cpu", False, ["No NVIDIA GPU found, so FastSurfer runs on the CPU."]) - capability, driver = gpu + driver = min(d for _, d in gpus) + capabilities = [c for c, _ in gpus if c is not None] def runs(build: tuple[tuple[int, int], str]) -> bool: (cuda_major, _), prefix = build if driver < MIN_DRIVER[cuda_major]: return False - if capability is None: - return True - if prefix == LEGACY_BUILD[1] and capability > LEGACY_MAX_CAPABILITY: - return False - return capability >= MIN_CAPABILITY[prefix] + too_new = prefix == LEGACY_BUILD[1] and any(c > LEGACY_MAX_CAPABILITY for c in capabilities) + return not too_new and all(c >= MIN_CAPABILITY[prefix] for c in capabilities) if runs(NEWEST_BUILD): return Build(NEWEST_BUILD[1], True) newest_min_driver = MIN_DRIVER[NEWEST_BUILD[0][0]] if runs(LEGACY_BUILD): notes = [] - if driver < newest_min_driver and (capability is None or capability >= MIN_CAPABILITY[NEWEST_BUILD[1]]): + if driver < newest_min_driver and all(c >= MIN_CAPABILITY[NEWEST_BUILD[1]] for c in capabilities): notes.append( f"The NVIDIA driver {driver} is older than {newest_min_driver}, so this uses the image for " f"CUDA {LEGACY_BUILD[0][0]}.{LEGACY_BUILD[0][1]}. Update the driver to use the default image." ) return Build(LEGACY_BUILD[1], True, notes) - if capability is not None and capability < MIN_CAPABILITY[LEGACY_BUILD[1]]: - reason = f"The GPU (compute capability {capability[0]}.{capability[1]}) is too old for FastSurfer's images" + oldest = min(capabilities) if capabilities else None + newest = max(capabilities) if capabilities else None + if oldest is not None and oldest < MIN_CAPABILITY[LEGACY_BUILD[1]]: + reason = f"The GPU (compute capability {oldest[0]}.{oldest[1]}) is too old for FastSurfer's images" + elif (driver >= newest_min_driver and oldest < MIN_CAPABILITY[NEWEST_BUILD[1]] + and newest > LEGACY_MAX_CAPABILITY): + reason = (f"No FastSurfer image supports both GPUs of compute capability {oldest[0]}.{oldest[1]} " + f"and {newest[0]}.{newest[1]}; select GPUs of one kind with --device cuda:") else: reason = (f"The NVIDIA driver {driver} is too old for FastSurfer's images, update it to " f"{newest_min_driver} or newer") - if device.startswith("cuda"): + if use == "cuda": raise LauncherError(f"{reason}.") return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) +def cuda_indices(value: str) -> list[int] | None: + """ + The GPU indices a device value names, e.g. [0, 1, 2, 5] for cuda:0-2,5, and None if it names none. + + Accepts the forms run_fastsurfer.sh and brun_fastsurfer.sh accept: cuda:, lists and ranges. + """ + if not value.startswith("cuda:"): + return None + indices = [] + for part in value[len("cuda:"):].split(","): + match = re.fullmatch(r"(\d+)(?:-(\d+))?", part) + if match is None: + raise LauncherError(f"Cannot read the device {value}; use e.g. cuda:0, cuda:0,1 or cuda:0-3.") + first, last = int(match.group(1)), int(match.group(2) or match.group(1)) + indices.extend(range(first, last + 1)) + return indices + + +def plan_gpus(args: list[str], gpus: list[tuple[tuple[int, int] | None, int]], brun: bool, + ) -> tuple[str, list[tuple[tuple[int, int] | None, int]]]: + """ + Whether FastSurfer would use a GPU with these arguments, and which of the GPUs. + + --device decides the segmentation, --viewagg_device the view aggregation, which runs on the CPU + with --device cpu unless it names cuda itself. Without an index, run_fastsurfer.sh uses the first + GPU and brun_fastsurfer.sh may use all of them. + + Returns + ------- + str + "cuda", "auto" or "cpu", as `use` of choose_build. + list[tuple] + The GPUs that will be used. + """ + device = _last_value(args, "--device") or "auto" + viewagg = _last_value(args, "--viewagg_device") or "auto" + requested = [v for v in (device, viewagg) if v == "cuda" or v.startswith("cuda:")] + if requested: + use = "cuda" + elif device == "auto": + use = "auto" + else: + return "cpu", [] + indices: set[int] = set() + for value in requested or [device]: + named = cuda_indices(value) + indices.update(named if named is not None else (range(len(gpus)) if brun else [0])) + missing = sorted(i for i in indices if i >= len(gpus)) + if missing and gpus: + raise LauncherError(f"There is no GPU {missing[0]}, nvidia-smi found {len(gpus)}.") + return use, [gpus[i] for i in sorted(indices) if i < len(gpus)] + + def image_reference(prefix: str, version: str) -> str: """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" return f"{DOCKER_REPO}:{prefix}-v{version}" @@ -359,12 +422,14 @@ def _subjects_list_mounts(subjects_list: Path) -> list[Path]: _, _, t1 = tokens[0].partition("=") paths = [t1] + [tokens[k + 1] for k, tok in enumerate(tokens[1:-1], start=1) if tok in INPUT_FLAGS] for value in paths: - if not os.path.isabs(os.path.expanduser(value)): + # brun_fastsurfer.sh expands nothing in a subjects list, so ~ is not the home directory there + if not os.path.isabs(value): raise LauncherError( - f"{subjects_list}, line {number}: {value} is a relative path. Use absolute paths in a " - f"subjects list, the container does not start in the current directory." + f"{subjects_list}, line {number}: {value} is not an absolute path. Use absolute paths " + f"in a subjects list, without ~: the container does not start in the current directory, " + f"and nothing in the list is expanded." ) - mounts.append(_parent_of_file(Path(os.path.expanduser(value)))) + mounts.append(_parent_of_file(Path(value))) return mounts @@ -438,11 +503,12 @@ def docker_memory_gib() -> float: return int(out.stdout.strip()) / 2**30 -def _fastsurfer_device(args: list[str]) -> str: - for i in range(len(args) - 1): - if args[i] == "--device": +def _last_value(args: list[str], flag: str) -> str | None: + """The value of the last `flag` in the arguments, as the scripts apply repeated flags in order.""" + for i in range(len(args) - 2, -1, -1): + if args[i] == flag: return args[i + 1] - return "auto" + return None def _license_args(args: list[str], cwd: Path) -> list[str]: @@ -505,7 +571,7 @@ def info(message: str, level: str = "INFO") -> None: package_hint = (f"install the FastSurfer package for macOS instead, {MACOS_PACKAGE_URL} (see {MACOS_DOC_URL}). " f"It runs natively and uses the Mac's GPU: in the console it opens, call run_fastsurfer.sh " f"with the same flags.") - if apple_silicon and options.runtime == "auto": + if apple_silicon and options.runtime != "docker": raise LauncherError(f"On a Mac with Apple silicon, {package_hint} To run the Intel image in Docker " f"under emulation anyway, which is much slower, pass --runtime docker.") if apple_silicon: @@ -525,18 +591,19 @@ def info(message: str, level: str = "INFO") -> None: if "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - device = _fastsurfer_device(args) - index = int(device.split(":", 1)[1]) if device.startswith("cuda:") else 0 - gpu = query_nvidia_gpu(index) if device != "cpu" else None - # whether the newest images can use this GPU; a GPU they do not support runs them on the CPU - gpu_unsupported = gpu is not None and not choose_build(gpu, "auto").gpu + if options.image and options.image.endswith(".sif") and runtime == "docker": + raise LauncherError("A .sif image needs Apptainer or Singularity: pass --runtime apptainer or " + "--runtime singularity, or a Docker image with --image.") + use, gpus = plan_gpus(args, query_nvidia_gpus(), options.brun) + # whether the newest images can use these GPUs; GPUs they do not support run them on the CPU + gpu_unsupported = bool(gpus) and not choose_build(gpus, "auto").gpu if options.image: # an own or an older official image: give it the GPU unless the run is on the CPU - build = Build("custom", gpu is not None) + build = Build("custom", bool(gpus)) image = options.image requested, flag = official_image_version(image), "--image" else: - build = choose_build(gpu, device) + build = choose_build(gpus, use) requested, flag = options.image_version, "--image_version" version = requested if version is None: From 1a949d4b698a5407343614fdccc38be342164b6e Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Mon, 12 Oct 2026 16:36:47 +0200 Subject: [PATCH 06/18] Let the launcher pick the best supported GPU when none is named --- doc/overview/CONTAINERS.md | 3 +- test/utils/test_fastsurfer_container.py | 101 +++++++---- tools/fastsurfer_container.py | 220 ++++++++++++++++-------- 3 files changed, 224 insertions(+), 100 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 45725ae9a..2179b7080 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -26,7 +26,8 @@ The launcher - uses Apptainer, Singularity or Docker, whichever is installed (choose one with `--runtime`), - picks the image that runs on your NVIDIA GPU and driver, or the CPU image without one, see - [which image fits your GPU](install/LINUX.md#nvidia-gpus), + [which image fits your GPU](install/LINUX.md#nvidia-gpus); with several GPUs and no + `--device cuda:`, it uses the newest one with enough memory and says which, - downloads the image once (for Apptainer, into `$HOME/my_singularity_images`), - shares the files and directories the flags name with the container, at the same paths, so relative paths work too, diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index cf04ff12c..c2d9cb109 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -28,62 +28,81 @@ _spec.loader.exec_module(launcher) +def gpu(capability, driver=580, index=0, memory=24.0): + return launcher.Gpu(index, capability, driver, memory) + + @pytest.mark.parametrize( - ("gpu", "prefix"), + ("capability", "driver", "prefix"), [ - (((8, 6), 580), "cu132"), # Ampere, current driver - (((12, 0), 590), "cu132"), # Blackwell - (((8, 6), 550), "cu126"), # driver too old for CUDA 13 - (((6, 1), 580), "cu126"), # Pascal, dropped by CUDA 13 - (((7, 0), 600), "cu126"), # Volta - ((None, 580), "cu132"), # capability unknown, the driver decides - ((None, 550), "cu126"), + ((8, 6), 580, "cu132"), # Ampere, current driver + ((12, 0), 590, "cu132"), # Blackwell + ((8, 6), 550, "cu126"), # driver too old for CUDA 13 + ((6, 1), 580, "cu126"), # Pascal, dropped by CUDA 13 + ((7, 0), 600, "cu126"), # Volta + (None, 580, "cu132"), # capability unknown, the driver decides + (None, 550, "cu126"), ], ) -def test_choose_build_gpu(gpu, prefix): - build = launcher.choose_build([gpu], "auto") +def test_choose_build_gpu(capability, driver, prefix): + build = launcher.choose_build([gpu(capability, driver)], "auto") assert (build.prefix, build.gpu) == (prefix, True) def test_choose_build_explains_the_older_image(): - build = launcher.choose_build([((8, 6), 550)], "auto") + build = launcher.choose_build([gpu((8, 6), 550)], "auto") assert build.notes and "550" in build.notes[0] - assert launcher.choose_build([((6, 1), 580)], "auto").notes == [] + assert launcher.choose_build([gpu((6, 1))], "auto").notes == [] def test_choose_build_several_gpus(): # the image has to run on every GPU used: an Ampere and a Pascal need the CUDA 12.6 image - assert launcher.choose_build([((8, 6), 580), ((6, 1), 580)], "auto").prefix == "cu126" + assert launcher.choose_build([gpu((8, 6)), gpu((6, 1), index=1)], "auto").prefix == "cu126" # a Blackwell and a Pascal have no image in common - build = launcher.choose_build([((12, 0), 580), ((6, 1), 580)], "auto") + build = launcher.choose_build([gpu((12, 0)), gpu((6, 1), index=1)], "auto") assert build.prefix == "cpu" and "both GPUs" in build.notes[0] with pytest.raises(launcher.LauncherError): - launcher.choose_build([((12, 0), 580), ((6, 1), 580)], "cuda") + launcher.choose_build([gpu((12, 0)), gpu((6, 1), index=1)], "cuda") @pytest.mark.parametrize( - "gpu", + ("capability", "driver"), [ ((12, 0), 550), # Blackwell needs CUDA 13, whose driver is missing ((3, 7), 580), # Kepler, no image supports it ((8, 6), 470), # driver too old for every image ], ) -def test_choose_build_falls_back_to_cpu(gpu): - build = launcher.choose_build([gpu], "auto") +def test_choose_build_falls_back_to_cpu(capability, driver): + build = launcher.choose_build([gpu(capability, driver)], "auto") assert (build.prefix, build.gpu) == ("cpu", False) assert build.notes with pytest.raises(launcher.LauncherError): - launcher.choose_build([gpu], "cuda") + launcher.choose_build([gpu(capability, driver)], "cuda") def test_choose_build_cpu(): - assert launcher.choose_build([((8, 6), 580)], "cpu").prefix == "cpu" + assert launcher.choose_build([gpu((8, 6))], "cpu").prefix == "cpu" assert launcher.choose_build([], "auto").prefix == "cpu" with pytest.raises(launcher.LauncherError): launcher.choose_build([], "cuda") +def test_select_gpus(): + kepler, pascal, ampere = gpu((3, 7), index=0), gpu((6, 1), index=1), gpu((8, 6), index=2) + # the newest GPU an image supports, shown alone so FastSurfer uses it as cuda:0 + build = launcher.select_gpus([kepler, pascal, ampere], brun=False) + assert (build.prefix, build.devices) == ("cu132", [2]) + assert build.notes[0] == "Using GPU 2 (compute capability 8.6, 24 GB)." + # a GPU with too little memory only when none has enough + small = gpu((8, 9), index=3, memory=4.0) + assert launcher.select_gpus([pascal, small], brun=False).devices == [1] + assert launcher.select_gpus([small, gpu((7, 5), index=4, memory=6.0)], brun=False).devices == [4] + # brun_fastsurfer.sh gets every GPU the chosen image supports + assert launcher.select_gpus([kepler, pascal, ampere, gpu((7, 5), index=4)], brun=True).devices == [2, 4] + assert launcher.select_gpus([kepler], brun=False) is None + + def test_cuda_indices(): assert launcher.cuda_indices("cuda:1") == [1] assert launcher.cuda_indices("cuda:0,1") == [0, 1] @@ -93,20 +112,31 @@ def test_cuda_indices(): launcher.cuda_indices("cuda:first") -def test_plan_gpus(): - gpus = [((8, 6), 580), ((6, 1), 580), ((7, 5), 580)] - assert launcher.plan_gpus([], gpus, brun=False) == ("auto", [gpus[0]]) - # brun_fastsurfer.sh may spread "auto" over every GPU - assert launcher.plan_gpus([], gpus, brun=True) == ("auto", gpus) - assert launcher.plan_gpus(["--device", "cuda:1-2"], gpus, brun=True) == ("cuda", gpus[1:]) - assert launcher.plan_gpus(["--device", "cpu"], gpus, brun=False) == ("cpu", []) +def test_gpu_request(): + assert launcher.gpu_request([]) == ("auto", None) + assert launcher.gpu_request(["--device", "cuda"]) == ("cuda", None) + assert launcher.gpu_request(["--device", "cuda:0-2,5"]) == ("cuda", [0, 1, 2, 5]) + assert launcher.gpu_request(["--device", "cpu"]) == ("cpu", None) # the view aggregation can need a GPU while the segmentation runs on the CPU - assert launcher.plan_gpus(["--device", "cpu", "--viewagg_device", "cuda:2"], gpus, False) == ("cuda", [gpus[2]]) + assert launcher.gpu_request(["--device", "cpu", "--viewagg_device", "cuda:2"]) == ("cuda", [2]) # the last of repeated flags counts, as in the scripts - assert launcher.plan_gpus(["--device", "cpu", "--device", "cuda:1"], gpus, False) == ("cuda", [gpus[1]]) - assert launcher.plan_gpus(["--device", "cuda:1", "--device", "cpu"], gpus, False) == ("cpu", []) + assert launcher.gpu_request(["--device", "cpu", "--device", "cuda:1"]) == ("cuda", [1]) + assert launcher.gpu_request(["--device", "cuda:1", "--device", "cpu"]) == ("cpu", None) + + +def test_plan_build(): + pascal, ampere = gpu((6, 1), index=0), gpu((8, 6), index=1) + # not named: the best GPU + assert launcher.plan_build([], [pascal, ampere], brun=False).devices == [1] + # named: used as named, with every GPU visible so the index keeps its meaning + build = launcher.plan_build(["--device", "cuda:0"], [pascal, ampere], brun=False) + assert (build.prefix, build.devices) == ("cu126", None) with pytest.raises(launcher.LauncherError): - launcher.plan_gpus(["--device", "cuda:3"], gpus, brun=False) + launcher.plan_build(["--device", "cuda:2"], [pascal, ampere], brun=False) + # a named GPU that is too old: stop, with the reason + with pytest.raises(launcher.LauncherError, match="too old"): + launcher.plan_build(["--device", "cuda:0"], [gpu((3, 7)), ampere], brun=False) + assert launcher.plan_build([], [gpu((3, 7))], brun=False).prefix == "cpu" def test_newest_version(): @@ -197,7 +227,14 @@ def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") assert docker == ["docker", "run", "--rm", "--user", "1000:1000", "--gpus", "all", - "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + # selected GPUs, quoted so Docker does not split the list + selected = launcher.docker_command("img:tag", True, [], "run_fastsurfer.sh", [], [], "1:1", devices=[1, 3]) + assert selected[5:7] == ["--gpus", '"device=1,3"'] + sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], "run_fastsurfer.sh", [], [], + devices=[2]) + assert sif_gpu[2:8] == ["--nv", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "--env", "CUDA_VISIBLE_DEVICES=2", + "--no-mount"] emulated = launcher.docker_command("img:tag", False, [tmp_path], "run_fastsurfer.sh", args, [], "501:20", True) assert emulated[:7] == ["docker", "run", "--rm", "--user", "501:20", "--platform", "linux/amd64"] brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index ff685da92..614904ad4 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -58,6 +58,8 @@ MIN_DRIVER = {12: 525, 13: 580} MIN_CAPABILITY = {"cu132": (7, 5), "cu126": (5, 0)} LEGACY_MAX_CAPABILITY = (9, 0) +# the graphics memory FastSurfer needs at 0.8 mm, see the system requirements in the README +MIN_GPU_MEMORY_GIB = 8 # flags of run_fastsurfer.sh and brun_fastsurfer.sh whose value is a file that has to exist INPUT_FLAGS = {"--t1", "--t2", "--lesion_mask", "--fs_license", "--subjects_list", "--subject_list"} @@ -74,13 +76,24 @@ class LauncherError(Exception): """A problem with the request, reported as one message without a traceback.""" +@dataclass +class Gpu: + """An NVIDIA GPU as nvidia-smi reports it; capability and memory are None if it cannot.""" + + index: int + capability: tuple[int, int] | None + driver: int + memory_gib: float | None = None + + @dataclass class Build: - """The image build to run, and whether the container gets the GPU.""" + """The image build to run, whether the container gets GPUs, and which (None for all).""" prefix: str gpu: bool notes: list[str] = field(default_factory=list) + devices: list[int] | None = None def parse_version(version: str) -> tuple[int, ...] | None: @@ -179,18 +192,18 @@ def local_tags(runtime: str, image_dir: Path) -> list[str]: return out.stdout.split() if out.returncode == 0 else [] -def query_nvidia_gpus() -> list[tuple[tuple[int, int] | None, int]]: +def query_nvidia_gpus() -> list[Gpu]: """ - The compute capability and the driver version of each NVIDIA GPU, from nvidia-smi, in index order. + The NVIDIA GPUs, from nvidia-smi, in its (PCI bus) order. - Empty without an NVIDIA GPU. A capability is None if the driver cannot report it, e.g. as N/A. + Empty without an NVIDIA GPU. A capability or memory the driver cannot report, e.g. as N/A, is None. """ if shutil.which("nvidia-smi") is None: return [] - for fields in ("compute_cap,driver_version", "driver_version"): + for fields in ("index,compute_cap,driver_version,memory.total", "index,driver_version,memory.total"): try: out = subprocess.run( - ["nvidia-smi", f"--query-gpu={fields}", "--format=csv,noheader"], + ["nvidia-smi", f"--query-gpu={fields}", "--format=csv,noheader,nounits"], capture_output=True, text=True, check=True, timeout=30, ).stdout.strip().splitlines() except (OSError, subprocess.SubprocessError): @@ -199,26 +212,55 @@ def query_nvidia_gpus() -> list[tuple[tuple[int, int] | None, int]]: try: for line in out: values = [v.strip() for v in line.split(",")] - driver = int(values[-1].split(".")[0]) capability = None - if len(values) > 1 and re.fullmatch(r"\d+\.\d+", values[0]): - major, minor = values[0].split(".") + if len(values) == 4 and re.fullmatch(r"\d+\.\d+", values[1]): + major, minor = values[1].split(".") capability = (int(major), int(minor)) - gpus.append((capability, driver)) + memory = float(values[-1]) / 1024 if re.fullmatch(r"\d+(\.\d+)?", values[-1]) else None + gpus.append(Gpu(int(values[0]), capability, int(values[-2].split(".")[0]), memory)) except (ValueError, IndexError): - continue # an unreadable driver version: try the next query + continue # an unreadable index or driver version: try the next query return gpus return [] -def choose_build(gpus: list[tuple[tuple[int, int] | None, int]], use: str) -> Build: +def _runs(build: tuple[tuple[int, int], str], gpus: list[Gpu]) -> bool: + """Whether the image of this build runs on all of these GPUs with their driver.""" + (cuda_major, _), prefix = build + for gpu in gpus: + if gpu.driver < MIN_DRIVER[cuda_major]: + return False + if gpu.capability is not None: + if prefix == LEGACY_BUILD[1] and gpu.capability > LEGACY_MAX_CAPABILITY: + return False + if gpu.capability < MIN_CAPABILITY[prefix]: + return False + return True + + +def _best_build(gpus: list[Gpu]) -> tuple[tuple[int, int], str] | None: + """The newest CUDA build whose image runs on all of these GPUs, None if there is none.""" + return next((build for build in (NEWEST_BUILD, LEGACY_BUILD) if _runs(build, gpus)), None) + + +def describe(gpu: Gpu) -> str: + """E.g. "GPU 1 (compute capability 8.6, 24 GB)".""" + details = [] + if gpu.capability is not None: + details.append(f"compute capability {gpu.capability[0]}.{gpu.capability[1]}") + if gpu.memory_gib is not None: + details.append(f"{gpu.memory_gib:.0f} GB") + return f"GPU {gpu.index}" + (f" ({', '.join(details)})" if details else "") + + +def choose_build(gpus: list[Gpu], use: str) -> Build: """ The image that runs on all of these GPUs and their driver, preferring the default CUDA version. Parameters ---------- - gpus : list[tuple] - The GPUs FastSurfer will use: compute capability (None if unknown) and driver major version. + gpus : list[Gpu] + The GPUs FastSurfer will use. use : str "cuda" if a GPU was requested explicitly, "auto" if FastSurfer would use one if it can, and "cpu" if it runs on the CPU only. @@ -234,20 +276,13 @@ def choose_build(gpus: list[tuple[tuple[int, int] | None, int]], use: str) -> Bu if use == "cuda": raise LauncherError("A cuda device was requested, but nvidia-smi found no NVIDIA GPU.") return Build("cpu", False, ["No NVIDIA GPU found, so FastSurfer runs on the CPU."]) - driver = min(d for _, d in gpus) - capabilities = [c for c, _ in gpus if c is not None] - - def runs(build: tuple[tuple[int, int], str]) -> bool: - (cuda_major, _), prefix = build - if driver < MIN_DRIVER[cuda_major]: - return False - too_new = prefix == LEGACY_BUILD[1] and any(c > LEGACY_MAX_CAPABILITY for c in capabilities) - return not too_new and all(c >= MIN_CAPABILITY[prefix] for c in capabilities) - - if runs(NEWEST_BUILD): - return Build(NEWEST_BUILD[1], True) + driver = min(gpu.driver for gpu in gpus) + capabilities = [gpu.capability for gpu in gpus if gpu.capability is not None] newest_min_driver = MIN_DRIVER[NEWEST_BUILD[0][0]] - if runs(LEGACY_BUILD): + build = _best_build(gpus) + if build == NEWEST_BUILD: + return Build(NEWEST_BUILD[1], True) + if build == LEGACY_BUILD: notes = [] if driver < newest_min_driver and all(c >= MIN_CAPABILITY[NEWEST_BUILD[1]] for c in capabilities): notes.append( @@ -271,6 +306,32 @@ def runs(build: tuple[tuple[int, int], str]) -> bool: return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) +def select_gpus(gpus: list[Gpu], brun: bool) -> Build | None: + """ + The GPU to run on when none was named: one an image supports, with enough memory, the newest. + + The container then sees only that GPU, so FastSurfer uses it as cuda:0. With brun_fastsurfer.sh, + it sees every GPU the chosen image supports. None if no image supports any of the GPUs. + """ + supported = [gpu for gpu in gpus if _best_build([gpu]) is not None] + if not supported: + return None + + def rank(gpu: Gpu) -> tuple: + memory = gpu.memory_gib or 0.0 + if gpu.memory_gib is None or memory >= MIN_GPU_MEMORY_GIB: + return True, gpu.capability or (0, 0), memory + return False, (memory,), gpu.capability or (0, 0) + + best = max(supported, key=rank) + build_tuple = _best_build([best]) + chosen = [gpu for gpu in supported if _runs(build_tuple, [gpu])] if brun else [best] + build = choose_build([best], "auto") + build.devices = [gpu.index for gpu in chosen] + build.notes.insert(0, "Using " + ", ".join(describe(gpu) for gpu in chosen) + ".") + return build + + def cuda_indices(value: str) -> list[int] | None: """ The GPU indices a device value names, e.g. [0, 1, 2, 5] for cuda:0-2,5, and None if it names none. @@ -289,39 +350,45 @@ def cuda_indices(value: str) -> list[int] | None: return indices -def plan_gpus(args: list[str], gpus: list[tuple[tuple[int, int] | None, int]], brun: bool, - ) -> tuple[str, list[tuple[tuple[int, int] | None, int]]]: +def gpu_request(args: list[str]) -> tuple[str, list[int] | None]: """ - Whether FastSurfer would use a GPU with these arguments, and which of the GPUs. + Whether FastSurfer would use a GPU with these arguments, and the indices of the GPUs it names. - --device decides the segmentation, --viewagg_device the view aggregation, which runs on the CPU - with --device cpu unless it names cuda itself. Without an index, run_fastsurfer.sh uses the first - GPU and brun_fastsurfer.sh may use all of them. + --device decides the segmentation and --viewagg_device the view aggregation, which runs on the CPU + with --device cpu unless it names cuda itself. Returns ------- str - "cuda", "auto" or "cpu", as `use` of choose_build. - list[tuple] - The GPUs that will be used. + "cuda" if a GPU is requested, "auto" if FastSurfer would use one if it can, "cpu" otherwise. + list[int], None + The GPUs named with cuda:, None if none is named. """ - device = _last_value(args, "--device") or "auto" - viewagg = _last_value(args, "--viewagg_device") or "auto" - requested = [v for v in (device, viewagg) if v == "cuda" or v.startswith("cuda:")] - if requested: - use = "cuda" - elif device == "auto": - use = "auto" - else: - return "cpu", [] - indices: set[int] = set() - for value in requested or [device]: - named = cuda_indices(value) - indices.update(named if named is not None else (range(len(gpus)) if brun else [0])) - missing = sorted(i for i in indices if i >= len(gpus)) - if missing and gpus: - raise LauncherError(f"There is no GPU {missing[0]}, nvidia-smi found {len(gpus)}.") - return use, [gpus[i] for i in sorted(indices) if i < len(gpus)] + values = [_last_value(args, "--device") or "auto", _last_value(args, "--viewagg_device") or "auto"] + cuda = [v for v in values if v == "cuda" or v.startswith("cuda:")] + named = [i for v in cuda for i in (cuda_indices(v) or [])] + use = "cuda" if cuda else ("auto" if values[0] == "auto" else "cpu") + return use, sorted(set(named)) if named else None + + +def plan_build(args: list[str], gpus: list[Gpu], brun: bool) -> Build: + """ + The image and the GPUs for these arguments. + + Named GPUs are used as they are, and explained in detail if no image supports them. Otherwise the + best GPU an image supports is selected, and the CPU only used if there is none. + """ + use, named = gpu_request(args) + if use == "cpu": + return Build("cpu", False) + if named is not None: + by_index = {gpu.index: gpu for gpu in gpus} + missing = [i for i in named if i not in by_index] + if missing: + raise LauncherError(f"There is no GPU {missing[0]}, nvidia-smi found {len(gpus)}.") + # the container sees all GPUs, so the named indices keep their meaning + return choose_build([by_index[i] for i in named], "cuda") + return select_gpus(gpus, brun) or choose_build(gpus, use) def image_reference(prefix: str, version: str) -> str: @@ -453,19 +520,37 @@ def find_runtime(requested: str) -> str: raise LauncherError(f"{requested} is not installed.") +# CUDA numbers GPUs fastest first, nvidia-smi and the launcher by PCI bus; this makes them agree +CUDA_ORDER = "CUDA_DEVICE_ORDER=PCI_BUS_ID" + + def apptainer_command(runtime: str, sif: Path, gpu: bool, mounts: list[Path], script: str, - args: list[str], extra: list[str]) -> list[str]: - """The Apptainer/Singularity command; the home directory is not mounted, see the docs.""" - cmd = [runtime, "exec"] + (["--nv"] if gpu else []) + ["--no-mount", "home,cwd", "-e"] + args: list[str], extra: list[str], devices: list[int] | None = None) -> list[str]: + """The Apptainer/Singularity command, showing only `devices` if given; the home directory is not mounted.""" + cmd = [runtime, "exec"] + if gpu: + cmd += ["--nv", "--env", CUDA_ORDER] + if devices is not None: + cmd += ["--env", f"CUDA_VISIBLE_DEVICES={','.join(map(str, devices))}"] + cmd += ["--no-mount", "home,cwd", "-e"] for mount in mounts: cmd += ["-B", str(mount)] return cmd + extra + [str(sif), f"/fastsurfer/{script}"] + args def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: list[str], - extra: list[str], user: str, emulate: bool = False) -> list[str]: - """The Docker command, running as the calling user; `emulate` runs the Intel image on Apple silicon.""" - cmd = ["docker", "run", "--rm", "--user", user] + (["--gpus", "all"] if gpu else []) + extra: list[str], user: str, emulate: bool = False, devices: list[int] | None = None, + ) -> list[str]: + """ + The Docker command, running as the calling user and showing only `devices` if given. + + `emulate` runs the Intel image on Apple silicon. + """ + cmd = ["docker", "run", "--rm", "--user", user] + if gpu: + # the quotes are part of the value: without them, Docker splits a list of devices at the comma + selection = "all" if devices is None else f'"device={",".join(map(str, devices))}"' + cmd += ["--gpus", selection, "--env", CUDA_ORDER] if emulate: cmd += ["--platform", "linux/amd64"] for mount in mounts: @@ -594,16 +679,16 @@ def info(message: str, level: str = "INFO") -> None: if options.image and options.image.endswith(".sif") and runtime == "docker": raise LauncherError("A .sif image needs Apptainer or Singularity: pass --runtime apptainer or " "--runtime singularity, or a Docker image with --image.") - use, gpus = plan_gpus(args, query_nvidia_gpus(), options.brun) - # whether the newest images can use these GPUs; GPUs they do not support run them on the CPU - gpu_unsupported = bool(gpus) and not choose_build(gpus, "auto").gpu + gpus = query_nvidia_gpus() + # whether the newest images support none of the GPUs, so they run on the CPU here + gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None if options.image: - # an own or an older official image: give it the GPU unless the run is on the CPU - build = Build("custom", bool(gpus)) + # an own or an older official image: give it the GPUs unless the run is on the CPU + build = Build("custom", gpu_request(args)[0] != "cpu" and bool(gpus)) image = options.image requested, flag = official_image_version(image), "--image" else: - build = choose_build(gpus, use) + build = plan_build(args, gpus, options.brun) requested, flag = options.image_version, "--image_version" version = requested if version is None: @@ -637,7 +722,8 @@ def info(message: str, level: str = "INFO") -> None: user = f"{os.getuid()}:{os.getgid()}" # official images are built for Intel/AMD only; an own image may be built for this Mac emulate = apple_silicon and (not options.image or official_image_version(options.image) is not None) - commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user, emulate)) + commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user, emulate, + build.devices)) if not options.print_only: memory = docker_memory_gib() if memory < MIN_DOCKER_MEMORY_GIB: @@ -654,7 +740,7 @@ def info(message: str, level: str = "INFO") -> None: sif = options.image_dir / (image.rsplit("/", 1)[-1].replace(":", "-") + ".sif") if not sif.is_file(): commands.append([runtime, "build", str(sif), f"docker://{image}"]) - commands.append(apptainer_command(runtime, sif, build.gpu, mounts, script, args, extra)) + commands.append(apptainer_command(runtime, sif, build.gpu, mounts, script, args, extra, build.devices)) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 From 799d88d8e7a5b178e952ea3c056ccb529c1e7dd2 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Mon, 12 Oct 2026 21:47:10 +0200 Subject: [PATCH 07/18] Reject downward GPU ranges and mount subject-specific output paths in the launcher --- test/utils/test_fastsurfer_container.py | 13 +++++-- tools/fastsurfer_container.py | 45 ++++++++++++++++--------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index c2d9cb109..0d43736de 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -108,8 +108,10 @@ def test_cuda_indices(): assert launcher.cuda_indices("cuda:0,1") == [0, 1] assert launcher.cuda_indices("cuda:0-2,5") == [0, 1, 2, 5] assert launcher.cuda_indices("cuda") is None and launcher.cuda_indices("auto") is None - with pytest.raises(launcher.LauncherError): - launcher.cuda_indices("cuda:first") + # an empty range would read as no GPU named, so a downward range is an error + for value in ("cuda:first", "cuda:3-0"): + with pytest.raises(launcher.LauncherError): + launcher.cuda_indices(value) def test_gpu_request(): @@ -212,6 +214,13 @@ def test_plan_paths_brun(tmp_path): _, mounts = launcher.plan_paths(["--subjects_list", str(subjects), "--sd", str(tmp_path / "out")], tmp_path) assert set(launcher.minimal_mounts(mounts)) == {a, b, subjects.parent, tmp_path / "out"} + # subject-specific outputs, e.g. a subject's own --sd: created and shared like the general ones + own_sd, log = tmp_path / "other" / "sd", tmp_path / "logs" / "sub2.log" + subjects.write_text(f"sub2={a / 't1.mgz'} --sd {own_sd} --seg_log {log}\n") + _, mounts = launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) + assert {own_sd, log.parent} <= set(launcher.minimal_mounts(mounts)) + assert own_sd.is_dir() and log.parent.is_dir() + out, mounts = launcher.plan_paths(["--subjects", "sub1=a/t1.mgz", "--sd", "out"], tmp_path) assert out[:2] == ["--subjects", f"sub1={a / 't1.mgz'}"] diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 614904ad4..50509b8e4 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -346,6 +346,9 @@ def cuda_indices(value: str) -> list[int] | None: if match is None: raise LauncherError(f"Cannot read the device {value}; use e.g. cuda:0, cuda:0,1 or cuda:0-3.") first, last = int(match.group(1)), int(match.group(2) or match.group(1)) + if last < first: + # an empty range would read as no GPU named, and the launcher would choose one itself + raise LauncherError(f"Cannot read the device {value}; a range has to go upward, e.g. cuda:0-3.") indices.extend(range(first, last + 1)) return indices @@ -422,6 +425,16 @@ def _parent_of_file(path: Path) -> Path: return path if path.is_dir() else path.parent +def _mount_for(flag: str, path: Path, create: bool) -> Path: + """The directory to share for a path flag; an output directory is created, as a mount needs it.""" + if flag not in OUTPUT_FLAGS: + return _parent_of_file(path) + directory = path if flag == "--sd" else path.parent + if create: + directory.mkdir(parents=True, exist_ok=True) + return directory + + def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[str], list[Path]]: """ Make the paths in FastSurfer's arguments absolute and find the directories to share. @@ -451,17 +464,11 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st i += 1 if flag in INPUT_FLAGS | OUTPUT_FLAGS and i < len(args): path = _absolute(args[i], cwd) - if flag in INPUT_FLAGS: - if not path.exists(): - raise LauncherError(f"{flag} {args[i]}: no such file.") - mounts.append(_parent_of_file(path)) - if flag in ("--subjects_list", "--subject_list"): - mounts.extend(_subjects_list_mounts(path)) - else: - directory = path if flag == "--sd" else path.parent - if create: - directory.mkdir(parents=True, exist_ok=True) - mounts.append(directory) + if flag in INPUT_FLAGS and not path.exists(): + raise LauncherError(f"{flag} {args[i]}: no such file.") + mounts.append(_mount_for(flag, path, create)) + if flag in ("--subjects_list", "--subject_list"): + mounts.extend(_subjects_list_mounts(path, create)) out.append(str(path)) i += 1 elif flag == "--subjects": @@ -479,16 +486,22 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st return out, mounts -def _subjects_list_mounts(subjects_list: Path) -> list[Path]: - """The directories of the files a brun_fastsurfer.sh subjects list names, which must be absolute.""" +def _subjects_list_mounts(subjects_list: Path, create: bool = True) -> list[Path]: + """ + The directories to share for a brun_fastsurfer.sh subjects list, whose paths must be absolute. + + Covers the T1 of each subject and the paths in its subject-specific options, inputs and outputs + (e.g. its own --sd), and creates output directories like plan_paths. + """ mounts = [] for number, line in enumerate(subjects_list.read_text().splitlines(), start=1): tokens = shlex.split(line, comments=True) if not tokens: continue _, _, t1 = tokens[0].partition("=") - paths = [t1] + [tokens[k + 1] for k, tok in enumerate(tokens[1:-1], start=1) if tok in INPUT_FLAGS] - for value in paths: + paths = [("--t1", t1)] + [(tok, tokens[k + 1]) for k, tok in enumerate(tokens[1:-1], start=1) + if tok in INPUT_FLAGS | OUTPUT_FLAGS] + for flag, value in paths: # brun_fastsurfer.sh expands nothing in a subjects list, so ~ is not the home directory there if not os.path.isabs(value): raise LauncherError( @@ -496,7 +509,7 @@ def _subjects_list_mounts(subjects_list: Path) -> list[Path]: f"in a subjects list, without ~: the container does not start in the current directory, " f"and nothing in the list is expanded." ) - mounts.append(_parent_of_file(Path(value))) + mounts.append(_mount_for(flag, Path(value), create)) return mounts From 817b26e5cc5748dee3fc1a7e44057aae4d3b5e78 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 07:02:05 +0200 Subject: [PATCH 08/18] Allow GPU lists and ranges in the launcher only with --brun --- test/utils/test_fastsurfer_container.py | 8 +++++++- tools/fastsurfer_container.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 0d43736de..cccc090d1 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -117,7 +117,13 @@ def test_cuda_indices(): def test_gpu_request(): assert launcher.gpu_request([]) == ("auto", None) assert launcher.gpu_request(["--device", "cuda"]) == ("cuda", None) - assert launcher.gpu_request(["--device", "cuda:0-2,5"]) == ("cuda", [0, 1, 2, 5]) + assert launcher.gpu_request(["--device", "cuda:1"]) == ("cuda", [1]) + # lists and ranges only with brun_fastsurfer.sh: run_fastsurfer.sh takes one GPU + assert launcher.gpu_request(["--device", "cuda:0-2,5"], brun=True) == ("cuda", [0, 1, 2, 5]) + for value in ("cuda:0,1", "cuda:0-3"): + for flag in ("--device", "--viewagg_device"): + with pytest.raises(launcher.LauncherError, match="--brun"): + launcher.gpu_request([flag, value]) assert launcher.gpu_request(["--device", "cpu"]) == ("cpu", None) # the view aggregation can need a GPU while the segmentation runs on the CPU assert launcher.gpu_request(["--device", "cpu", "--viewagg_device", "cuda:2"]) == ("cuda", [2]) diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 50509b8e4..c32d33a5b 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -353,12 +353,13 @@ def cuda_indices(value: str) -> list[int] | None: return indices -def gpu_request(args: list[str]) -> tuple[str, list[int] | None]: +def gpu_request(args: list[str], brun: bool = False) -> tuple[str, list[int] | None]: """ Whether FastSurfer would use a GPU with these arguments, and the indices of the GPUs it names. --device decides the segmentation and --viewagg_device the view aggregation, which runs on the CPU - with --device cpu unless it names cuda itself. + with --device cpu unless it names cuda itself. Lists and ranges of GPUs (cuda:0,1, cuda:0-3) are + for brun_fastsurfer.sh only; run_fastsurfer.sh takes one GPU. Returns ------- @@ -369,6 +370,10 @@ def gpu_request(args: list[str]) -> tuple[str, list[int] | None]: """ values = [_last_value(args, "--device") or "auto", _last_value(args, "--viewagg_device") or "auto"] cuda = [v for v in values if v == "cuda" or v.startswith("cuda:")] + several = [v for v in cuda if v.startswith("cuda:") and not re.fullmatch(r"cuda:\d+", v)] + if several and not brun: + raise LauncherError(f"{several[0]}: run_fastsurfer.sh uses one GPU, e.g. cuda:1. Lists and ranges of " + f"GPUs are for processing several subjects with --brun.") named = [i for v in cuda for i in (cuda_indices(v) or [])] use = "cuda" if cuda else ("auto" if values[0] == "auto" else "cpu") return use, sorted(set(named)) if named else None @@ -381,7 +386,7 @@ def plan_build(args: list[str], gpus: list[Gpu], brun: bool) -> Build: Named GPUs are used as they are, and explained in detail if no image supports them. Otherwise the best GPU an image supports is selected, and the CPU only used if there is none. """ - use, named = gpu_request(args) + use, named = gpu_request(args, brun) if use == "cpu": return Build("cpu", False) if named is not None: @@ -697,7 +702,7 @@ def info(message: str, level: str = "INFO") -> None: gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None if options.image: # an own or an older official image: give it the GPUs unless the run is on the CPU - build = Build("custom", gpu_request(args)[0] != "cpu" and bool(gpus)) + build = Build("custom", gpu_request(args, options.brun)[0] != "cpu" and bool(gpus)) image = options.image requested, flag = official_image_version(image), "--image" else: From dde6b40a431ca1d9c7fd7f465f64d5e722d5fdca Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 11:11:15 +0200 Subject: [PATCH 09/18] Build launcher images atomically, follow symlinked inputs and stop on old Singularity --- doc/overview/CONTAINERS.md | 3 + test/utils/test_fastsurfer_container.py | 45 ++++++++++ tools/fastsurfer_container.py | 108 ++++++++++++++++++++---- 3 files changed, 140 insertions(+), 16 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 2179b7080..6cc540171 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -54,6 +54,9 @@ subjects; the paths in a subjects list have to be absolute. `--print_only` print instead of running them, and `--help` lists all launcher options. The sections below explain the commands it runs. +Before starting many runs at once, e.g. a SLURM array, run the launcher once (e.g. on one subject), +so the image is downloaded and built only once rather than by every job. + On a Mac with Apple silicon, use the [macOS package](install/MACOS.md) instead: the images are built for Intel and AMD processors, which such a Mac only emulates, much more slowly and without its GPU. The launcher therefore stops there and points to the package, unless you pass `--runtime docker`. With diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index cccc090d1..ef25c950c 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -130,6 +130,8 @@ def test_gpu_request(): # the last of repeated flags counts, as in the scripts assert launcher.gpu_request(["--device", "cpu", "--device", "cuda:1"]) == ("cuda", [1]) assert launcher.gpu_request(["--device", "cuda:1", "--device", "cpu"]) == ("cpu", None) + # a GPU named only for the view aggregation: the segmentation runs on the first GPU, which counts too + assert launcher.gpu_request(["--viewagg_device", "cuda:2"]) == ("cuda", [0, 2]) def test_plan_build(): @@ -238,6 +240,49 @@ def test_plan_paths_brun(tmp_path): launcher.plan_paths(["--subjects_list", str(subjects)], tmp_path) +def test_link_directories(tmp_path): + # a chain of links, as in a DataLad dataset: the link, a relative link in another directory, the file + for name in ("data", "annex/key", "objects/abc"): + (tmp_path / name).mkdir(parents=True) + target = tmp_path / "objects" / "abc" / "t1.mgz" + target.write_bytes(b"") + (tmp_path / "annex" / "key" / "t1.mgz").symlink_to("../../objects/abc/t1.mgz") + (tmp_path / "data" / "t1.mgz").symlink_to(tmp_path / "annex" / "key" / "t1.mgz") + directories = launcher._link_directories(tmp_path / "data" / "t1.mgz") + real = Path(str(tmp_path)).resolve() + assert {Path(d).resolve() for d in directories} == {real / "data", real / "annex/key", real / "objects/abc"} + # every directory on the way is shared, not a common parent of them + _, mounts = launcher.plan_paths(["--t1", str(tmp_path / "data" / "t1.mgz")], tmp_path) + assert tmp_path not in launcher.minimal_mounts(mounts) + + +@pytest.mark.parametrize( + ("text", "too_old"), + [ + ("apptainer version 1.3.4", False), + ("singularity-ce version 3.11.4", False), + ("singularity version 3.7.4", False), + ("singularity version 3.5.3", True), + ("SingularityCE ...", False), # unreadable: no judgement + ], +) +def test_runtime_too_old(text, too_old): + assert (launcher.runtime_too_old(text) is not None) == too_old + + +def test_build_image(tmp_path): + # a stand-in for `apptainer build ` that writes the target, or fails without writing + builder = tmp_path / "builder.py" + builder.write_text("import sys\nif sys.argv[2] == 'fail': sys.exit(1)\nopen(sys.argv[1], 'w').write('image')\n") + sif = tmp_path / "images" / "fastsurfer-cpu-v2.6.0.sif" + assert launcher.build_image([sys.executable, str(builder), str(sif), "docker://image"]) + assert sif.read_text() == "image" + # a failed build leaves neither the image nor a partial file behind + failed = tmp_path / "images" / "fastsurfer-cu132-v2.6.0.sif" + assert not launcher.build_image([sys.executable, str(builder), str(failed), "fail"]) + assert sorted(p.name for p in sif.parent.iterdir()) == [sif.name] + + def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index c32d33a5b..3635a51a3 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -290,17 +290,21 @@ def choose_build(gpus: list[Gpu], use: str) -> Build: f"CUDA {LEGACY_BUILD[0][0]}.{LEGACY_BUILD[0][1]}. Update the driver to use the default image." ) return Build(LEGACY_BUILD[1], True, notes) - oldest = min(capabilities) if capabilities else None - newest = max(capabilities) if capabilities else None - if oldest is not None and oldest < MIN_CAPABILITY[LEGACY_BUILD[1]]: + driver_reason = (f"The NVIDIA driver {driver} is too old for FastSurfer's images, update it to " + f"{newest_min_driver} or newer") + if not capabilities: + # with unknown capabilities, only the driver can rule the images out + reason = driver_reason + elif min(capabilities) < MIN_CAPABILITY[LEGACY_BUILD[1]]: + oldest = min(capabilities) reason = f"The GPU (compute capability {oldest[0]}.{oldest[1]}) is too old for FastSurfer's images" - elif (driver >= newest_min_driver and oldest < MIN_CAPABILITY[NEWEST_BUILD[1]] - and newest > LEGACY_MAX_CAPABILITY): + elif (driver >= newest_min_driver and min(capabilities) < MIN_CAPABILITY[NEWEST_BUILD[1]] + and max(capabilities) > LEGACY_MAX_CAPABILITY): + oldest, newest = min(capabilities), max(capabilities) reason = (f"No FastSurfer image supports both GPUs of compute capability {oldest[0]}.{oldest[1]} " f"and {newest[0]}.{newest[1]}; select GPUs of one kind with --device cuda:") else: - reason = (f"The NVIDIA driver {driver} is too old for FastSurfer's images, update it to " - f"{newest_min_driver} or newer") + reason = driver_reason if use == "cuda": raise LauncherError(f"{reason}.") return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) @@ -375,6 +379,9 @@ def gpu_request(args: list[str], brun: bool = False) -> tuple[str, list[int] | N raise LauncherError(f"{several[0]}: run_fastsurfer.sh uses one GPU, e.g. cuda:1. Lists and ranges of " f"GPUs are for processing several subjects with --brun.") named = [i for v in cuda for i in (cuda_indices(v) or [])] + if named and values[0] in ("auto", "cuda"): + # only the view aggregation names a GPU; the segmentation then runs on the first one + named.append(0) use = "cuda" if cuda else ("auto" if values[0] == "auto" else "cpu") return use, sorted(set(named)) if named else None @@ -430,14 +437,29 @@ def _parent_of_file(path: Path) -> Path: return path if path.is_dir() else path.parent -def _mount_for(flag: str, path: Path, create: bool) -> Path: - """The directory to share for a path flag; an output directory is created, as a mount needs it.""" +def _link_directories(path: Path) -> list[Path]: + """ + The directories to share so that `path` can be read in the container: its own, and for a symbolic link + that of every link on the way to the file it finally names (e.g. a DataLad or git-annex dataset). + """ + directories, seen = [], set() + while path.is_symlink() and path not in seen: + seen.add(path) + directories.append(path.parent) + # a relative target is relative to the real directory of the link + path = Path(os.path.normpath(Path(os.path.realpath(path.parent)) / os.readlink(path))) + directories.append(_parent_of_file(path)) + return directories + + +def _mount_for(flag: str, path: Path, create: bool) -> list[Path]: + """The directories to share for a path flag; an output directory is created, as a mount needs it.""" if flag not in OUTPUT_FLAGS: - return _parent_of_file(path) + return _link_directories(path) directory = path if flag == "--sd" else path.parent if create: directory.mkdir(parents=True, exist_ok=True) - return directory + return [directory] def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[str], list[Path]]: @@ -471,7 +493,7 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st path = _absolute(args[i], cwd) if flag in INPUT_FLAGS and not path.exists(): raise LauncherError(f"{flag} {args[i]}: no such file.") - mounts.append(_mount_for(flag, path, create)) + mounts.extend(_mount_for(flag, path, create)) if flag in ("--subjects_list", "--subject_list"): mounts.extend(_subjects_list_mounts(path, create)) out.append(str(path)) @@ -485,7 +507,7 @@ def plan_paths(args: list[str], cwd: Path, create: bool = True) -> tuple[list[st path = _absolute(t1, cwd) if not path.exists(): raise LauncherError(f"--subjects {args[i]}: no such file {t1}.") - mounts.append(path.parent) + mounts.extend(_link_directories(path)) out.append(f"{subject}={path}") i += 1 return out, mounts @@ -514,7 +536,9 @@ def _subjects_list_mounts(subjects_list: Path, create: bool = True) -> list[Path f"in a subjects list, without ~: the container does not start in the current directory, " f"and nothing in the list is expanded." ) - mounts.append(_mount_for(flag, Path(value), create)) + if flag not in OUTPUT_FLAGS and not Path(value).exists(): + raise LauncherError(f"{subjects_list}, line {number}: no such file {value}.") + mounts.extend(_mount_for(flag, Path(value), create)) return mounts @@ -527,6 +551,27 @@ def minimal_mounts(paths: list[Path]) -> list[Path]: return sorted(kept) +def runtime_too_old(version_text: str) -> str | None: + """ + Why this Apptainer/Singularity is too old for the launcher's command, None if it is not. + + The command needs --no-mount and --env, which Singularity has from 3.7; every Apptainer has them. + + Parameters + ---------- + version_text : str + The output of ` --version`, e.g. "singularity-ce version 3.11.4". + """ + match = re.search(r"(apptainer|singularity)\S*\s+version\s+(\d+)\.(\d+)", version_text) + if match is None or match.group(1) == "apptainer": + return None + if (int(match.group(2)), int(match.group(3))) >= (3, 7): + return None + return (f"Singularity {match.group(2)}.{match.group(3)} is too old for this launcher, which needs " + f"Singularity 3.7 or newer, or Apptainer. Ask for a newer version on this machine, or write the " + f"container command yourself, see {DOC_URL}.") + + def find_runtime(requested: str) -> str: """The container runtime to use: Apptainer first, then Singularity, then Docker.""" names = ["apptainer", "singularity", "docker"] if requested == "auto" else [requested] @@ -627,6 +672,26 @@ def _license_args(args: list[str], cwd: Path) -> list[str]: return [] +def build_image(command: list[str]) -> bool: + """ + Run ` build ` into a temporary file, and move it in place when it is complete. + + Parallel runs, e.g. the jobs of a SLURM array, and an interrupted build then never leave a partial + image at the path the next run uses. Of two parallel builds, the last one to finish replaces the other. + """ + sif = Path(command[2]) + sif.parent.mkdir(parents=True, exist_ok=True) + partial = sif.with_name(f".{sif.stem}.{os.getpid()}.partial.sif") + try: + if subprocess.run(command[:2] + [str(partial)] + command[3:]).returncode != 0: + return False + os.replace(partial, sif) + return True + finally: + if partial.exists(): + partial.unlink() + + def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="fastsurfer_container.py", @@ -682,6 +747,18 @@ def info(message: str, level: str = "INFO") -> None: f"macOS package and cannot use the GPU. Better {package_hint}", level="WARNING") cwd = Path.cwd() runtime = find_runtime(options.runtime) + if runtime != "docker": + try: + version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, + timeout=30).stdout + except (OSError, subprocess.SubprocessError): + version_text = "" + problem = runtime_too_old(version_text) + if problem is not None: + raise LauncherError(problem) + if options.brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): + # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get + raise LauncherError("--brun needs the subjects in --subjects_list or --subjects = ...") fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) mounts = minimal_mounts(mounts) @@ -769,8 +846,7 @@ def info(message: str, level: str = "INFO") -> None: return 0 for command in commands[:-1]: info(f"Building the image once: {shlex.join(command)}") - Path(command[2]).parent.mkdir(parents=True, exist_ok=True) - if subprocess.run(command).returncode != 0: + if not build_image(command): print("ERROR: Building the image failed.", file=sys.stderr) return 1 info(f"Running: {shlex.join(commands[-1])}") From ddabd865055a098c6a302e53ca48ad132b9636f0 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 12:29:18 +0200 Subject: [PATCH 10/18] Replace --image_version with one --image flag, prefer installed images, and add --update and --update_launcher --- doc/overview/CONTAINERS.md | 27 +- test/utils/test_fastsurfer_container.py | 120 +++++- tools/fastsurfer_container.py | 543 ++++++++++++++++++------ 3 files changed, 528 insertions(+), 162 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 6cc540171..9043c7202 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -44,15 +44,24 @@ python3 fastsurfer_container.py \ --3T --threads 4 ``` -It runs the newest FastSurfer release and says which one. For reproducible results, keep a study on -one version with `--image_version `; for an older version, it says how many releases it is -behind. A GPU that the newest images no longer support runs them on the CPU. The image of an older -release may still support that GPU, but the launcher recommends the newest release on the CPU, or a -machine with a newer GPU, over an old version. Without internet, it uses the newest FastSurfer image -already on the machine. With `--brun`, it runs `brun_fastsurfer.sh` for several -subjects; the paths in a subjects list have to be absolute. `--print_only` prints the commands -instead of running them, and `--help` lists all launcher options. The sections below explain the -commands it runs. +The first run downloads the newest FastSurfer release. Later runs use the image already installed, +also when a newer release comes out: the launcher then says so, and `--update` switches to it. It +says which image it runs, e.g. `cu132-v2.6.0`. To keep a study on exactly that image, pass it with +`--image cu132-v2.6.0`: the image type matters as well as the version, because the CPU and the GPU +images do not compute exactly the same numbers. + +`--image` also takes a version alone (`--image 2.6.0`, the image type is then chosen for the +machine), your own Docker image, or an Apptainer `.sif` file. If several FastSurfer images are +installed, the launcher asks which one to use. It stops if an installed image differs from the +release of the same name, e.g. a local build; `--update` replaces it with the release. For an older +version, it says how many releases it is behind. A GPU that the newest images no longer support runs +them on the CPU. The image of an older release may still support that GPU, but the launcher +recommends the newest release on the CPU, or a machine with a newer GPU, over an old version. + +With `--brun`, it runs `brun_fastsurfer.sh` for several subjects; the paths in a subjects list have +to be absolute. `--print_only` prints the commands instead of running them, `--update_launcher` +replaces the launcher with the current one, and `--help` lists all launcher options. The sections +below explain the commands it runs. Before starting many runs at once, e.g. a SLURM array, run the launcher once (e.g. on one subject), so the image is downloaded and built only once rather than by every job. diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index ef25c950c..c8679f6e1 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -158,15 +158,6 @@ def test_newest_version(): assert launcher.newest_version(tags, "cpu") is None -def test_official_image_version(): - assert launcher.official_image_version("deepmi/fastsurfer:cu118-v2.2.0") == "2.2.0" - assert launcher.official_image_version("docker.io/deepmi/fastsurfer:cpu-v2.6.0") == "2.6.0" - assert launcher.official_image_version("/images/fastsurfer-cu118-v2.2.0.sif") == "2.2.0" - for image in ("deepmi/fastsurfer:latest", "my_fastsurfer:cu132-v2.6.0", "fastsurfer-myimage.sif", - "deepmi/fastsurfer:cu132-v2.7.0-dev0"): - assert launcher.official_image_version(image) is None - - def test_age_note(): releases = [(2, 2, 0), (2, 3, 3), (2, 4, 2), (2, 5, 4), (2, 6, 0)] assert launcher.age_note("2.6.0", releases, gpu_unsupported=True) is None @@ -174,16 +165,115 @@ def test_age_note(): level, message = launcher.age_note("2.5.4", releases, gpu_unsupported=False) assert level == "INFO" and "1 release older" in message # an old GPU image instead of the newest release on the CPU: recommend against it - level, message = launcher.age_note("2.2.0", releases, gpu_unsupported=True, flag="--image") + level, message = launcher.age_note("2.2.0", releases, gpu_unsupported=True) assert level == "WARNING" and "4 releases older than the newest, 2.6.0" in message and "CPU" in message -def test_local_tags_apptainer(tmp_path): - for name in ("fastsurfer-cpu-v2.5.4.sif", "fastsurfer-cpu-v2.6.0.sif", "fastsurfer-myimage.sif", "other.sif"): +@pytest.mark.parametrize( + ("spec", "parsed"), + [ + (None, ("auto", "")), + ("2.6.0", ("version", "2.6.0")), + ("cu132-v2.6.0", ("tag", "cu132-v2.6.0")), + ("cu118-v2.4.2", ("tag", "cu118-v2.4.2")), + ("deepmi/fastsurfer:cu132-v2.6.0", ("release", "cu132-v2.6.0")), + ("docker.io/deepmi/fastsurfer:cpu-v2.6.0", ("release", "cpu-v2.6.0")), + ("deepmi/fastsurfer:latest", ("docker", "deepmi/fastsurfer:latest")), + ("my_fastsurfer:cu132-v2.6.0", ("docker", "my_fastsurfer:cu132-v2.6.0")), + ("/images/fastsurfer-cu132-v2.6.0.sif", ("sif", "/images/fastsurfer-cu132-v2.6.0.sif")), + ], +) +def test_parse_image_spec(spec, parsed): + assert launcher.parse_image_spec(spec) == parsed + + +def test_local_sif_images(tmp_path): + for name in ("fastsurfer-cpu-v2.5.4.sif", "fastsurfer-cpu-v2.6.0.sif", "other.sif", ".fastsurfer-x.1.partial.sif"): (tmp_path / name).write_bytes(b"") - tags = launcher.local_tags("apptainer", tmp_path) - assert sorted(tags) == ["cpu-v2.5.4", "cpu-v2.6.0", "myimage"] - assert launcher.newest_version(tags, "cpu") == "2.6.0" + (tmp_path / "fastsurfer-cpu-v2.6.0.sif.digest").write_text("sha256:c\n") + assert launcher.local_sif_images(tmp_path) == {"cpu-v2.5.4": "", "cpu-v2.6.0": "sha256:c"} + assert launcher.local_sif_images(tmp_path / "missing") == {} + + +HUB = {"cu132-v2.5.4": {"sha256:b"}, "cu132-v2.6.0": {"sha256:a", "sha256:a64"}, "cu126-v2.6.0": {"sha256:d"}, + "cpu-v2.6.0": {"sha256:c"}} +BOTH = {"apptainer", "docker"} + + +def decide(tmp_path, kind="auto", value="", runtime="auto", installed=BOTH, docker=None, sif=None, hub=HUB, + update=False, local_id=None): + return launcher.decide_image(kind, value, "cu132", runtime, installed, docker, sif or {}, hub, tmp_path, + update, local_id) + + +def test_decide_image_nothing_installed(tmp_path): + # the newest release, built for Apptainer, remembering its digest + plan = decide(tmp_path) + assert (plan.runtime, plan.image) == ("apptainer", str(tmp_path / "fastsurfer-cu132-v2.6.0.sif")) + assert plan.build_from == "docker://deepmi/fastsurfer:cu132-v2.6.0" and plan.record in HUB["cu132-v2.6.0"] + assert "--image cu132-v2.6.0" in plan.notes[0][1] + assert decide(tmp_path, installed={"docker"}).image == "deepmi/fastsurfer:cu132-v2.6.0" + with pytest.raises(launcher.LauncherError, match="cannot be reached"): + decide(tmp_path, hub=None) + + +def test_decide_image_one_installed(tmp_path): + # the installed image, where it is, and a note about the newer release + plan = decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}) + assert (plan.runtime, plan.image, plan.pull) == ("docker", "deepmi/fastsurfer:cu132-v2.5.4", False) + assert any("2.6.0 is available" in note for _, note in plan.notes) + # offline, it is used all the same + assert decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, hub=None).image.endswith("cu132-v2.5.4") + # the same version in both places counts once, and runs with Apptainer + plan = decide(tmp_path, docker={"cu132-v2.6.0": "sha256:a"}, sif={"cu132-v2.6.0": "sha256:a"}) + assert plan.runtime == "apptainer" and plan.build_from is None + # images of another type do not count + assert decide(tmp_path, docker={"cpu-v2.5.4": "sha256:x"}).tag == "cu132-v2.6.0" + + +def test_decide_image_several_installed(tmp_path): + with pytest.raises(launcher.LauncherError, match="Several"): + decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}) + # --update takes the newest, and lists the older ones with how to remove them + plan = decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}, update=True) + assert plan.build_from is not None and plan.tag == "cu132-v2.6.0" + assert any("docker image rm deepmi/fastsurfer:cu132-v2.5.4" in note for _, note in plan.notes) + + +def test_decide_image_digest(tmp_path): + # a local build tagged like a release: stop, unless named in full or replaced with --update + local_build = {"cu132-v2.6.0": ""} + with pytest.raises(launcher.LauncherError, match="differs from the release"): + decide(tmp_path, docker=local_build) + with pytest.raises(launcher.LauncherError, match="differs from the release"): + decide(tmp_path, kind="tag", value="cu132-v2.6.0", docker=local_build) + assert decide(tmp_path, kind="release", value="cu132-v2.6.0", docker=local_build).runtime == "docker" + assert decide(tmp_path, docker=local_build, update=True).pull + # either digest of the release matches, and a .sif without a recorded digest is not judged + assert decide(tmp_path, docker={"cu132-v2.6.0": "sha256:a64"}).runtime == "docker" + assert decide(tmp_path, sif={"cu132-v2.6.0": ""}).build_from is None + with pytest.raises(launcher.LauncherError, match="differs from the release"): + decide(tmp_path, sif={"cu132-v2.6.0": "sha256:other"}) + + +def test_decide_image_named(tmp_path): + # a version the type of this machine was not released in: the images of that version are listed + with pytest.raises(launcher.LauncherError, match="cpu-v2.6.0"): + decide(tmp_path, kind="version", value="2.6.0", hub={"cpu-v2.6.0": {"sha256:c"}}) + with pytest.raises(launcher.LauncherError, match="not released"): + decide(tmp_path, kind="tag", value="cu132-v9.9.9") + with pytest.raises(launcher.LauncherError, match="Docker"): + decide(tmp_path, kind="sif", value="x.sif", runtime="docker") + # an own Docker image runs with Docker, or with Apptainer when asked, converted from the local Docker + assert decide(tmp_path, kind="docker", value="my_fs:cu132", local_id="sha256:1").runtime == "docker" + plan = decide(tmp_path, kind="docker", value="my_fs:cu132", runtime="apptainer", local_id="sha256:1") + assert (plan.build_from, plan.record) == ("docker-daemon://my_fs:cu132", "sha256:1") + (tmp_path / "my_fs-cu132.sif").write_bytes(b"") + (tmp_path / "my_fs-cu132.sif.digest").write_text("sha256:1\n") + own = {"kind": "docker", "value": "my_fs:cu132", "runtime": "apptainer"} + assert decide(tmp_path, **own, local_id="sha256:1").build_from is None + # a changed local Docker image is converted again + assert decide(tmp_path, **own, local_id="sha256:2").build_from def test_plan_paths_run(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 3635a51a3..45b69bd5c 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -21,8 +21,9 @@ python3 fastsurfer_container.py --t1 data/t1.nii.gz --sd results --sid subject1 -It runs the newest FastSurfer release unless --image_version names another, picks the image that -fits the NVIDIA GPU and driver (or the CPU image), downloads it once, +It runs the FastSurfer release image installed on the machine (the newest release if there is none, +or the image --image names), picks the image type that fits the NVIDIA GPU and driver (or the CPU +image), downloads it once, shares every file and directory the flags name with the container at the same path, passes the FreeSurfer license from $FS_LICENSE if --fs_license is not given, and runs the container as the calling user. @@ -46,6 +47,9 @@ DOCKER_REPO = "deepmi/fastsurfer" DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/CONTAINERS.html" +LAUNCHER_URL = "https://raw.githubusercontent.com/Deep-MI/FastSurfer/stable/tools/fastsurfer_container.py" +# seconds to wait for Docker Hub and GitHub, so a machine without internet is not held up +HUB_TIMEOUT = 5 MACOS_PACKAGE_URL = "https://github.com/Deep-MI/FastSurfer/releases/latest/download/FastSurfer-macos-darwin_arm64.pkg" MACOS_DOC_URL = "https://deep-mi.org/FastSurfer/stable/overview/install/MACOS.html" # the 15 GB the macOS page asks Docker to have, less what the kernel of Docker's VM keeps for itself @@ -119,49 +123,18 @@ def newest_version(tags: list[str], prefix: str) -> str | None: return ".".join(str(v) for v in versions[-1]) if versions else None -def published_tags(name: str) -> list[str] | None: - """The tags of the image on Docker Hub that contain `name`; None if Docker Hub cannot be reached.""" - import json - import urllib.parse - import urllib.request - - url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name={urllib.parse.quote(name)}" - tags: list[str] = [] - try: - while url and len(tags) < 1000: - with urllib.request.urlopen(url, timeout=20) as response: - page = json.load(response) - tags += [result["name"] for result in page.get("results", [])] - url = page.get("next") - except (OSError, ValueError, KeyError): - return None - return tags - - -def official_image_version(image: str) -> str | None: - """The FastSurfer version of an official image, e.g. 2.2.0 for deepmi/fastsurfer:cu118-v2.2.0, else None.""" - name = Path(image).name if image.endswith(".sif") else image - pattern = (rf"{re.escape(DOCKER_REPO.rsplit('/', 1)[-1])}-[a-z0-9.]+-v(.+)\.sif" if image.endswith(".sif") - else rf"(?:docker\.io/)?{re.escape(DOCKER_REPO)}:[a-z0-9.]+-v(.+)") - match = re.fullmatch(pattern, name) - return match.group(1) if match and parse_version(match.group(1)) else None - - -def age_note(version: str, releases: list[tuple[int, ...]], gpu_unsupported: bool, - flag: str = "--image_version") -> tuple[str, str] | None: +def age_note(version: str, releases: list[tuple[int, ...]], gpu_unsupported: bool) -> tuple[str, str] | None: """ A note for a run of an older release, as (level, message), None for the newest or an unknown version. Parameters ---------- version : str - The requested version. + The version --image requested. releases : list[tuple[int, ...]] All releases, oldest first. gpu_unsupported : bool Whether the newest images do not support this GPU, so an old image is the only way to use it. - flag : str, default="--image_version" - The flag that requested the version. """ requested = parse_version(version) if requested is None or not releases or requested >= releases[-1]: @@ -172,24 +145,10 @@ def age_note(version: str, releases: list[tuple[int, ...]], gpu_unsupported: boo if gpu_unsupported: return "WARNING", ( f"{message} The images of newer versions do not support this GPU, so they run on the CPU here. " - f"Running the newest release on the CPU (leave out --image and --image_version), or on a machine " - f"with a newer GPU, is recommended over this old version, which lacks the improvements and fixes " - f"since." + f"Running the newest release on the CPU (--update without --image), or on a machine with a newer " + f"GPU, is recommended over this old version, which lacks the improvements and fixes since." ) - return "INFO", f"{message} Leave out {flag} to run the newest release." - - -def local_tags(runtime: str, image_dir: Path) -> list[str]: - """The tags of the FastSurfer images already on this machine, for runs without internet.""" - if runtime != "docker": - prefix = DOCKER_REPO.rsplit("/", 1)[-1] + "-" - return [p.name[len(prefix):-len(".sif")] for p in image_dir.glob(f"{prefix}*.sif")] - try: - out = subprocess.run(["docker", "image", "ls", DOCKER_REPO, "--format", "{{.Tag}}"], - capture_output=True, text=True, timeout=60) - except (OSError, subprocess.SubprocessError): - return [] - return out.stdout.split() if out.returncode == 0 else [] + return "INFO", f"{message} --update without --image switches to the newest release." def query_nvidia_gpus() -> list[Gpu]: @@ -411,22 +370,278 @@ def image_reference(prefix: str, version: str) -> str: return f"{DOCKER_REPO}:{prefix}-v{version}" -def resolve_version(prefix: str, runtime: str, image_dir: Path) -> tuple[str, str]: +RELEASE_TAG = re.compile(r"([a-z0-9.]+)-v(\d+\.\d+\.\d+)") +# official Apptainer images are named fastsurfer-.sif, as in the documentation +SIF_PREFIX = DOCKER_REPO.rsplit("/", 1)[-1] + "-" + + +def hub_tags() -> dict[str, set[str]] | None: """ - The newest released FastSurfer version for this image, and where that was found. + The release tags on Docker Hub and the digests each is known by, None if Docker Hub cannot be reached. - Asks Docker Hub, and without internet looks at the images already on this machine. + One request per run: every decision about versions comes from this list. A tag is known by the + digest of its index and those of its platform images, so a local copy matches whichever one Docker + recorded. """ - version = newest_version(published_tags(f"{prefix}-v") or [], prefix) - if version is not None: - return version, "the newest release" - version = newest_version(local_tags(runtime, image_dir), prefix) - if version is not None: - return version, "the newest image on this machine, Docker Hub could not be reached" - raise LauncherError( - "Could not find out the newest FastSurfer release, because Docker Hub cannot be reached and no " - "FastSurfer image is on this machine yet. Pass the version with --image_version, e.g. 2.6.0." - ) + import json + import urllib.request + + url = f"https://hub.docker.com/v2/repositories/{DOCKER_REPO}/tags?page_size=100&name=-v" + tags: dict[str, set[str]] = {} + try: + while url and len(tags) < 2000: + with urllib.request.urlopen(url, timeout=HUB_TIMEOUT) as response: + page = json.load(response) + for result in page.get("results", []): + digests = {result.get("digest")} | {image.get("digest") for image in result.get("images", [])} + tags[result["name"]] = {d for d in digests if d} + url = page.get("next") + except (OSError, ValueError, KeyError): + return None + return tags + + +def local_docker_images() -> dict[str, str] | None: + """ + The tags of the official images in the local Docker with the digest each was pulled with. + + The digest is "" for an image not pulled from Docker Hub, e.g. a local build tagged like a release. + None if Docker is not installed or not running. + """ + if shutil.which("docker") is None: + return None + try: + out = subprocess.run(["docker", "image", "ls", DOCKER_REPO, "--format", "{{.Tag}}\t{{.Digest}}"], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + images = {} + for line in out.stdout.splitlines(): + tag, _, digest = line.partition("\t") + if tag and tag != "": + images[tag] = "" if digest in ("", "") else digest + return images + + +def local_docker_id(reference: str) -> str | None: + """The ID of an image in the local Docker, None if it is not there.""" + if shutil.which("docker") is None: + return None + try: + out = subprocess.run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() or None if out.returncode == 0 else None + + +def sif_path(image_dir: Path, tag: str) -> Path: + """Where the Apptainer image of a release tag is kept, e.g. fastsurfer-cu132-v2.6.0.sif.""" + return image_dir / f"{SIF_PREFIX}{tag}.sif" + + +def _recorded(sif: Path) -> str: + """What the launcher wrote next to an image it built: the digest or image ID of its source, or "".""" + record = sif.with_name(sif.name + ".digest") + return record.read_text().strip() if record.is_file() else "" + + +def local_sif_images(image_dir: Path) -> dict[str, str]: + """The tags of the Apptainer images in image_dir, with the digest recorded when they were built ("").""" + if not image_dir.is_dir(): + return {} + return {p.name[len(SIF_PREFIX):-len(".sif")]: _recorded(p) for p in image_dir.glob(f"{SIF_PREFIX}*.sif")} + + +def parse_image_spec(spec: str | None) -> tuple[str, str]: + """ + What --image names. + + Returns + ------- + tuple[str, str] + ("auto", "") without --image, ("version", "2.6.0"), ("tag", "cu132-v2.6.0") for a release tag, + ("release", "cu132-v2.6.0") for the full reference deepmi/fastsurfer:cu132-v2.6.0, ("sif", path) + for an Apptainer file and ("docker", reference) for any other Docker image. + """ + if not spec: + return "auto", "" + if spec.endswith(".sif"): + return "sif", spec + if parse_version(spec): + return "version", spec + if RELEASE_TAG.fullmatch(spec): + return "tag", spec + official = re.fullmatch(rf"(?:docker\.io/)?{re.escape(DOCKER_REPO)}:(.+)", spec) + if official and RELEASE_TAG.fullmatch(official.group(1)): + return "release", official.group(1) + return "docker", spec + + +@dataclass +class ImagePlan: + """The image to run and how to get it.""" + + runtime: str + # the Docker reference, or the path of the .sif file for Apptainer + image: str + # the source to build the .sif file from, docker://... or docker-daemon://..., if it has to be built + build_from: str | None = None + # what to write next to a built .sif file, to recognise its source later + record: str | None = None + pull: bool = False + tag: str | None = None + notes: list[tuple[str, str]] = field(default_factory=list) + + +def _apptainer_runtime(runtime: str, installed: set[str]) -> str | None: + """The Apptainer or Singularity to use, None if none is requested or installed.""" + if runtime in ("apptainer", "singularity"): + return runtime + if runtime == "auto": + return next((name for name in ("apptainer", "singularity") if name in installed), None) + return None + + +def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: set[str], + docker_images: dict[str, str] | None, sif_images: dict[str, str], + hub: dict[str, set[str]] | None, image_dir: Path, update: bool, + local_id: str | None = None) -> ImagePlan: + """ + Which image to run, with which runtime, and whether it has to be downloaded or built first. + + Without --image, the official release image of this machine's type (`prefix`) already installed is + used, wherever it is; several stop the launcher, none downloads the newest release. With --update, + the newest release (or the one named) is downloaded again. A release image whose content differs + from Docker Hub stops the launcher, unless it was named in full. + + Parameters + ---------- + kind, value : str + What --image names, see parse_image_spec. + prefix : str + The image type for this machine, e.g. cu132. + runtime : str + --runtime: auto, apptainer, singularity or docker. + installed : set[str] + The runtimes installed. + docker_images : dict[str, str], None + Tags and digests of the official images in the local Docker, None without Docker. + sif_images : dict[str, str] + Tags and recorded digests of the official images in `image_dir`. + hub : dict[str, set[str]], None + Release tags and digests on Docker Hub, None if it cannot be reached. + image_dir : Path + Where Apptainer images are kept. + update : bool + --update: download or build the image again. + local_id : str, None + For a "docker" image, its ID in the local Docker, None if it is not there. + """ + apptainer = _apptainer_runtime(runtime, installed) + docker_images = docker_images or {} + + def require(name: str) -> str: + if name not in installed: + raise LauncherError(f"{name} is not installed.") + return name + + def place_release(tag: str, checked: bool, forced: bool = update) -> ImagePlan: + in_docker, in_sif = tag in docker_images, tag in sif_images + if runtime == "docker" or (runtime == "auto" and not (in_sif and apptainer) + and ((in_docker and "docker" in installed) or apptainer is None)): + chosen = require("docker") + else: + chosen = require(apptainer) + local = docker_images.get(tag) if chosen == "docker" else sif_images.get(tag) + known = hub.get(tag) if hub is not None else None + # a recorded digest of "" on a .sif file means it was not built by the launcher: nothing to compare + comparable = local is not None and (chosen == "docker" or local) + if checked and not forced and known and comparable and local not in known: + where = (f"Docker image {DOCKER_REPO}:{tag}" if chosen == "docker" + else f"image {sif_path(image_dir, tag)}") + explicit = f"{DOCKER_REPO}:{tag}" if chosen == "docker" else str(sif_path(image_dir, tag)) + raise LauncherError(f"The local {where} differs from the release on Docker Hub, e.g. a local build " + f"or an image replaced since. --update replaces it with the release; " + f"--image {explicit} runs the local one anyway.") + if chosen == "docker": + return ImagePlan("docker", f"{DOCKER_REPO}:{tag}", pull=forced, tag=tag) + sif = sif_path(image_dir, tag) + build = forced or not in_sif + record = sorted(known)[0] if known and build else None + return ImagePlan(chosen, str(sif), build_from=f"docker://{DOCKER_REPO}:{tag}" if build else None, + record=record, tag=tag) + + def release_exists(tag: str) -> None: + if hub is not None and tag not in hub and tag not in docker_images and tag not in sif_images: + version = tag.rsplit("-v", 1)[-1] + images = sorted(t for t in hub if t.endswith(f"-v{version}")) + raise LauncherError(f"There is no image {DOCKER_REPO}:{tag}. " + ( + f"The images of FastSurfer {version} are {', '.join(images)}." if images + else f"FastSurfer {version} is not released.")) + + if kind == "sif": + if runtime == "docker": + raise LauncherError("A .sif image needs Apptainer or Singularity, not Docker.") + if apptainer is None: + raise LauncherError("A .sif image needs Apptainer or Singularity, and neither is installed.") + return ImagePlan(require(apptainer), value) + if kind == "docker": + if runtime == "docker" or (runtime == "auto" and (local_id is not None or apptainer is None)): + return ImagePlan(require("docker"), value, pull=update) + # Apptainer runs a Docker image as a .sif file, converted from the local Docker if it is there + sif = image_dir / (value.rsplit("/", 1)[-1].replace(":", "-") + ".sif") + source = f"docker-daemon://{value}" if local_id else f"docker://{value}" + stale = sif.is_file() and local_id is not None and _recorded(sif) != local_id + build = update or not sif.is_file() or stale + return ImagePlan(require(apptainer), str(sif), build_from=source if build else None, + record=local_id if build else None) + if kind in ("tag", "release"): + release_exists(value) + return place_release(value, checked=kind == "tag") + if kind == "version": + tag = f"{prefix}-v{value}" + release_exists(tag) + return place_release(tag, checked=True) + + # no --image: the installed release image of this type, or the newest release + candidates = sorted({t for t in [*docker_images, *sif_images] + if (m := RELEASE_TAG.fullmatch(t)) and m.group(1) == prefix}, + key=lambda t: parse_version(t.rsplit("-v", 1)[-1])) + newest = newest_version(list(hub), prefix) if hub is not None else None + if update or not candidates: + if newest is None: + raise LauncherError("Docker Hub cannot be reached, so the newest FastSurfer release is not known " + + ("and --update cannot download it." if update else + "and no FastSurfer image is on this machine yet. Pass the image with --image, " + "e.g. --image 2.6.0.")) + tag = f"{prefix}-v{newest}" + plan = place_release(tag, checked=True, forced=update) + old = [t for t in candidates if t != tag] + if old: + removal = [f"docker image rm {DOCKER_REPO}:{t}" for t in old if t in docker_images] + removal += [f"rm {sif_path(image_dir, t)}" for t in old if t in sif_images] + plan.notes.append(("INFO", f"Older images are still installed ({', '.join(old)}), so the next run " + f"would ask which one to use. Remove them with: {'; '.join(removal)}")) + elif len(candidates) == 1: + tag = candidates[0] + plan = place_release(tag, checked=True) + version = tag.rsplit("-v", 1)[-1] + if newest is not None and parse_version(newest) > parse_version(version): + plan.notes.append(("INFO", f"FastSurfer {newest} is available ({prefix}-v{newest}); --update " + f"switches to it.")) + else: + def stores(t: str) -> str: + return " and ".join(s for s, has in (("Docker", t in docker_images), ("Apptainer", t in sif_images)) if has) + + where = [f"{t} ({stores(t)})" for t in candidates] + raise LauncherError(f"Several FastSurfer images are installed: {'; '.join(where)}. Choose one with " + f"--image , e.g. --image {candidates[-1]}, or remove the others.") + plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a study, " + f"keep this image with --image {plan.tag}.")) + return plan def _absolute(value: str, cwd: Path) -> Path: @@ -572,17 +787,6 @@ def runtime_too_old(version_text: str) -> str | None: f"container command yourself, see {DOC_URL}.") -def find_runtime(requested: str) -> str: - """The container runtime to use: Apptainer first, then Singularity, then Docker.""" - names = ["apptainer", "singularity", "docker"] if requested == "auto" else [requested] - for name in names: - if shutil.which(name): - return name - if requested == "auto": - raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") - raise LauncherError(f"{requested} is not installed.") - - # CUDA numbers GPUs fastest first, nvidia-smi and the launcher by PCI bus; this makes them agree CUDA_ORDER = "CUDA_DEVICE_ORDER=PCI_BUS_ID" @@ -692,6 +896,43 @@ def build_image(command: list[str]) -> bool: partial.unlink() +def stable_launcher() -> bytes | None: + """The launcher on the stable branch, which knows the images of the newest release; None if unreachable.""" + import urllib.request + + try: + with urllib.request.urlopen(LAUNCHER_URL, timeout=HUB_TIMEOUT) as response: + return response.read() + except OSError: + return None + + +def update_launcher() -> int: + """--update_launcher: replace this file with the launcher of the stable branch.""" + me = Path(__file__).resolve() + if any((parent / ".git").exists() for parent in me.parents): + print(f"ERROR: {me} is part of a git checkout, update it with git pull instead.", file=sys.stderr) + return 1 + new = stable_launcher() + if new is None: + print(f"ERROR: Could not download {LAUNCHER_URL}.", file=sys.stderr) + return 1 + try: + compile(new, me.name, "exec") + except SyntaxError: + print(f"ERROR: {LAUNCHER_URL} is not a valid launcher, {me} is unchanged.", file=sys.stderr) + return 1 + if new == me.read_bytes(): + print(f"INFO: {me} is up to date.", file=sys.stderr) + return 0 + partial = me.with_name(f".{me.name}.{os.getpid()}.new") + partial.write_bytes(new) + os.chmod(partial, me.stat().st_mode) + os.replace(partial, me) + print(f"INFO: Updated {me} from {LAUNCHER_URL}.", file=sys.stderr) + return 0 + + def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="fastsurfer_container.py", @@ -703,16 +944,22 @@ def make_parser() -> argparse.ArgumentParser: ), allow_abbrev=False, ) - parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", - help="the container runtime (default: auto, the first installed of these)") - parser.add_argument("--image_version", - help="the FastSurfer version to run, e.g. 2.6.0 (default: the newest release)") parser.add_argument("--image", - help="run this image instead: a Docker image (e.g. my_fastsurfer:cu132) or an Apptainer " - ".sif file") + help="the image to run: a release tag such as cu132-v2.6.0 (to keep a study on one " + "image), a version such as 2.6.0 (the image type is chosen for this machine), a " + "Docker image (e.g. my_fastsurfer:cu132) or an Apptainer .sif file. Default: the " + "release image installed on this machine, or the newest release if there is none") + parser.add_argument("--update", action="store_true", + help="download the newest release (or the one --image names) again, even if an image " + "is installed, and run it") + parser.add_argument("--update_launcher", action="store_true", + help="replace this launcher with the current one and stop") + parser.add_argument("--runtime", choices=["auto", "apptainer", "singularity", "docker"], default="auto", + help="the container runtime (default: auto, the one that has the image, else Apptainer, " + "Singularity, then Docker)") parser.add_argument("--image_dir", type=Path, default=Path(os.environ.get("FASTSURFER_IMAGE_DIR", "~/my_singularity_images")).expanduser(), - help="where Apptainer/Singularity images are stored (default: $FASTSURFER_IMAGE_DIR or " + help="where Apptainer/Singularity images are kept (default: $FASTSURFER_IMAGE_DIR or " "~/my_singularity_images)") parser.add_argument("--brun", action="store_true", help="run brun_fastsurfer.sh to process several subjects") parser.add_argument("--runtime_args", default="", @@ -724,6 +971,8 @@ def make_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: parser = make_parser() options, fastsurfer_args = parser.parse_known_args(argv) + if options.update_launcher: + return update_launcher() if not fastsurfer_args: parser.print_help() return 1 @@ -745,20 +994,16 @@ def info(message: str, level: str = "INFO") -> None: if apple_silicon: info(f"Docker on a Mac with Apple silicon emulates the Intel image, which is much slower than the " f"macOS package and cannot use the GPU. Better {package_hint}", level="WARNING") - cwd = Path.cwd() - runtime = find_runtime(options.runtime) - if runtime != "docker": - try: - version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, - timeout=30).stdout - except (OSError, subprocess.SubprocessError): - version_text = "" - problem = runtime_too_old(version_text) - if problem is not None: - raise LauncherError(problem) + installed = {name for name in ("apptainer", "singularity", "docker") if shutil.which(name)} + if not installed: + raise LauncherError(f"Neither Apptainer, Singularity nor Docker is installed, see {DOC_URL}.") + if options.runtime != "auto" and options.runtime not in installed: + raise LauncherError(f"{options.runtime} is not installed.") if options.brun and not {"--subjects", "--subjects_list", "--subject_list"} & set(fastsurfer_args): # brun_fastsurfer.sh would read the subjects from stdin, which the container does not get raise LauncherError("--brun needs the subjects in --subjects_list or --subjects = ...") + + cwd = Path.cwd() fastsurfer_args = fastsurfer_args + _license_args(fastsurfer_args, cwd) args, mounts = plan_paths(fastsurfer_args, cwd, create=not options.print_only) mounts = minimal_mounts(mounts) @@ -771,42 +1016,56 @@ def info(message: str, level: str = "INFO") -> None: if "--fs_license" not in args and "--seg_only" not in args: info("No FreeSurfer license given (--fs_license or $FS_LICENSE); the surfaces need one.") - if options.image and options.image.endswith(".sif") and runtime == "docker": - raise LauncherError("A .sif image needs Apptainer or Singularity: pass --runtime apptainer or " - "--runtime singularity, or a Docker image with --image.") + # the GPUs and the image type + kind, value = parse_image_spec(options.image) gpus = query_nvidia_gpus() # whether the newest images support none of the GPUs, so they run on the CPU here gpu_unsupported = bool(gpus) and select_gpus(gpus, False) is None - if options.image: - # an own or an older official image: give it the GPUs unless the run is on the CPU - build = Build("custom", gpu_request(args, options.brun)[0] != "cpu" and bool(gpus)) - image = options.image - requested, flag = official_image_version(image), "--image" - else: + wants_gpu = gpu_request(args, options.brun)[0] != "cpu" and bool(gpus) + if kind in ("auto", "version"): build = plan_build(args, gpus, options.brun) - requested, flag = options.image_version, "--image_version" - version = requested - if version is None: - version, source = resolve_version(build.prefix, runtime, options.image_dir) - info(f"Running FastSurfer {version}, {source}. Pass --image_version {version} to keep " - f"using this version when newer ones are released.") - image = image_reference(build.prefix, version) - if requested is not None: - # an older version may have no image for this GPU, e.g. none for a newer CUDA version - available = published_tags(f"-v{requested}") - tag = image.rsplit(":", 1)[-1] - if available is not None and tag not in available: - images = sorted(t for t in available if t.endswith(f"-v{requested}")) - raise LauncherError( - f"There is no image {image}. " - + (f"The images of FastSurfer {requested} are {', '.join(images)}; pass one with " - f"--image {DOCKER_REPO}:." if images else f"FastSurfer {requested} is not released.") - ) - if requested is not None: - note = age_note(requested, release_versions(published_tags("cpu-v") or [], "cpu"), - gpu_unsupported and build.gpu, flag) + elif kind in ("tag", "release"): + prefix = RELEASE_TAG.fullmatch(value).group(1) + build = Build(prefix, prefix != "cpu" and wants_gpu) + else: + build = Build("custom", wants_gpu) + + # the image: one request to Docker Hub decides about versions + hub = None if kind in ("sif", "docker") else hub_tags() + if hub is not None and kind in ("auto", "version") and build.prefix != "cpu": + newest_release = newest_version(list(hub), "cpu") + if newest_release and f"{build.prefix}-v{newest_release}" not in hub: + raise LauncherError(f"FastSurfer {newest_release} has no {build.prefix} image, which means this " + f"launcher does not know the images of the newest release. Update it with " + f"--update_launcher, or download it again from {LAUNCHER_URL}.") + local_id = local_docker_id(value) if kind == "docker" else None + plan = decide_image(kind, value, build.prefix, options.runtime, installed, + local_docker_images() if "docker" in installed else None, + local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id) + runtime = plan.runtime + if hub is not None: + stable = stable_launcher() + if stable is not None and stable != Path(__file__).resolve().read_bytes(): + info("A newer launcher is available, update it with --update_launcher.") + + if runtime != "docker": + try: + version_text = subprocess.run([runtime, "--version"], capture_output=True, text=True, + timeout=30).stdout + except (OSError, subprocess.SubprocessError): + version_text = "" + problem = runtime_too_old(version_text) + if problem is not None: + raise LauncherError(problem) + + requested = value if kind == "version" else (plan.tag if kind in ("tag", "release") else None) + if requested is not None and hub is not None: + version = requested.rsplit("-v", 1)[-1] + note = age_note(version, release_versions(list(hub), "cpu"), gpu_unsupported and build.gpu) if note is not None: info(note[1], level=note[0]) + for level, note in plan.notes: + info(note, level=level) for note in build.notes: info(note) @@ -816,8 +1075,10 @@ def info(message: str, level: str = "INFO") -> None: if runtime == "docker": user = f"{os.getuid()}:{os.getgid()}" # official images are built for Intel/AMD only; an own image may be built for this Mac - emulate = apple_silicon and (not options.image or official_image_version(options.image) is not None) - commands.append(docker_command(image, build.gpu, mounts, script, args, extra, user, emulate, + emulate = apple_silicon and (kind != "docker" or value.startswith(DOCKER_REPO)) + if plan.pull: + commands.append(["docker", "pull"] + (["--platform", "linux/amd64"] if emulate else []) + [plan.image]) + commands.append(docker_command(plan.image, build.gpu, mounts, script, args, extra, user, emulate, build.devices)) if not options.print_only: memory = docker_memory_gib() @@ -827,15 +1088,12 @@ def info(message: str, level: str = "INFO") -> None: info(f"Docker can use only {memory:.1f} GB of memory, and FastSurfer may fail with less than " f"15 GB. Increase the memory {where}.", level="WARNING") else: - if image.endswith(".sif"): - sif = _absolute(image, cwd) - if not sif.is_file(): - raise LauncherError(f"--image {image}: no such file.") - else: - sif = options.image_dir / (image.rsplit("/", 1)[-1].replace(":", "-") + ".sif") - if not sif.is_file(): - commands.append([runtime, "build", str(sif), f"docker://{image}"]) - commands.append(apptainer_command(runtime, sif, build.gpu, mounts, script, args, extra, build.devices)) + if kind == "sif" and not Path(plan.image).is_file(): + raise LauncherError(f"--image {value}: no such file.") + if plan.build_from is not None: + commands.append([runtime, "build", plan.image, plan.build_from]) + commands.append(apptainer_command(runtime, Path(plan.image), build.gpu, mounts, script, args, extra, + build.devices)) except LauncherError as error: print(f"ERROR: {error}", file=sys.stderr) return 1 @@ -845,9 +1103,18 @@ def info(message: str, level: str = "INFO") -> None: print(shlex.join(command)) return 0 for command in commands[:-1]: - info(f"Building the image once: {shlex.join(command)}") - if not build_image(command): - print("ERROR: Building the image failed.", file=sys.stderr) + if command[1] == "build": + info(f"Building the image once: {shlex.join(command)}") + if not build_image(command): + print("ERROR: Building the image failed.", file=sys.stderr) + return 1 + record = Path(command[2]).with_name(Path(command[2]).name + ".digest") + if plan.record: + record.write_text(plan.record + "\n") + elif record.exists(): + record.unlink() + elif subprocess.run(command).returncode != 0: + print(f"ERROR: {shlex.join(command)} failed.", file=sys.stderr) return 1 info(f"Running: {shlex.join(commands[-1])}") sys.stdout.flush() From fa627b4f17a71781f738575aacad0b33e8a684a6 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:13:32 +0200 Subject: [PATCH 11/18] Count installed images of older types that run on the machine in the launcher --- doc/overview/CONTAINERS.md | 3 +- test/utils/test_fastsurfer_container.py | 30 ++++++++++++- tools/fastsurfer_container.py | 57 +++++++++++++++++++------ 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/doc/overview/CONTAINERS.md b/doc/overview/CONTAINERS.md index 9043c7202..4738085ca 100644 --- a/doc/overview/CONTAINERS.md +++ b/doc/overview/CONTAINERS.md @@ -45,7 +45,8 @@ python3 fastsurfer_container.py \ ``` The first run downloads the newest FastSurfer release. Later runs use the image already installed, -also when a newer release comes out: the launcher then says so, and `--update` switches to it. It +also one of an earlier release or CUDA version, as long as it runs on the machine, and also when a +newer release comes out: the launcher then says so, and `--update` switches to it. It says which image it runs, e.g. `cu132-v2.6.0`. To keep a study on exactly that image, pass it with `--image cu132-v2.6.0`: the image type matters as well as the version, because the CPU and the GPU images do not compute exactly the same numbers. diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index c8679f6e1..9f7fd88b0 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -201,9 +201,9 @@ def test_local_sif_images(tmp_path): def decide(tmp_path, kind="auto", value="", runtime="auto", installed=BOTH, docker=None, sif=None, hub=HUB, - update=False, local_id=None): + update=False, local_id=None, gpus=None): return launcher.decide_image(kind, value, "cu132", runtime, installed, docker, sif or {}, hub, tmp_path, - update, local_id) + update, local_id, gpus) def test_decide_image_nothing_installed(tmp_path): @@ -231,6 +231,32 @@ def test_decide_image_one_installed(tmp_path): assert decide(tmp_path, docker={"cpu-v2.5.4": "sha256:x"}).tag == "cu132-v2.6.0" +def test_image_type_runs(): + turing = [gpu((7, 5), driver=580)] + assert launcher.image_type_runs("cu132", turing) and launcher.image_type_runs("cu126", turing) + # older CUDA types are judged by the driver alone + assert launcher.image_type_runs("cu128", turing) and launcher.image_type_runs("cu118", turing) + assert not launcher.image_type_runs("cu128", [gpu((7, 5), driver=470)]) + assert not launcher.image_type_runs("cu132", [gpu((6, 1))]) # Pascal, dropped by CUDA 13 + # the CPU image only for a run on the CPU, and no AMD or unknown types on an NVIDIA GPU + assert not launcher.image_type_runs("cpu", turing) and launcher.image_type_runs("cpu", []) + assert not launcher.image_type_runs("rocm7.2", turing) and not launcher.image_type_runs("gpu", turing) + + +def test_decide_image_older_types(tmp_path): + # images of older types that run on this GPU count too, as on a machine with earlier releases + turing = [gpu((7, 5), driver=580)] + installed = {"cu128-v2.5.4": "sha256:1", "cu126-v2.5.4": "sha256:2", "cpu-v2.5.4": "sha256:3"} + hub = {**HUB, "cu128-v2.5.4": {"sha256:1"}, "cu126-v2.5.4": {"sha256:2"}, "cpu-v2.5.4": {"sha256:3"}} + with pytest.raises(launcher.LauncherError, match="cu126-v2.5.4.*cu128-v2.5.4.*--update"): + decide(tmp_path, docker=installed, hub=hub, gpus=turing) + # one of them alone is used, with a note about the newest release + plan = decide(tmp_path, docker={"cu128-v2.5.4": "sha256:1"}, hub=hub, gpus=turing) + assert plan.tag == "cu128-v2.5.4" and any("2.6.0 is available" in note for _, note in plan.notes) + # a run on the CPU only counts the CPU image + assert decide(tmp_path, docker=installed, hub=hub, gpus=[]).tag == "cpu-v2.5.4" + + def test_decide_image_several_installed(tmp_path): with pytest.raises(launcher.LauncherError, match="Several"): decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}) diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 45b69bd5c..c8987c819 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -59,7 +59,8 @@ # each supports and the newest the legacy one does. The test in test/lint keeps them in sync. NEWEST_BUILD = ((13, 2), "cu132") LEGACY_BUILD = ((12, 6), "cu126") -MIN_DRIVER = {12: 525, 13: 580} +# CUDA 11 for older releases, whose images can still be installed +MIN_DRIVER = {11: 450, 12: 525, 13: 580} MIN_CAPABILITY = {"cu132": (7, 5), "cu126": (5, 0)} LEGACY_MAX_CAPABILITY = (9, 0) # the graphics memory FastSurfer needs at 0.8 mm, see the system requirements in the README @@ -202,6 +203,24 @@ def _best_build(gpus: list[Gpu]) -> tuple[tuple[int, int], str] | None: return next((build for build in (NEWEST_BUILD, LEGACY_BUILD) if _runs(build, gpus)), None) +def image_type_runs(image_type: str, gpus: list[Gpu]) -> bool: + """ + Whether images of a type (cu132, cu118, cpu, ...) run on these GPUs, or, without GPUs, on the CPU. + + The GPUs of the current CUDA types are known; for older ones, only the driver can be checked. + """ + if not gpus: + return image_type == "cpu" + match = re.fullmatch(r"cu(\d+)(\d)", image_type) + if match is None: + return False + build = next((b for b in (NEWEST_BUILD, LEGACY_BUILD) if b[1] == image_type), None) + if build is not None: + return _runs(build, gpus) + major = int(match.group(1)) + return major in MIN_DRIVER and min(gpu.driver for gpu in gpus) >= MIN_DRIVER[major] + + def describe(gpu: Gpu) -> str: """E.g. "GPU 1 (compute capability 8.6, 24 GB)".""" details = [] @@ -508,21 +527,22 @@ def _apptainer_runtime(runtime: str, installed: set[str]) -> str | None: def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: set[str], docker_images: dict[str, str] | None, sif_images: dict[str, str], hub: dict[str, set[str]] | None, image_dir: Path, update: bool, - local_id: str | None = None) -> ImagePlan: + local_id: str | None = None, gpus: list[Gpu] | None = None) -> ImagePlan: """ Which image to run, with which runtime, and whether it has to be downloaded or built first. - Without --image, the official release image of this machine's type (`prefix`) already installed is - used, wherever it is; several stop the launcher, none downloads the newest release. With --update, - the newest release (or the one named) is downloaded again. A release image whose content differs - from Docker Hub stops the launcher, unless it was named in full. + Without --image, an official release image already installed that runs on this machine is used, + wherever it is, also one of an older type than the newest images; several stop the launcher, none + downloads the newest release. With --update, the newest release (or the one named) is downloaded + again. A release image whose content differs from Docker Hub stops the launcher, unless it was + named in full. Parameters ---------- kind, value : str What --image names, see parse_image_spec. prefix : str - The image type for this machine, e.g. cu132. + The image type of the newest images for this machine, e.g. cu132. runtime : str --runtime: auto, apptainer, singularity or docker. installed : set[str] @@ -539,6 +559,9 @@ def decide_image(kind: str, value: str, prefix: str, runtime: str, installed: se --update: download or build the image again. local_id : str, None For a "docker" image, its ID in the local Docker, None if it is not there. + gpus : list[Gpu], None + The GPUs the run will use, to tell which installed images run here; empty for a run on the CPU. + None counts only images of type `prefix`. """ apptainer = _apptainer_runtime(runtime, installed) docker_images = docker_images or {} @@ -606,10 +629,13 @@ def release_exists(tag: str) -> None: release_exists(tag) return place_release(tag, checked=True) - # no --image: the installed release image of this type, or the newest release + # no --image: the installed release image that runs here, or the newest release + def runs_here(image_type: str) -> bool: + return image_type == prefix if gpus is None else image_type_runs(image_type, gpus) + candidates = sorted({t for t in [*docker_images, *sif_images] - if (m := RELEASE_TAG.fullmatch(t)) and m.group(1) == prefix}, - key=lambda t: parse_version(t.rsplit("-v", 1)[-1])) + if (m := RELEASE_TAG.fullmatch(t)) and runs_here(m.group(1))}, + key=lambda t: (parse_version(t.rsplit("-v", 1)[-1]), t)) newest = newest_version(list(hub), prefix) if hub is not None else None if update or not candidates: if newest is None: @@ -637,8 +663,10 @@ def stores(t: str) -> str: return " and ".join(s for s, has in (("Docker", t in docker_images), ("Apptainer", t in sif_images)) if has) where = [f"{t} ({stores(t)})" for t in candidates] - raise LauncherError(f"Several FastSurfer images are installed: {'; '.join(where)}. Choose one with " - f"--image , e.g. --image {candidates[-1]}, or remove the others.") + update_hint = (f", or download the newest release ({prefix}-v{newest}) with --update" if newest is not None + else "") + raise LauncherError(f"Several FastSurfer images that run here are installed: {'; '.join(where)}. Choose " + f"one with --image , e.g. --image {candidates[-1]}{update_hint}.") plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a study, " f"keep this image with --image {plan.tag}.")) return plan @@ -1039,9 +1067,12 @@ def info(message: str, level: str = "INFO") -> None: f"launcher does not know the images of the newest release. Update it with " f"--update_launcher, or download it again from {LAUNCHER_URL}.") local_id = local_docker_id(value) if kind == "docker" else None + # the GPUs this run uses, to tell which installed images run here; none for a run on the CPU + used = [g for g in gpus if build.devices is None or g.index in build.devices] if build.gpu else [] plan = decide_image(kind, value, build.prefix, options.runtime, installed, local_docker_images() if "docker" in installed else None, - local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id) + local_sif_images(options.image_dir), hub, options.image_dir, options.update, local_id, + used) runtime = plan.runtime if hub is not None: stable = stable_launcher() From e6d3cab22d1142a353e9368408f298c151908a9b Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:16:23 +0200 Subject: [PATCH 12/18] Pass Ctrl-C through to FastSurfer in Docker and build Apptainer images from the local Docker --- test/utils/test_fastsurfer_container.py | 15 +++++++++++---- tools/fastsurfer_container.py | 19 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 9f7fd88b0..9f9cc0009 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -227,6 +227,11 @@ def test_decide_image_one_installed(tmp_path): # the same version in both places counts once, and runs with Apptainer plan = decide(tmp_path, docker={"cu132-v2.6.0": "sha256:a"}, sif={"cu132-v2.6.0": "sha256:a"}) assert plan.runtime == "apptainer" and plan.build_from is None + # a release pulled into Docker is converted from there for Apptainer, not downloaded again + plan = decide(tmp_path, runtime="apptainer", docker={"cu132-v2.6.0": "sha256:a"}) + assert plan.build_from == "docker-daemon://deepmi/fastsurfer:cu132-v2.6.0" + # but not a local build tagged like the release + assert decide(tmp_path, runtime="apptainer", docker={"cu132-v2.6.0": ""}).build_from.startswith("docker://") # images of another type do not count assert decide(tmp_path, docker={"cpu-v2.5.4": "sha256:x"}).tag == "cu132-v2.6.0" @@ -402,17 +407,19 @@ def test_build_image(tmp_path): def test_commands(tmp_path): args = ["--t1", str(tmp_path / "t1.mgz"), "--sd", str(tmp_path)] docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") - assert docker == ["docker", "run", "--rm", "--user", "1000:1000", "--gpus", "all", - "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + # --init, so Ctrl-C and a cancelled job reach FastSurfer + init = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1"] + assert docker == init + ["--user", "1000:1000", "--gpus", "all", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", + "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args # selected GPUs, quoted so Docker does not split the list selected = launcher.docker_command("img:tag", True, [], "run_fastsurfer.sh", [], [], "1:1", devices=[1, 3]) - assert selected[5:7] == ["--gpus", '"device=1,3"'] + assert selected[8:10] == ["--gpus", '"device=1,3"'] sif_gpu = launcher.apptainer_command("apptainer", tmp_path / "fs.sif", True, [], "run_fastsurfer.sh", [], [], devices=[2]) assert sif_gpu[2:8] == ["--nv", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", "--env", "CUDA_VISIBLE_DEVICES=2", "--no-mount"] emulated = launcher.docker_command("img:tag", False, [tmp_path], "run_fastsurfer.sh", args, [], "501:20", True) - assert emulated[:7] == ["docker", "run", "--rm", "--user", "501:20", "--platform", "linux/amd64"] + assert emulated[:10] == init + ["--user", "501:20", "--platform", "linux/amd64"] brun = launcher.docker_command("img:tag", False, [tmp_path], "brun_fastsurfer.sh", args, [], "1000:1000") assert brun[-len(args) - 4:-len(args)] == ["--entrypoint", "/fastsurfer/tools/Docker/entrypoint.sh", "img:tag", "/fastsurfer/brun_fastsurfer.sh"] diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index c8987c819..bd5e74a51 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -593,9 +593,13 @@ def place_release(tag: str, checked: bool, forced: bool = update) -> ImagePlan: return ImagePlan("docker", f"{DOCKER_REPO}:{tag}", pull=forced, tag=tag) sif = sif_path(image_dir, tag) build = forced or not in_sif - record = sorted(known)[0] if known and build else None - return ImagePlan(chosen, str(sif), build_from=f"docker://{DOCKER_REPO}:{tag}" if build else None, - record=record, tag=tag) + # the release pulled into the local Docker: build from there, without a second download (and + # without Docker Hub credentials, which Apptainer may send even for this public image) + pulled = docker_images.get(tag, "") + from_docker = "docker" in installed and pulled and (known is None or pulled in known) + source = f"docker-daemon://{DOCKER_REPO}:{tag}" if from_docker else f"docker://{DOCKER_REPO}:{tag}" + record = (sorted(known)[0] if known else pulled or None) if build else None + return ImagePlan(chosen, str(sif), build_from=source if build else None, record=record, tag=tag) def release_exists(tag: str) -> None: if hub is not None and tag not in hub and tag not in docker_images and tag not in sif_images: @@ -841,7 +845,9 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: `emulate` runs the Intel image on Apple silicon. """ - cmd = ["docker", "run", "--rm", "--user", user] + # --init: the image's entrypoint is a shell, which as the first process ignores Ctrl-C and the + # signal a cancelled cluster job sends; tini passes them to all of FastSurfer's processes instead + cmd = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1", "--user", user] if gpu: # the quotes are part of the value: without them, Docker splits a list of devices at the comma selection = "all" if devices is None else f'"device={",".join(map(str, devices))}"' @@ -1137,7 +1143,10 @@ def info(message: str, level: str = "INFO") -> None: if command[1] == "build": info(f"Building the image once: {shlex.join(command)}") if not build_image(command): - print("ERROR: Building the image failed.", file=sys.stderr) + print("ERROR: Building the image failed. If it reports 'unauthorized' for this public image, " + "Apptainer sent stored Docker Hub credentials that are wrong or expired: check " + "~/.docker/config.json, `apptainer remote list` and $APPTAINER_DOCKER_USERNAME.", + file=sys.stderr) return 1 record = Path(command[2]).with_name(Path(command[2]).name + ".digest") if plan.record: From 1f24b3357f816c33d9f36f4a30f1c6a0c0cc6fc9 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:29:24 +0200 Subject: [PATCH 13/18] Show local time in Docker logs and suggest the installed newest release in the launcher --- test/utils/test_fastsurfer_container.py | 17 +++++++++++++---- tools/fastsurfer_container.py | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 9f9cc0009..dc8ae9ebc 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -15,6 +15,7 @@ """Test the image choice and the path handling of tools/fastsurfer_container.py.""" import importlib.util +import platform import sys from pathlib import Path @@ -253,7 +254,8 @@ def test_decide_image_older_types(tmp_path): turing = [gpu((7, 5), driver=580)] installed = {"cu128-v2.5.4": "sha256:1", "cu126-v2.5.4": "sha256:2", "cpu-v2.5.4": "sha256:3"} hub = {**HUB, "cu128-v2.5.4": {"sha256:1"}, "cu126-v2.5.4": {"sha256:2"}, "cpu-v2.5.4": {"sha256:3"}} - with pytest.raises(launcher.LauncherError, match="cu126-v2.5.4.*cu128-v2.5.4.*--update"): + # listed newest first; the newest release is not installed, so --update is offered + with pytest.raises(launcher.LauncherError, match="cu128-v2.5.4.*cu126-v2.5.4.*--update"): decide(tmp_path, docker=installed, hub=hub, gpus=turing) # one of them alone is used, with a note about the newest release plan = decide(tmp_path, docker={"cu128-v2.5.4": "sha256:1"}, hub=hub, gpus=turing) @@ -263,8 +265,12 @@ def test_decide_image_older_types(tmp_path): def test_decide_image_several_installed(tmp_path): - with pytest.raises(launcher.LauncherError, match="Several"): + # the newest release is among them: suggested first, and no --update, which would download it again + with pytest.raises(launcher.LauncherError, match="Several") as stop: decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}) + message = str(stop.value) + assert message.index("cu132-v2.6.0") < message.index("cu132-v2.5.4") + assert "--image cu132-v2.6.0, the newest release" in message and "--update" not in message # --update takes the newest, and lists the older ones with how to remove them plan = decide(tmp_path, docker={"cu132-v2.5.4": "sha256:b"}, sif={"cu132-v2.6.0": ""}, update=True) assert plan.build_from is not None and plan.tag == "cu132-v2.6.0" @@ -409,8 +415,11 @@ def test_commands(tmp_path): docker = launcher.docker_command("img:tag", True, [tmp_path], "run_fastsurfer.sh", args, [], "1000:1000") # --init, so Ctrl-C and a cancelled job reach FastSurfer init = ["docker", "run", "--rm", "--init", "--env", "TINI_KILL_PROCESS_GROUP=1"] - assert docker == init + ["--user", "1000:1000", "--gpus", "all", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID", - "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args + # the host's time zone, on Linux + zone = (["-v", "/etc/localtime:/etc/localtime:ro"] + if platform.system() == "Linux" and Path("/etc/localtime").exists() else []) + assert docker == init + ["--user", "1000:1000", "--gpus", "all", "--env", "CUDA_DEVICE_ORDER=PCI_BUS_ID"] + zone + [ + "-v", f"{tmp_path}:{tmp_path}", "img:tag"] + args # selected GPUs, quoted so Docker does not split the list selected = launcher.docker_command("img:tag", True, [], "run_fastsurfer.sh", [], [], "1:1", devices=[1, 3]) assert selected[8:10] == ["--gpus", '"device=1,3"'] diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index bd5e74a51..d6a9978d6 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -666,11 +666,16 @@ def runs_here(image_type: str) -> bool: def stores(t: str) -> str: return " and ".join(s for s, has in (("Docker", t in docker_images), ("Apptainer", t in sif_images)) if has) - where = [f"{t} ({stores(t)})" for t in candidates] - update_hint = (f", or download the newest release ({prefix}-v{newest}) with --update" if newest is not None - else "") + where = [f"{t} ({stores(t)})" for t in reversed(candidates)] # the newest first + newest_tag = f"{prefix}-v{newest}" if newest is not None else None + if newest_tag in candidates: + hint = f"e.g. --image {newest_tag}, the newest release" + elif newest_tag is not None: + hint = f"e.g. --image {candidates[-1]}, or download the newest release ({newest_tag}) with --update" + else: + hint = f"e.g. --image {candidates[-1]}" raise LauncherError(f"Several FastSurfer images that run here are installed: {'; '.join(where)}. Choose " - f"one with --image , e.g. --image {candidates[-1]}{update_hint}.") + f"one with --image , {hint}.") plan.notes.insert(0, ("INFO", f"Running FastSurfer {plan.tag.rsplit('-v', 1)[-1]} ({plan.tag}). For a study, " f"keep this image with --image {plan.tag}.")) return plan @@ -854,6 +859,9 @@ def docker_command(image: str, gpu: bool, mounts: list[Path], script: str, args: cmd += ["--gpus", selection, "--env", CUDA_ORDER] if emulate: cmd += ["--platform", "linux/amd64"] + if platform.system() == "Linux" and Path("/etc/localtime").exists(): + # the host's time zone, so the logs show local times as with Apptainer, which shares this file itself + cmd += ["-v", "/etc/localtime:/etc/localtime:ro"] for mount in mounts: cmd += ["-v", f"{mount}:{mount}"] cmd += extra From 3766183af52f7a98c3f5c79789bcc789c3fb5e51 Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:32:37 +0200 Subject: [PATCH 14/18] Select a supported GPU for release images named with --image in the launcher --- test/utils/test_fastsurfer_container.py | 19 +++++++++ tools/fastsurfer_container.py | 54 ++++++++++++++++++++----- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index dc8ae9ebc..494d5b4c1 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -150,6 +150,25 @@ def test_plan_build(): assert launcher.plan_build([], [gpu((3, 7))], brun=False).prefix == "cpu" +def test_plan_build_for_type(): + pascal, blackwell = gpu((6, 1), index=0), gpu((12, 0), index=1) + # a pinned release image: the best GPU among those its type supports, shown alone + build = launcher.plan_build_for_type([], [pascal, blackwell], False, "cu132") + assert (build.gpu, build.devices) == (True, [1]) and build.notes[0].startswith("Using GPU 1") + assert launcher.plan_build_for_type([], [pascal, blackwell], False, "cu126").devices == [0] + # an older type is judged by the driver; with --brun, every GPU the type supports + assert launcher.plan_build_for_type([], [pascal, blackwell], True, "cu118").devices == [0, 1] + # a named GPU is used as named, or stops the launcher if the type does not support it + assert launcher.plan_build_for_type(["--device", "cuda:1"], [pascal, blackwell], False, "cu132").devices is None + with pytest.raises(launcher.LauncherError, match="do not support GPU 0"): + launcher.plan_build_for_type(["--device", "cuda:0"], [pascal, blackwell], False, "cu132") + # no supported GPU: the CPU, or a stop when a GPU was requested + assert launcher.plan_build_for_type([], [pascal], False, "cu132").gpu is False + with pytest.raises(launcher.LauncherError, match="support none"): + launcher.plan_build_for_type(["--device", "cuda"], [pascal], False, "cu132") + assert launcher.plan_build_for_type([], [pascal, blackwell], False, "cpu").gpu is False + + def test_newest_version(): tags = ["cu132-v2.6.0", "cu132-v2.10.1", "cu132-v2.9.3", "cu126-v2.11.0", "cu132-v2.11.0-dev0", "latest", "cpu-latest", "cu132-v2.10.1_abc123"] diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index d6a9978d6..ca58ee86a 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -288,6 +288,14 @@ def choose_build(gpus: list[Gpu], use: str) -> Build: return Build("cpu", False, [f"{reason}, so FastSurfer runs on the CPU."]) +def _gpu_rank(gpu: Gpu) -> tuple: + """Sorts the better GPU for FastSurfer last: enough memory first, then the newest, else the most memory.""" + memory = gpu.memory_gib or 0.0 + if gpu.memory_gib is None or memory >= MIN_GPU_MEMORY_GIB: + return True, gpu.capability or (0, 0), memory + return False, (memory,), gpu.capability or (0, 0) + + def select_gpus(gpus: list[Gpu], brun: bool) -> Build | None: """ The GPU to run on when none was named: one an image supports, with enough memory, the newest. @@ -298,14 +306,7 @@ def select_gpus(gpus: list[Gpu], brun: bool) -> Build | None: supported = [gpu for gpu in gpus if _best_build([gpu]) is not None] if not supported: return None - - def rank(gpu: Gpu) -> tuple: - memory = gpu.memory_gib or 0.0 - if gpu.memory_gib is None or memory >= MIN_GPU_MEMORY_GIB: - return True, gpu.capability or (0, 0), memory - return False, (memory,), gpu.capability or (0, 0) - - best = max(supported, key=rank) + best = max(supported, key=_gpu_rank) build_tuple = _best_build([best]) chosen = [gpu for gpu in supported if _runs(build_tuple, [gpu])] if brun else [best] build = choose_build([best], "auto") @@ -384,6 +385,40 @@ def plan_build(args: list[str], gpus: list[Gpu], brun: bool) -> Build: return select_gpus(gpus, brun) or choose_build(gpus, use) +def plan_build_for_type(args: list[str], gpus: list[Gpu], brun: bool, image_type: str) -> Build: + """ + The GPUs for a release image named with --image, e.g. cu132-v2.6.0. + + As without --image, the best GPU is selected when none is named, but only among those this image + type supports; named GPUs are used as named, and stop the launcher if the type does not support them. + """ + use, named = gpu_request(args, brun) + if use == "cpu" or image_type == "cpu": + return Build(image_type, False) + if named is not None: + by_index = {gpu.index: gpu for gpu in gpus} + missing = [i for i in named if i not in by_index] + if missing: + raise LauncherError(f"There is no GPU {missing[0]}, nvidia-smi found {len(gpus)}.") + unsupported = [by_index[i] for i in named if not image_type_runs(image_type, [by_index[i]])] + if unsupported: + raise LauncherError(f"The {image_type} images do not support {', '.join(map(describe, unsupported))}.") + # the container sees all GPUs, so the named indices keep their meaning + return Build(image_type, True) + supported = [gpu for gpu in gpus if image_type_runs(image_type, [gpu])] + if not supported: + if not gpus: + reason = "No NVIDIA GPU found" + else: + reason = f"The {image_type} images support none of the GPUs here ({', '.join(map(describe, gpus))})" + if use == "cuda": + raise LauncherError(f"{reason}.") + return Build(image_type, False, [f"{reason}, so FastSurfer runs on the CPU."]) + chosen = supported if brun else [max(supported, key=_gpu_rank)] + return Build(image_type, True, ["Using " + ", ".join(map(describe, chosen)) + "."], + devices=[gpu.index for gpu in chosen]) + + def image_reference(prefix: str, version: str) -> str: """The Docker image to run, e.g. deepmi/fastsurfer:cu132-v2.6.0.""" return f"{DOCKER_REPO}:{prefix}-v{version}" @@ -1067,8 +1102,7 @@ def info(message: str, level: str = "INFO") -> None: if kind in ("auto", "version"): build = plan_build(args, gpus, options.brun) elif kind in ("tag", "release"): - prefix = RELEASE_TAG.fullmatch(value).group(1) - build = Build(prefix, prefix != "cpu" and wants_gpu) + build = plan_build_for_type(args, gpus, options.brun, RELEASE_TAG.fullmatch(value).group(1)) else: build = Build("custom", wants_gpu) From e85b4c65ef0f561696855f5be7d596637e8df5ed Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:39:01 +0200 Subject: [PATCH 15/18] Check that the launcher's GPU table covers the current CUDA builds --- test/lint/test_launcher_builds.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/lint/test_launcher_builds.py b/test/lint/test_launcher_builds.py index 64c3912d5..df44de20c 100644 --- a/test/lint/test_launcher_builds.py +++ b/test/lint/test_launcher_builds.py @@ -48,6 +48,11 @@ def test_launcher_builds_match_gpu_support(): assert launcher.MIN_DRIVER[major] == gpu_support.MIN_DRIVER[major] +def test_launcher_capabilities_cover_builds(): + # the GPU range of each build comes from its PyTorch, so a new build needs its entry checked by hand + assert set(launcher.MIN_CAPABILITY) == {launcher.NEWEST_BUILD[1], launcher.LEGACY_BUILD[1]} + + def test_launcher_default_build_matches_pyproject(): with open(FASTSURFER_HOME / "pyproject.toml", "rb") as fp: cuda = tomllib.load(fp)["tool"]["cuda"]["version"] From 065ca380e07deb6f3436df81b183d629642bc7be Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:45:54 +0200 Subject: [PATCH 16/18] Name expired browser logins in the launcher's hint for failed Apptainer builds --- tools/fastsurfer_container.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index ca58ee86a..004506c6d 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -1186,8 +1186,10 @@ def info(message: str, level: str = "INFO") -> None: info(f"Building the image once: {shlex.join(command)}") if not build_image(command): print("ERROR: Building the image failed. If it reports 'unauthorized' for this public image, " - "Apptainer sent stored Docker Hub credentials that are wrong or expired: check " - "~/.docker/config.json, `apptainer remote list` and $APPTAINER_DOCKER_USERNAME.", + "Apptainer sent stored Docker Hub credentials that are wrong or expired, often the " + "short-lived token of a browser login in ~/.docker/config.json: log in again with " + "`docker login -u ` and an access token as the password, or check " + "`apptainer registry list` and $APPTAINER_DOCKER_USERNAME.", file=sys.stderr) return 1 record = Path(command[2]).with_name(Path(command[2]).name + ".digest") From a025cbf28008c30b4b7e2910d4133ed261a1a0cd Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 14:49:12 +0200 Subject: [PATCH 17/18] Build official images in Apptainer without stored Docker Hub logins --- test/utils/test_fastsurfer_container.py | 7 ++++++ tools/fastsurfer_container.py | 29 ++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index 494d5b4c1..b8feeaef5 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -427,6 +427,13 @@ def test_build_image(tmp_path): failed = tmp_path / "images" / "fastsurfer-cu132-v2.6.0.sif" assert not launcher.build_image([sys.executable, str(builder), str(failed), "fail"]) assert sorted(p.name for p in sif.parent.iterdir()) == [sif.name] + # anonymous: the build reads an empty login file, which is removed afterwards + reader = tmp_path / "reader.py" + reader.write_text("import sys\nassert sys.argv[1] == '--authfile'\n" + "open(sys.argv[3], 'w').write(open(sys.argv[2]).read())\n") + assert launcher.build_image([sys.executable, str(reader), str(failed), "docker://image"], anonymous=True) + assert failed.read_text() == "{}\n" + assert sorted(p.name for p in sif.parent.iterdir()) == sorted([sif.name, failed.name]) def test_commands(tmp_path): diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 004506c6d..948c9cc9e 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -953,24 +953,41 @@ def _license_args(args: list[str], cwd: Path) -> list[str]: return [] -def build_image(command: list[str]) -> bool: +def supports_authfile(runtime: str) -> bool: + """Whether ` build` takes --authfile, which older versions do not.""" + try: + return "--authfile" in subprocess.run([runtime, "build", "--help"], capture_output=True, text=True, + timeout=30).stdout + except (OSError, subprocess.SubprocessError): + return False + + +def build_image(command: list[str], anonymous: bool = False) -> bool: """ Run ` build ` into a temporary file, and move it in place when it is complete. Parallel runs, e.g. the jobs of a SLURM array, and an interrupted build then never leave a partial image at the path the next run uses. Of two parallel builds, the last one to finish replaces the other. + With `anonymous`, the build gets an empty login file (--authfile), so stored Docker Hub credentials, + e.g. the expired token of a browser login, cannot make the download of a public image fail. """ sif = Path(command[2]) sif.parent.mkdir(parents=True, exist_ok=True) partial = sif.with_name(f".{sif.stem}.{os.getpid()}.partial.sif") + no_login = sif.with_name(f".{sif.stem}.{os.getpid()}.no-login.json") + login = [] + if anonymous: + no_login.write_text("{}\n") + login = ["--authfile", str(no_login)] try: - if subprocess.run(command[:2] + [str(partial)] + command[3:]).returncode != 0: + if subprocess.run(command[:2] + login + [str(partial)] + command[3:]).returncode != 0: return False os.replace(partial, sif) return True finally: - if partial.exists(): - partial.unlink() + for temporary in (partial, no_login): + if temporary.exists(): + temporary.unlink() def stable_launcher() -> bytes | None: @@ -1184,7 +1201,9 @@ def info(message: str, level: str = "INFO") -> None: for command in commands[:-1]: if command[1] == "build": info(f"Building the image once: {shlex.join(command)}") - if not build_image(command): + # the official images are public, so their download needs no login + anonymous = command[3].startswith(f"docker://{DOCKER_REPO}:") and supports_authfile(command[0]) + if not build_image(command, anonymous): print("ERROR: Building the image failed. If it reports 'unauthorized' for this public image, " "Apptainer sent stored Docker Hub credentials that are wrong or expired, often the " "short-lived token of a browser login in ~/.docker/config.json: log in again with " From 03497f170914da0d8db7476a5f0661e01b84952e Mon Sep 17 00:00:00 2001 From: Martin Reuter Date: Tue, 13 Oct 2026 15:06:09 +0200 Subject: [PATCH 18/18] Say there are no Docker images instead of no release in the launcher --- test/utils/test_fastsurfer_container.py | 2 +- tools/fastsurfer_container.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/utils/test_fastsurfer_container.py b/test/utils/test_fastsurfer_container.py index b8feeaef5..07539e205 100644 --- a/test/utils/test_fastsurfer_container.py +++ b/test/utils/test_fastsurfer_container.py @@ -316,7 +316,7 @@ def test_decide_image_named(tmp_path): # a version the type of this machine was not released in: the images of that version are listed with pytest.raises(launcher.LauncherError, match="cpu-v2.6.0"): decide(tmp_path, kind="version", value="2.6.0", hub={"cpu-v2.6.0": {"sha256:c"}}) - with pytest.raises(launcher.LauncherError, match="not released"): + with pytest.raises(launcher.LauncherError, match="no Docker images of FastSurfer 9.9.9"): decide(tmp_path, kind="tag", value="cu132-v9.9.9") with pytest.raises(launcher.LauncherError, match="Docker"): decide(tmp_path, kind="sif", value="x.sif", runtime="docker") diff --git a/tools/fastsurfer_container.py b/tools/fastsurfer_container.py index 948c9cc9e..746f428cd 100644 --- a/tools/fastsurfer_container.py +++ b/tools/fastsurfer_container.py @@ -642,7 +642,7 @@ def release_exists(tag: str) -> None: images = sorted(t for t in hub if t.endswith(f"-v{version}")) raise LauncherError(f"There is no image {DOCKER_REPO}:{tag}. " + ( f"The images of FastSurfer {version} are {', '.join(images)}." if images - else f"FastSurfer {version} is not released.")) + else f"There are no Docker images of FastSurfer {version}.")) if kind == "sif": if runtime == "docker":