brun_fastsurfer.sh

#!/bin/bash

# Copyright 2023 DeepMI Lab, German Center for Neurodegenerative Diseases (DZNE), Bonn
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# script for batch-running FastSurfer

subjects=()
subjects_stdin="true"
POSITIONAL_FASTSURFER=()
task_count=""
task_id=""
surf_only="false"
seg_only="false"
debug="false"
run_fastsurfer=()
parallel_pipelines="1"
num_parallel_surf="1"
num_parallel_seg="1"
statusfile=""
python="python3 -s"

function usage()
{
  cat << EOF
Script to run FastSurfer on multiple subjects in parallel/series.

Usage:
brun_fastsurfer.sh --subjects_list <subjects_list_path> [other options]
OR
brun_fastsurfer.sh --subjects <subject_id>=<t1_path> [<subject_id>=<t1_path>
    [...]] [other options]
OR
brun_fastsurfer.sh [other options]

Other options:
brun_fastsurfer.sh [...] [--batch "<i>/<n>"] [--parallel <n>|max]
    [--parallel_seg <n>|max] [--parallel_surf <n>|max]
    [--run_fastsurfer <command/run_fastsurfer_script>]
    [--statusfile <filename>] [--debug] [--help]
    [<fastsurfer_flags>]

License:  Apache License, Version 2.0

Documentation of Options:
Generally, brun_fastsurfer works similar to run_fastsurfer, but loops over
multiple subjects from
i. a list passed through stdin of the format (one subject per line)
---
<subject_id>=<t1_path>[ <subject_specific_parameters>[ ...]]
...
---
ii. a subjects list file using the same format (use Ctrl-D to end the input), or
iii. a list of subjects directly passed (this does not support subject-specific
  parameters)

A path or parameter that contains a space has to be quoted or escaped as it
would be in the shell, i.e. '/data/my subject/t1.mgz', "/data/my subject/t1.mgz"
or /data/my\ subject/t1.mgz. Single quotes are the simplest, because everything
inside them is taken literally, including backslashes. No expansion is performed
in either kind of quotes, so a \$ or a \` is just that character.

--batch "<i>/<n>": run the i-th of n batches (starting at 1) of the full list of
  subjects (default: 1/1, == run all). "slurm_task_id" is a valid option for
  "<i>".
  Note, brun_fastsurfer.sh will also automatically detect being run in a SLURM
  JOBARRAY and split according to \$SLURM_ARRAY_TASK_ID and
  \$SLURM_ARRAY_TASK_COUNT (unless values are specifically assigned with the
  --batch argument).
--parallel <n>|max: parallel execution of run_fastsurfer for <n> images. Creates
  <n> processes with each process performing segmentation and surface
  reconstruction. The default is this serial execution mode with n=1:
  '--parallel 1'.
--parallel_seg <n>|max and
--parallel_surf <m>|max: activate independent segmentation and surface
  reconstruction pipelines. Segmentation and Surface reconstruction have
  independent processing queues. After successful segmentation (<n> parallel
  processes), cases are transferred into the surface queue (<m> parallel
  processes). Together max. m+n processes will run. Logfiles unchanged, console
  output for individual subjects is interleaved with subject_id prepended.
--run_fastsurfer <command>: This option enables the startup of fastsurfer in a
  more controlled manner, for example to delegate the fastsurfer run to
  container:
  --run_fastsurfer "singularity exec --nv --no-mount home,cwd -e -B <host_dir>:/data /fastsurfer/run_fastsurfer.sh"
  Note, paths to files and --sd have to be defined in the container file system
  in this case.
--statusfile <filename>: a file to document which subject ran successfully. Also
  used to skip surface recon, if the previous segmentation failed.
--threads <n>,
--threads_seg <n>, and
--threads_surf <n>: specify number of threads for each parallel "process", i.e.
  total_threads=num_seg_processes * num_seg_threads + num_surf_processes *
  num_surf_threads.
--debug: Additional debug output.
--help: print this help.

With the exception of --t1 and --sid, all run_fastsurfer.sh options are
supported, see 'run_fastsurfer.sh --help'.

This tool requires functions in stools.sh (expected in same folder as this
script).
EOF
}

if [ -z "${BASH_SOURCE[0]}" ]; then THIS_SCRIPT="$0"
else THIS_SCRIPT="${BASH_SOURCE[0]}"
fi

# PRINT USAGE if called without params
if [[ $# -eq 0 ]]
then
  usage
  exit
fi

function verify_parallel()
{
  # 1: flag, 2: value
  value="$(echo "$2" | tr '[:upper:]' '[:lower:]')"
  if [[ "$value" =~ ^(max|-[0-9]+|0)$ ]] ; then verify_value="max"
  elif [[ "$value" =~ ^[0-9]+$ ]] ; then verify_value="$value"
  else echo "ERROR: Invalid value for $1: '$2', must be integer or 'max'." ; exit 1
  fi
  export verify_value
}

function warn_old()
{
  echo "WARNING: The syntax to ${key-'--parallel_subjects'} seg=/surf= is outdated and will be removed in FastSurfer 3,"
  echo "  use --parallel <n>, --parallel_seg <n>, or --parallel_surf <n>!"
}

function get_running_jobs()
{
  # the pids of the still-running background jobs, in $running_jobs.
  # `jobs -pr` runs in a subshell here but still reports this shell's jobs, verified on bash 3.2 and
  # 5.3. read rather than mapfile, which is bash 4+ while macOS ships bash 3.2 as /bin/bash.
  running_jobs=()
  local pid
  while IFS= read -r pid ; do running_jobs+=("$pid") ; done < <(jobs -pr)
}

# PARSE Command line
inputargs=("$@")
POSITIONAL=()
res_device="auto"
res_viewagg_device="auto"
# [[:space:]] rather than \s, and a backslash-escaped literal newline rather than \n: both are GNU
# extensions. BSD/macOS sed read \s as the letter s, so it stripped a trailing "s" from every
# subject line while leaving trailing whitespace and blank lines in place.
SED_CLEANUP_SUBJECTS='s/\r$//;s/[[:space:]]*\r[[:space:]]*/\
/g;s/[[:space:]]*$//;/^[[:space:]]*$/d'
prev_ifs="$IFS"
while [[ $# -gt 0 ]]
do
# make key lowercase
arg="$1"
key=$(echo "$arg" | tr '[:upper:]' '[:lower:]')
shift # past argument

case $key in
  # parse/get the subjects to iterate over
  #===================================================
  --subject_list|--subjects_list)
    if [[ ! -f "$1" ]]
    then
      echo "ERROR: Could not find the subjects list $1!"
      exit 1
    fi
    # append the subjects in the listfile (cleanup first) to the subjects array.
    # read rather than mapfile, which is bash 4+ while macOS ships bash 3.2; appending mirrors what
    # mapfile's -O ${#subjects[@]} did. The `|| [[ -n ... ]]` keeps the last line of a file that has
    # no final newline, which read alone would report as a failure and drop, unlike mapfile.
    while IFS= read -r subject_line || [[ -n "$subject_line" ]]
    do subjects+=("$subject_line") ; done < <(sed "$SED_CLEANUP_SUBJECTS" "$1")
    subjects_stdin="false"
    shift # past value
    ;;
  --subjects) subjects_stdin="false" ; while [[ ! "$1" =~ ^-- ]] ; do subjects+=("$1") ; shift ; done ;;
  # brun_fastsurfer-specific/custom options
  #===================================================
  --batch) task_count=$(echo "$1" | cut -f2 -d/) ;  task_id=$(echo "$1" | cut -f1 -d/) ; shift ;;
  --run_fastsurfer) IFS=" " ; run_fastsurfer=($1) ; shift ;;
  --parallel|--parallel_subjects)
    # future syntax: --parallel n|max|n/m
    # currently the former syntax --parallel_subjects seg=n|max, etc. is still supported but deprecated
    if [[ "$key" == "--parallel_subjects" ]] ; then
      echo "WARNING: The --parallel_subjects option is obsolete and replaced with --parallel <option>."
    fi
    if [[ "$#" -lt 1 ]] || [[ "$1" =~ ^-- ]]
    then
      if [[ "$key" == "--parallel_subjects" ]] ; then
        parallel_pipelines="1" ; num_parallel_surf="max" ; num_parallel_seg="max" ; warn_old
      else POSITIONAL_FASTSURFER+=("--parallel")
        echo "WARNING: --parallel without any argument for hemisphere parallelization is superseded by --threads 2+"
        echo "  and will be removed in FastSurfer 3."
      # this is just the parallel option
      fi
    else # has parameter
      value="$(echo "$1" | tr '[:upper:]' '[:lower:]')"
      if [[ "$value" =~ ^surf(=-[0-9]*|=max)?$ ]] ; then num_parallel_surf="max" ; parallel_pipelines="2" ; warn_old
      elif [[ "$value" =~ ^surf=[0-9]*$ ]] ; then  num_parallel_surf="${value:5}" ; parallel_pipelines="2" ; warn_old
      elif [[ "$value" =~ ^seg(=-[0-9]*|=max)?$ ]] ; then num_parallel_seg="max" ; parallel_pipelines="2" ; warn_old
      elif [[ "$value" =~ ^seg=[0-9]*$ ]] ; then num_parallel_seg="${value:4}" ; parallel_pipelines="2" ; warn_old
      elif [[ "$value" =~ ^(-[0-9]+|max)$ ]] ; then parallel_pipelines=1 ; num_parallel_seg="max" ; num_parallel_surf="max"
      elif [[ "$value" =~ ^[0-9]+$ ]] ; then parallel_pipelines=1 ; num_parallel_seg="$value" ; num_parallel_surf="$value"
      elif [[ "$value" =~ ^[0-9]+/[-9]+$ ]] ; then parallel_pipelines=2
        num_parallel_seg="$(echo "$value" | cut -d/ -f1)" ; num_parallel_surf="$(echo "$value" | cut -d/ -f2)"
      else echo "Invalid option for $key: $1" ; exit 1
      fi
      shift
    fi
    ;;
  --parallel_seg)
    parallel_pipelines=2 ; verify_parallel "$key" "$1" ; num_parallel_seg="$verify_value"; shift ;;
  --parallel_surf)
    parallel_pipelines=2 ; verify_parallel "$key" "$1" ; num_parallel_surf="$verify_value" ; shift ;;
  --statusfile) statusfile="$1" ; shift ;;
  --py) python="$1" ; POSITIONAL_FASTSURFER+=(--py "$1") ; shift ;; # this may be needed to get the device count
  --debug) debug="true" ;;
  --help) usage ; exit ;;
  # run_fastsurfer.sh options, with extra effect in brun_fastsurfer
  #===================================================
  --device) res_device="$1" ; shift ;;
  --viewagg_device) res_viewagg_device="$1" ; shift ;;
  --surf_only) surf_only="true" ;;
  --seg_only) seg_only="true" ;;
  --sid|--t1)
    echo "ERROR: --sid and --t1 are not valid for brun_fastsurfer.sh, these values are populated"
    echo "  via --subjects or --subjects_list."
    exit 1
    ;;
  *)    # unknown option/run_fastsurfer.sh option, make sure this is arg (to keep the case)
    POSITIONAL_FASTSURFER+=("$arg")
    ;;
esac
done
 # restore positional parameters
if [[ "${#POSITIONAL[@]}" -gt 0 ]] ; then set -- "${POSITIONAL[@]}" ; fi
IFS="$prev_ifs"

source "$(dirname "$THIS_SCRIPT")/stools.sh"

if [[ -n "$SLURM_ARRAY_TASK_ID" ]]
then
  if [[ -z "$task_count" ]] ; then task_count=$SLURM_ARRAY_TASK_COUNT ; fi
  if [[ -z "$task_id" ]] || [[ "$task_id" == "slurm_task_id" ]] ; then task_id=$SLURM_ARRAY_TASK_ID ; fi
  echo "SLURM TASK ARRAY detected"
fi

function get_device_list()
{
  # outputs device list numbers separated with "," to stdout
  # error messages are sent to stderr
  local list="" host="" out="" i prev_ifs="$IFS"
  if [[ "$1" =~ ^(cpu|mps|auto|cuda)$ ]] || [[ "$1" =~ ^cuda:[0-9]+(,[0-9]+)*$ ]] ; then list=",$1"
  elif [[ "$1" =~ ^cuda:[0-9]+([-,][0-9]+(-[0-9]+)?)* ]] ; then
    IFS="," ; host=${1:0:5}
    for i in ${1:5} ; do IFS="-" ; v=($i) ; list+=",$(seq -s"," "${v[0]}" "${v[1]}")" ; done
  else
    echo "ERROR: Invalid format for device|viewagg_device: $1 must be auto|cpu|mps|cuda[:X[,Y][-Z]...]"
    exit 1
  fi
  IFS=","
  for i in ${list:1} ; do
    if [[ "${out/$i/}" != "$out" ]] ; then echo "WARNING: Duplicate device $i in $1!" ; fi
    if [[ -n "$i" ]] ; then out+=",$i" ; fi
  done
  IFS="$prev_ifs"
  device_value="$host${out:1}"  # remove leading ","
  export device_value
}

# we do not know here, whether a gpu is available, so even if we have none or one, give a warning message.
if [[ "$num_parallel_seg" != 1 ]] && [[ "$surf_only" != "true" ]] && [[ "$res_device" =~ ^auto|cuda$ ]]
then
  # device is auto or cuda, auto-detect the device count and make it match with num_parallel_seg
  detected_devices=$($python -c "from torch.cuda import device_count as d ; print(*range(d()), sep=',')")
  _devices=($detected_devices)
  num_devices="${#_devices[@]}"
  echo "INFO: Auto-detecting CUDA-capable devices to parallelize segmentation, found $num_devices device(s)."
  if [[ "${#detected_devices[@]}" -le 1 ]] ; then echo "  => No changes!" # keep auto/cuda
  elif [[ "$num_parallel_seg" == "max" ]] || [[ "$num_parallel_seg" -gt "$num_devices" ]] ; then
    res_device="cuda:$detected_devices"
    num_parallel_seg=$num_devices
    echo "  => Setting number of parallel segmentations to number of devices (one segmentation per device)."
  else
    res_device="cuda:$(seq -s"," 0 $((num_parallel_seg - 1)))"
    echo "  => Limited by $num_parallel_seg parallel segmentations."
  fi
else
  get_device_list "$res_device" ; res_device="$device_value"
  get_device_list "$res_viewagg_device" ; res_viewagg_device="$device_value"
fi
if [[ "$subjects_stdin" == "true" ]]
then
  if [[ -t 0 ]] || [[ "$debug" == "true" ]]; then
    echo "Reading subjects from stdin, press Ctrl-D to end input (one subject per line)"
  fi
  # as for --subjects_list: keep a final line that the producer did not terminate with a newline
  while IFS= read -r subject_line || [[ -n "$subject_line" ]]
  do subjects+=("$subject_line") ; done < <(sed "$SED_CLEANUP_SUBJECTS")
fi

echo "$THIS_SCRIPT ${inputargs[*]}"
date -R
echo ""

if [[ "$debug" == "true" ]]
then
  echo "---START DEBUG---"
  echo "Debug parameters to script brun_fastsurfer:"
  echo ""
  echo "subjects: "
  printf "%s\n" "${subjects[@]}"
  echo "---"
  echo "task_id/task_count: ${task_id:-not specified}/${task_count:-not specified}"
  if [[ "$parallel_pipelines" == "1" ]]
  then
    if [[ "$num_parallel_seg" == 1 ]] ; then echo "parallelization: no subject parallelization"
    else echo "parallelization: $num_parallel_seg parallel subjects, segmentation/surface in series"
    fi
  else
    echo "parallelization: Independent segmentation and surface parallelization:"
    echo "  segmentation: $num_parallel_seg parallel subjects, surface: $num_parallel_surf parallel subjects"
  fi
  IFS=" "
  if [[ "${run_fastsurfer[*]}" == "" ]] ;  then echo "running default run_fastsurfer"
  else echo "running ${run_fastsurfer[*]}"
  fi
  if [[ -n "$statusfile" ]] ;  then echo "statusfile: $statusfile" ; fi
  printf "\nFastSurfer parameters:"
  if [[ "$seg_only" == "true" ]]; then printf "\n--seg_only" ; fi
  if [[ "$surf_only" == "true" ]]; then printf "\n--surf_only" ; fi
  if [[ -n "$res_device" ]] ; then printf "\n--device %s" "$res_device" ; fi
  if [[ -n "$res_viewagg_device" ]] ; then printf "\n--viewagg_device %s" "$res_viewagg_device" ; fi
  for p in "${POSITIONAL_FASTSURFER[@]}"
  do
    if [[ "$p" = --* ]]; then printf "\n%s" "$p" ;
    else printf " %s" "$p" ;
    fi
  done
  printf "\n\n"
  shell=$(ls -l "/proc/$$/exe" | cut -d">" -f2)
  printf "Running in shell %s: %s\n" "$shell" "$($shell --version 2>/dev/null | head -n 1)"
  echo "---END DEBUG  ---"
fi

if [[ "${#subjects[@]}" == 0 ]] ; then echo "ERROR: No subjects specified!" ; exit 1 ; fi

if [[ -z "${run_fastsurfer[*]}" ]]
then
  if [[ -n "$FASTSURFER_HOME" ]]
  then
    run_fastsurfer=("$FASTSURFER_HOME/run_fastsurfer.sh")
    echo "INFO: run_fastsurfer not explicitly specified, using \$FASTSURFER_HOME/run_fastsurfer.sh."
  elif [[ -f "$(dirname "$THIS_SCRIPT")/run_fastsurfer.sh" ]]
  then
    run_fastsurfer=("$(dirname "$THIS_SCRIPT")/run_fastsurfer.sh")
    echo "INFO: run_fastsurfer not explicitly specified, using ${run_fastsurfer[0]}."
  elif [[ -f "/fastsurfer/run_fastsurfer.sh" ]]
  then
    run_fastsurfer=("/fastsurfer/run_fastsurfer.sh")
    echo "INFO: run_fastsurfer not explicitly specified, using /fastsurfer/run_fastsurfer.sh."
  else
    echo "ERROR: Could not find FastSurfer, please set the \$FASTSURFER_HOME environment variable."
  fi
fi

num_subjects=${#subjects[@]}
if [[ -z "$task_id" ]] && [[ -z "$task_count" ]]
then
  subject_start=0
  subject_end=$num_subjects
elif [[ -z "$task_id" ]] || [[ -z "$task_count" ]]
then
  echo "ERROR: Both task_id and task_count have to be defined, invalid --batch argument?"
  exit 1
else
  subject_start=$(((task_id - 1) * num_subjects / task_count))
  subject_end=$((task_id * num_subjects / task_count))
  subject_end=$((subject_end < num_subjects ? subject_end : num_subjects))
  echo "INFO: Processing subjects $((subject_start + 1)) to $subject_end"
fi

if [[ "$parallel_pipelines" == "2" ]] ; then
  if [[ "$seg_only" == "true" ]] ; then parallel_pipelines=1
  elif [[ "$surf_only" == "true" ]] ; then parallel_pipelines=1; num_parallel_seg=$num_parallel_surf
  fi
fi

# Without --threads or OMP_NUM_THREADS each case picks auto threads for itself, which several cases
# at once would each take in full. So they share what auto gives the machine, passed as --threads,
# which reaches the case through any --run_fastsurfer wrapper, a container included.
threads_sh="$(dirname "$THIS_SCRIPT")/recon_surf/threads.sh"
if [[ -f "$threads_sh" ]]
then
  source "$threads_sh"
  function at_once() { if [[ "$1" == "max" ]] || [[ "$1" -gt "$2" ]] ; then echo "$2" ; else echo "$1" ; fi ; }
  num_cases=$((subject_end - subject_start))
  num_at_once=$(at_once "$num_parallel_seg" "$num_cases")
  if [[ "$parallel_pipelines" == "2" ]] ; then num_at_once=$((num_at_once + $(at_once "$num_parallel_surf" "$num_cases"))) ; fi
  if [[ "$num_at_once" -gt 1 ]] && threads_left_to_auto "${POSITIONAL_FASTSURFER[@]}"
  then
    share="$(share_threads "$num_at_once")"
    # the caps auto would apply in each case, which an explicit --threads skips
    share_seg="$(seg_cap_for_device "$res_device")"
    if [[ "$share" -lt "$share_seg" ]] ; then share_seg="$share" ; fi
    share_surf="$thread_auto_cap_cpu"
    if [[ "$share" -lt "$share_surf" ]] ; then share_surf="$share" ; fi
    POSITIONAL_FASTSURFER+=(--threads_seg "$share_seg" --threads_surf "$share_surf")
    echo "INFO: Up to $num_at_once cases run at the same time and share the machine: each segmentation"
    echo "  uses $share_seg threads and each surface reconstruction $share_surf. Pass --threads to choose them."
    # the options of a subject line follow these, so its own --threads wins
    threads_regex='(^|[[:space:]])--threads(_seg|_surf)?([[:space:]]|$)'
    own_threads=()
    for subject in "${subjects[@]:$subject_start:$num_cases}" ; do
      if [[ "${subject#*=}" =~ $threads_regex ]] ; then own_threads+=("${subject%%=*}") ; fi
    done
    if [[ "${#own_threads[@]}" -gt 0 ]]
    then
      named="${own_threads[*]:0:5}" ; if [[ "${#own_threads[@]}" -gt 5 ]] ; then named+=" ..." ; fi
      echo "WARNING: ${#own_threads[@]} subject line(s) set their own threads ($named). Those cases use"
      echo "  them on top of the shared threads of the others, which can overload the machine."
    fi
  fi
fi

### IF THE SCRIPT GETS TERMINATED, ADD A MESSAGE
for signal in SIGINT SIGTERM ; do
  trap "$(printf 'echo "brun_fastsurfer.sh terminated via signal %s at $(date -R)!"' "$signal")" $signal
done

function iterate_subjects_with_token()
{
  # 1: subject_start
  # 2: subject_end
  # *: subjects
  subject_start="$1"
  subject_end="$2"
  shift
  shift
  subjects=("$@")
  # i is a 1-to-n index of the subject
  local i="$subject_start"
  for subject in "${subjects[@]:$subject_start:$((subject_end - subject_start))}"
  do
    echo "#@#!NEXT-SUBJECT:$subject"
  done
}

function run_single()
{
  # Usage:
  # run_single subject-specific-string "debug" "statusfile" "parallel_pipelines" "num_parallel_seg" "num_parallel_surf" "run_fastsurfer" "POSITIONAL_FASTSURFER..."
  # run_single "<subject_id>=<path-to-t1> <more options>" ...

  # 1: subject-specific-string
  # 2: debug
  # 3: statusfile
  # 4: parallel_pipelines
  # 5: num_parallel_seg
  # 6: num_parallel_surf
  # *: run_fastsurfer
  # |
  # *: POSITIONAL_FASTSURFER

  local subject_id="<undefined>" mode="" args=() do_seg="true" do_surf="true" skip="false" parallel cmd status=unknown
  local debug="$2" statusfile="$3" parallel_pipelines="$4" num_parallel_seg="$5" num_parallel_surf="$6" statustext
  local run_fastsurfer=() POSITIONAL_FASTSURFER=()
  local position=0 arg image_path="<undefined>" returncode=0
  # One shell-style token: a backslash-escaped character, a run of characters needing no quoting, or
  # a single- or double-quoted string. Matched with bash's own =~ (an ERE) below rather than `expr`,
  # whose BRE needed the GNU-only \| and \+: BSD/macOS expr rejects those and returned an empty
  # string for every input, so no subject parameters parsed at all there.
  # Built from pieces because a single quote cannot appear inside a single-quoted string.
  local sq="'"
  local escaped='\\.'
  local plain='[^'"$sq"'"[:space:]\]+'
  local single="$sq"'([^'"$sq"']*|'"$sq$sq"')*'"$sq"
  local double='"([^"\]+|\\.)*"'
  local token_re="^(($escaped|$plain|$single|$double)+)"
  subject_id=$(echo "$1" | cut -d= -f1)
  # no --output-delimiter: it is GNU-only, so BSD/macOS cut rejects it and this silently produced an
  # empty string, losing the t1 path. cut -f already joins with the input delimiter, as stools.sh does.
  image_parameters=$(echo "$1" | cut -d= -f2-1000)
  for run in {1..6}; do shift ; done
  for i in "$@" ; do shift; if [[ "$i" == "|" ]] ; then break ; fi ; run_fastsurfer+=("$i") ; done
  POSITIONAL_FASTSURFER=("$@")

  # read image_parameters into args and image_path
  while [[ "$position" -le "${#image_parameters}" ]]
  do
    if [[ -z "${image_parameters:$position}" ]]; then position=$((${#image_parameters} + 1)); continue ; fi
    if [[ "${image_parameters:$position} " =~ $token_re ]] ; then arg="${BASH_REMATCH[1]}" ; else arg="" ; fi
    if [[ -z "$arg" ]]
    then
      # could not parse
      echo "ERROR: Could not parse the line ${image_parameters:$position}, maybe incorrect quoting or escaping?"
      exit 1
    else
      # arg parsed, position is an integer.
      # unquote for the value we pass on, but advance by the length of the source token: arg still
      # holds the quotes and escapes, and $unquoted is usually shorter.
      unquote "$arg"
      if [[ "$position" == "0" ]]; then image_path="$unquoted" ; else args+=("$unquoted") ; fi
      position=$((position + ${#arg}))
    fi
    while [[ "${image_parameters:$position:1}" == " " ]] ; do position=$((position + 1)) ; done
  done

  args+=(--sid "$subject_id")
  for i in "${args[@]}"; do
    if [[ "$i" =~ ^--(viewagg_)?device$ ]] ; then
      echo "ERROR: --device or --viewagg_device may not be specified in the subject-specific arguments!"
      exit 1
    fi
  done
  # check if --seg_only or --surf_only are in the general or the subject-specific arguments, set mode, do_seg and
  # do_surf accordingly
  for i in "${POSITIONAL_FASTSURFER[@]}" "${args[@]}"; do
    if [[ "$i" == "--seg_only" ]] ; then mode="$i" ; do_surf="false"
    elif [[ "$i" == "--surf_only" ]] ; then mode="$i" ; do_seg="false"
    fi
  done
  if [[ "$do_seg" == "false" ]] && [[ "$do_surf" == "false" ]]
  then
    echo "INFO: Skipping subject_id $subject_id (deselected)"
    skip="true"
  elif [[ -n "$statusfile" ]] && [[ "$do_surf" == "true" ]] && [[ "$do_seg" == "false" ]]
  then
    ## if status in statusfile is "Failed" last, skip this
    prev_ifs="$IFS" ; IFS=''
    while read -r line ; do
      IFS=" "
      subject="$(echo "$line" | cut -d" " -f1)"
      if [[ "$subject" == "$subject_id:" ]] ; then
        statustext="${line:$((${#subject} + 1))}"
        if [[ "${statustext}" =~ ^Failed[[:space:]](--seg_only|seg|both) ]] ; then status="failed"
        elif [[ "${statustext}" =~ ^Finished[[:space:]](--seg_only|seg|both)[[:space:]]success ]] ; then
          status="success"
        fi
      fi
      IFS=""
    done < "$statusfile"
    IFS="$prev_ifs"
    if [[ "$status" == "failed" ]]
    then
      echo "INFO: Skipping $subject_id's surface recon because the segmentation failed."
      echo "$subject_id: Skipping surface recon (failed segmentation)" >> "$statusfile"
      skip="true"
    fi
  fi
  # if we are not skipping this, process
  if [[ "$skip" == "false" ]]
  then
    cmd=("${run_fastsurfer[@]}" --t1 "$image_path" "${POSITIONAL_FASTSURFER[@]}" "${args[@]}")
    if [[ "$debug" == "true" ]] ; then echo "DEBUG:" "${cmd[@]}" ; fi
    # multiple subjects in parallel is possible, then parallel = "true", else parallel = "false"
    if [[ "$num_parallel_seg" == "max" ]] || \
       { [[ "$parallel_pipelines" == 2 ]] && [[ "$num_parallel_surf" == "max" ]] ; }
    then parallel="true" # one of "running pipeline" is max
    else parallel=$([[ $((num_parallel_seg + (parallel_pipelines - 1) * num_parallel_surf)) == 2 ]] && echo "false" || echo "true")
    fi
    if [[ "$parallel" == "true" ]] ; then "${cmd[@]}" | prepend "$subject_id: " ; else "${cmd[@]}" ; fi
    returncode="${PIPESTATUS[0]}"
    if [[ -n "$statusfile" ]] ; then print_status "$subject_id" "$mode" "$returncode" >> "$statusfile"; fi
    if [[ "$returncode" != 0 ]]; then echo "WARNING: $subject_id finished with exit code $returncode!" ; fi
  fi
  # print the #@#!NEXT-SUBJECT token for the processing loop to trigger the next subject's processing
  # also include subject_id and image_parameters for debugging and verbosity
  echo "#@#!NEXT-SUBJECT:$subject_id=$image_parameters"
}

function unquote()
{
  # remove one level of shell quoting from $1, result in $unquoted.
  # The tokenizer in run_single matches the source text of a token, so the quotes and escapes are
  # still in it: '/d/a b.mgz' arrived at --t1 with the quotes attached and no such file exists. This
  # does what the shell would do, without eval, which would run substitutions from a subject list.
  # A literal backslash in a filename consequently has to be written \\, as in any shell.
  local rest="$1" out="" chunk sq="'"
  # the only characters a backslash escapes inside double quotes
  local dq_escapable='$`"\'
  while [[ -n "$rest" ]]
  do
    case "$rest" in
      "\\"?*)
        # backslash escape: take the next character as-is
        out="$out${rest:1:1}" ; rest="${rest:2}" ;;
      "'"*)
        # single quotes: everything up to the next one is literal
        rest="${rest:1}" ; chunk="${rest%%$sq*}" ; out="$out$chunk" ; rest="${rest:$((${#chunk} + 1))}" ;;
      '"'*)
        # double quotes: literal, except that a backslash escapes one of $ ` " \ only. Before any
        # other character it stays, as in the shell, where "a\ b" keeps its backslash.
        rest="${rest:1}"
        while [[ -n "$rest" ]] && [[ "${rest:0:1}" != '"' ]]
        do
          if [[ "${rest:0:1}" == "\\" ]] && [[ -n "${rest:1:1}" ]] \
             && [[ "$dq_escapable" == *"${rest:1:1}"* ]]
          then out="$out${rest:1:1}" ; rest="${rest:2}"
          else out="$out${rest:0:1}" ; rest="${rest:1}"
          fi
        done
        rest="${rest:1}" ;;
      *)
        out="$out${rest:0:1}" ; rest="${rest:1}" ;;
    esac
  done
  unquoted="$out"
}

# this function returns an integer, with 1 meaning "is not a numbered device" (to be used in if statements)
function is_numbered_device() { if [[ "$1" =~ ^auto|cpu|mps|cuda$ ]] ; then return 1 ; else return 0 ; fi }

# this is a handler to convert the device name to a number, effectively only removing the "cuda:" prefix
function device2number() { echo "${1:5}" ; }

timeout_read_token=5
function process_by_token()
{
  # 1: "surf" / "seg" / "both"
  # 2: timeout_read_token
  # 3: debug
  # 4: statusfile
  # 5: parallel_pipelines
  # 6: num_parallel_seg
  # 7: num_parallel_surf
  # 8: device
  # 9: viewagg_evice
  # *: run_fastsurfer
  # |
  # *: POSITIONAL_FASTSURFER

  local subject_id max_processes subject_buffer=() read_in="true" returncode spawn_task="true" mode="$1" line
  local timeout_read_token="$2" debug="$3" statusfile="$4" parallel_pipelines="$5" num_parallel_seg="$6" this_args=()
  local num_parallel_surf="$7" device=() vdevice=() regx="^cuda:[0-9]+," parallel_warn="false" res_args=()
  local device_ready=0 prev_ifs="$IFS"
  IFS=","
  if [[ "$8" =~ $regx ]] ; then for e in ${8:5} ; do device+=("${8:0:5}$e") ; done ; else device=("$8") ; fi
  if [[ "$9" =~ $regx ]] ; then for e in ${9:5} ; do vdevice+=("${9:0:5}$e") ; done ; else vdevice=("$9") ; fi
  IFS="$prev_ifs"

  for run in {1..9}; do shift ; done
  local run_single_args=("$debug" "$statusfile" "$parallel_pipelines" "$num_parallel_seg" "$num_parallel_surf" "$@")

  if [[ "$mode" == "surf" ]] ; then max_processes="$num_parallel_surf" ; else max_processes="$num_parallel_seg" ; fi
  while [[ "$read_in" == "true" ]] || [[ "${#subject_buffer[@]}" -gt 0 ]]
  do
    # this loop performs three tasks/main steps as long as i. we are still reading input (read_in) or ii. have subjects
    # in the buffer:
    # 1. read input and add to subject_buffer,
    # 2. check if we are at the process limit (if we want to process the next case),
    # 3. process the next case from the buffer.

    # initialize res_args
    res_args=()
    # 1. read input and add to subject_buffer
    #    we read from stdin (piping in input from iterate_subjects_with_token) or other process_by_token processes,
    #    if the input "sends" EOF, we stop reading (read_in="false"),
    #    if the input line starts with the token "#@#!NEXT-SUBJECT:", we extract the case data, which follows after the
    #    token, and add it to the subject_buffer -- the queue of cases waiting to be processed.
    #    if the line does not start with the token, we print it directly to stdout, making sure the output of a "parent"
    #    process_by_token gets forwarded to the console output for debugging and logging purposes.
    if [[ "$read_in" == "true" ]]
    then
      IFS=""
      read -r -t "$timeout_read_token" line
      returncode="$?"
      if [[ "$returncode" == 1 ]] ; then read_in="false" # EOF, terminate looking at for input
      elif [[ "$returncode" == 0 ]] ; then # successfully read a line
        if [[ "${line:0:17}" == "#@#!NEXT-SUBJECT:" ]] ; then
          if [[ "$debug" == "true" ]] ; then echo "DEBUG: $mode-subject ${line:17}" ; fi
          subject_buffer+=("${line:17}")
        else echo "$line"
        fi
      # else, e.g. 142 => timeout
      fi
    fi

    # 2. check if we are at the process limit (if we want to process the next case)
    #    check the number of child processes with jobs -pr,
    #    if the child process count is below the limit (max_processes), we want to spawn a new process, i.e. set
    #    spawn_task="true"
    #    if the child process count is at the limit and we are still reading input (read_in="true"), we do not spawn a
    #    new process, i.e. set spawn_task="false", but return to step 1: keep reading the input.
    #    if the child process count is at the limit, but we are no longer reading the input (read_in="false"), we first
    #    want to wait for a currently running child process to finish and then spawn the next process.
    # check job count
    if [[ "$max_processes" == "max" ]] ; then spawn_task="true"
    else
      get_running_jobs
      if [[ "${#running_jobs[@]}" -lt "$max_processes" ]] ; then spawn_task="true"
      elif [[ "$read_in" == "false" ]] ; then wait "${running_jobs[@]}" # wait for any task to finish (stdin is closed)
      else spawn_task="false"
      fi
    fi

    # 3. process the next case from the buffer
    #    if we want to spawn a new process (spawn_task="true") and we have cases in the subject_buffer, we need to
    #    find an available device (if we have multiple devices) and then spawn a new process for the next case in the
    #    buffer with the appropriate device assignment. So Substep 3.1 is to find an available device and substep 3.2 is
    #    to spawn the next process with the appropriate device assignment.
    if [[ "$spawn_task" == "true" ]] && [[ "${#subject_buffer[@]}" -gt 0 ]]
    then
      # Substep 3.1: find an available device (if we have multiple devices)
      # if mode is surf, we do not need to check for device availability, surface processing is only CPU-based.
      # if mode is seg, need to find an available device for both per-view inference and the view aggregation
      #   in both cases, have the list of available devices in the device and vdevice arrays, and we keep track of the
      #   device assignments to child processes in the used_device and used_vdevice arrays. If the child process exists,
      #   the device is still in use. If the child process does not exist, the device is available, and we remove the
      #   assignment.
      if [[ "$mode" == "surf" ]] ; then device_ready=2 ; dev="cpu" ; vdev="cpu"
      else
        device_ready=0
        # Substep 3.1a: Find a device for per-view inference, if device are available, we increase device_ready by 1.
        if [[ "${#device[@]}" -gt 1 ]] ; then
          # go through device assignments, if the processes finished, release the device assignment
          for name in "${device[@]}" ; do
            i="$(device2number "$name")"
            if [[ -z "${used_device[i]}" ]] || [[ -z "$(ps --no-headers "${used_device[i]}")" ]] ; then
              res_args+=("--device" "$name")
              used_device[i]=""
              device_ready=$((device_ready + 1))
              dev="$name"
              break
            fi
          done
        else device_ready=$((device_ready + 1)) ; res_args+=("--device" "${device[0]}") ; dev="${device[0]}"
        fi
        # Substep 3.1b: Find a device for view aggregation, if device are available, we increase device_ready by 1.
        if [[ "${#vdevice[@]}" -gt 1 ]] ; then
          # go through viewagg device assignments, if the processes finished, release the device assignment
          for name in "${vdevice[@]}" ; do
            i="$(device2number "$name")"
            if [[ -z "${used_vdevice[i]}" ]] || [[ -z "$(ps --no-headers "${used_vdevice[i]}")" ]] ; then
              res_args+=("--viewagg_device" "$name")
              used_vdevice[i]=""
              device_ready=$((device_ready + 1))
              vdev="$name"
              break
            fi
          done
        else device_ready=$((device_ready + 1)) ; res_args+=("--viewagg_device" "${vdevice[0]}") ; vdev="${vdevice[0]}"
        fi
      fi
      # at this point, device_ready is greater than 1, we have sufficient resources to start the next process.
      if [[ "$device_ready" -gt 1 ]]
      then
        subject_id=$(echo "${subject_buffer[0]}" | cut -d= -f1)
        if [[ "$debug" == "true" ]] ; then echo "DEBUG: Processing $subject_id with $mode" ; fi

        # start next process
        this_args=( "${run_single_args[@]}" "${res_args[@]}")
        if [[ "$mode" == "seg" ]] ; then run_single "${subject_buffer[0]}" "${this_args[@]}" --seg_only &
        elif [[ "$mode" == "surf" ]] ; then run_single "${subject_buffer[0]}" "${this_args[@]}" --surf_only &
        else run_single "${subject_buffer[0]}" "${this_args[@]}" &
        fi
        pid=$!
        # if dev or vdev is a numbered device, add the child process id to the used devices list (multiple devices)
        if is_numbered_device "$dev" ; then i=$(device2number "$dev") ; used_device[i]="$pid" ; fi
        if is_numbered_device "$vdev" ; then i=$(device2number "$vdev") ; used_vdevice[i]="$pid" ; fi
        # remove the first item from the subject_buffer
        subject_buffer=("${subject_buffer[@]:1}")
      else
        # we have not reached the process limit, but could not find available devices...
        if [[ $parallel_warn == "true" ]] ; then
          echo "WARNING: All devices are in use for parallel seg processing!"
        else
          echo "WARNING: All devices are in use, make sure you are not trying to use more parallel seg processes than"
          echo "  you have devices passed in --device AND --viewagg_device., e.g. '--device cuda:0,1 --viewagg_device"
          echo "  cpu --parallel_seg 2' is fine, but '--device cuda:0,1 --parallel_seg 3' or"
          echo "  '--viewagg_device cuda:0,1 --parallel_seg 3' will cause issues."
          parallel_warn="true"
        fi
        # wait for a device to become available
        sleep 5
      fi
    fi # if can spawn and has job in queue
  done
  get_running_jobs
  # wait for jobs to finish
  if [[ "$debug" == "true" ]]
  then
    echo "DEBUG: Finished scheduling $mode-jobs... waiting for ${#running_jobs[@]} jobs to finish:"
    IFS=" "
    echo "       ${running_jobs[*]}"
    wait "${running_jobs[@]}"
    echo "DEBUG: All $mode-jobs finished!"
  else
    wait "${running_jobs[@]}"
  fi
}

function filter_token()
{
  # just remove the lines with the #@#!NEXT-SUBJECT: token from the output.
  IFS=""
  while read -r line ; do if [[ "${line:0:17}" != "#@#!NEXT-SUBJECT:" ]] ; then echo "$line" ; fi ; done
}


process_by_token_args=(
  "$timeout_read_token" "$debug" "$statusfile" "$parallel_pipelines" "$num_parallel_seg" "$num_parallel_surf"
  "$res_device" "$res_viewagg_device" "${run_fastsurfer[@]}" "|" "${POSITIONAL_FASTSURFER[@]}"
)
iterate_subjects_with_token_args=("$subject_start" "$subject_end" "${subjects[@]}")
if [[ "$parallel_pipelines" == 1 ]] ; then
  mode=both
  if [[ "$seg_only" == "true" ]] ; then mode=seg
  elif [[ "$surf_only" == "true" ]] ; then mode=surf
  fi
  iterate_subjects_with_token "${iterate_subjects_with_token_args[@]}" | \
    process_by_token "$mode" "${process_by_token_args[@]}" | \
    filter_token
else
  # multiple pipelines
  iterate_subjects_with_token "${iterate_subjects_with_token_args[@]}" | \
    process_by_token "seg" "${process_by_token_args[@]}" | \
    process_by_token "surf" "${process_by_token_args[@]}" | \
    filter_token
fi

# always exit successful
exit 0