diff options
| author | Justin Bedo <cu@cua0.org> | 2018-11-26 15:33:58 +1100 |
|---|---|---|
| committer | Justin Bedo <cu@cua0.org> | 2018-11-26 15:44:30 +1100 |
| commit | ef0386b2137117578560d4ee83f2b7281b929040 (patch) | |
| tree | 78734f6d58ae5b8f6a927a28a0b74d2518bb48e8 | |
| parent | 29aaf9458dc5b7fd77b8c62cdb51da8637112ab1 (diff) | |
qsub: retry qsub if requested
| -rw-r--r-- | lib/qsub.nix | 13 |
1 files changed, 12 insertions, 1 deletions
diff --git a/lib/qsub.nix b/lib/qsub.nix index d6a34fe..438dd56 100644 --- a/lib/qsub.nix +++ b/lib/qsub.nix @@ -25,7 +25,18 @@ PATH=/usr/bin:/bin:/usr/sbin:/sbin SHELL=/bin/sh NIX_BUILD_CORES=${toString ppn} - id=$(qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script}) + + while : ; do + qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script} 2>&1 > id + if [ $? -eq 0 ] ; then + break + fi + if ! grep "Please retry" id > /dev/null ; then + exit 1 + fi + sleep 60 + done + id=$(cat id) function cleanup { qdel $id 2>/dev/null || true |