aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/qsub.nix13
1 files changed, 12 insertions, 1 deletions
diff --git a/lib/qsub.nix b/lib/qsub.nix
index d6a34fe..438dd56 100644
--- a/lib/qsub.nix
+++ b/lib/qsub.nix
@@ -25,7 +25,18 @@
PATH=/usr/bin:/bin:/usr/sbin:/sbin
SHELL=/bin/sh
NIX_BUILD_CORES=${toString ppn}
- id=$(qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script})
+
+ while : ; do
+ qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script} 2>&1 > id
+ if [ $? -eq 0 ] ; then
+ break
+ fi
+ if ! grep "Please retry" id > /dev/null ; then
+ exit 1
+ fi
+ sleep 60
+ done
+ id=$(cat id)
function cleanup {
qdel $id 2>/dev/null || true