From ef0386b2137117578560d4ee83f2b7281b929040 Mon Sep 17 00:00:00 2001
From: Justin Bedo <cu@cua0.org>
Date: Mon, 26 Nov 2018 15:33:58 +1100
Subject: qsub: retry qsub if requested

---
 lib/qsub.nix | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/qsub.nix b/lib/qsub.nix
index d6a34fe..438dd56 100644
--- a/lib/qsub.nix
+++ b/lib/qsub.nix
@@ -25,7 +25,18 @@
       PATH=/usr/bin:/bin:/usr/sbin:/sbin
       SHELL=/bin/sh
       NIX_BUILD_CORES=${toString ppn}
-      id=$(qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script})
+
+      while : ; do
+        qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} -N "${name}" ${script} 2>&1 > id
+        if [ $? -eq 0 ] ; then
+          break
+        fi
+        if ! grep "Please retry" id > /dev/null ; then
+          exit 1
+        fi
+        sleep 60
+      done
+      id=$(cat id)
 
       function cleanup {
         qdel $id 2>/dev/null || true
-- 
cgit v1.2.3