add multicore passthru flag

19 files changed, 83 insertions, 63 deletions

diff --git a/default.nix b/default.nix
index aa3d8d2..736eef0 100644
--- a/default.nix
+++ b/default.nix
@@ -63,7 +63,7 @@ let
     # Export nixpkgs and standard library lib
     pkgs = nixpkgs;
     lib = nixpkgs.lib // { types = types; };
-    stage = nixpkgs.stdenvNoCC.mkDerivation;
+    stage = x: { multicore = false; } // nixpkgs.stdenvNoCC.mkDerivation x;
 
     # splitting/joining
     splitFile = file: drv: stage {
diff --git a/lib/qsub.nix b/lib/qsub.nix
index 6b40431..6370a98 100644
--- a/lib/qsub.nix
+++ b/lib/qsub.nix
@@ -2,70 +2,73 @@
 
 with lib;
 
-{ ppn, mem, walltime, queue ? null, qsubFlags ? null, tmpDir, sleepTime}: drv: lib.overrideDerivation drv ({ args, builder, name, ... }: {
-  builder = "/bin/bash";
-  args = let
-    script = writeScript "qsub-script" ''
-      #!${stdenv.shell}
-      while [ ! -e ${tmpDir}/qsub-$PBS_JOBID ] ; do
-        sleep ${toString sleepTime}
-      done
-      set -a
-      . ${tmpDir}/qsub-$PBS_JOBID/nix-set
-      set +a
-      TMPDIR=${tmpDir}/qsub-$PBS_JOBID
-      TEMP=$TMPDIR
-      TMP=$TMPDIR
-      NIX_BUILD_TOP=$TMPDIR
-      cd $TMPDIR
-      ${builder} ${lib.escapeShellArgs args} &> qsub-log
-      echo $? > qsub-exit
-    '';
+{ ppn, mem, walltime, queue ? null, qsubFlags ? null, tmpDir, sleepTime}:
+drv:
+  let ppnReified = if drv.multicore then ppn else 1;
+  in lib.overrideDerivation drv ({ args, builder, name, ... }: {
+    builder = "/bin/bash";
+    args = let
+      script = writeScript "qsub-script" ''
+        #!${stdenv.shell}
+        while [ ! -e ${tmpDir}/qsub-$PBS_JOBID ] ; do
+          sleep ${toString sleepTime}
+        done
+        set -a
+        . ${tmpDir}/qsub-$PBS_JOBID/nix-set
+        set +a
+        TMPDIR=${tmpDir}/qsub-$PBS_JOBID
+        TEMP=$TMPDIR
+        TMP=$TMPDIR
+        NIX_BUILD_TOP=$TMPDIR
+        cd $TMPDIR
+        ${builder} ${lib.escapeShellArgs args} &> qsub-log
+        echo $? > qsub-exit
+      '';
 
-    qsub = writeScript "qsub" ''
-      #!${stdenv.shell}
-      PATH=/usr/bin:/bin:/usr/sbin:/sbin
-      SHELL=/bin/sh
-      NIX_BUILD_CORES=${toString ppn}
+      qsub = writeScript "qsub" ''
+        #!${stdenv.shell}
+        PATH=/usr/bin:/bin:/usr/sbin:/sbin
+        SHELL=/bin/sh
+        NIX_BUILD_CORES=${toString ppnReified}
 
-      while : ; do
-        qsub -l nodes=1:ppn=${toString ppn},mem=${toString mem}gb,walltime=${walltime} \
-          -N "${name}" \
-          ${optionalString (queue != null) "-q ${queue}"} \
-          ${optionalString (qsubFlags != null) qsubFlags} \
-          ${script} 2>&1 > id
-        if [ $? -eq 0 ] ; then
-          break
-        fi
-        if ! grep "Please retry" id > /dev/null ; then
-          cat id >&2
-          exit 1
-        fi
-        sleep ${toString sleepTime}
-      done
-      id=$(cat id)
-      echo $id
+        while : ; do
+          qsub -l nodes=1:ppn=${toString ppnReified},mem=${toString mem}gb,walltime=${walltime} \
+            -N "${name}" \
+            ${optionalString (queue != null) "-q ${queue}"} \
+            ${optionalString (qsubFlags != null) qsubFlags} \
+            ${script} 2>&1 > id
+          if [ $? -eq 0 ] ; then
+            break
+          fi
+          if ! grep "Please retry" id > /dev/null ; then
+            cat id >&2
+            exit 1
+          fi
+          sleep ${toString sleepTime}
+        done
+        id=$(cat id)
+        echo $id
 
-      function cleanup {
-        qdel $id 2>/dev/null || true
-        sleep ${toString sleepTime}
-        rm -rf ${tmpDir}/qsub-$id
-      }
-      trap cleanup INT TERM EXIT
+        function cleanup {
+          qdel $id 2>/dev/null || true
+          sleep ${toString sleepTime}
+          rm -rf ${tmpDir}/qsub-$id
+        }
+        trap cleanup INT TERM EXIT
 
-      cp -r $TMPDIR ${tmpDir}/qsub-$id
-      set > ${tmpDir}/qsub-$id/nix-set
-      until qstat -f ''${id%%.} 2>&1 | grep "\(Unknown Job\|job_state = C\)" > /dev/null ; do
-        sleep ${toString sleepTime}
-      done
-      cat ${tmpDir}/qsub-$id/qsub-log
-      if [ -e ${tmpDir}/qsub-$id/qsub-exit ]; then
-        exitCode=$(cat ${tmpDir}/qsub-$id/qsub-exit)
-      else
-        exitCode=1
-      fi
-      exit $exitCode
-    '';
+        cp -r $TMPDIR ${tmpDir}/qsub-$id
+        set > ${tmpDir}/qsub-$id/nix-set
+        until qstat -f ''${id%%.} 2>&1 | grep "\(Unknown Job\|job_state = C\)" > /dev/null ; do
+          sleep ${toString sleepTime}
+        done
+        cat ${tmpDir}/qsub-$id/qsub-log
+        if [ -e ${tmpDir}/qsub-$id/qsub-exit ]; then
+          exitCode=$(cat ${tmpDir}/qsub-$id/qsub-exit)
+        else
+          exitCode=1
+        fi
+        exit $exitCode
+      '';
 
-    in [ "-c" qsub ];
-})
+      in [ "-c" qsub ];
+  })
diff --git a/tools/ascat-callCNV.nix b/tools/ascat-callCNV.nix
index 357ab8a..f74bb18 100644
--- a/tools/ascat-callCNV.nix
+++ b/tools/ascat-callCNV.nix
@@ -34,4 +34,5 @@ stage rec {
       -protocol WGS \
       -cpus $NIX_BUILD_CORES
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/ascat-gccorrect.nix b/tools/ascat-gccorrect.nix
index c1838d9..584c2ae 100644
--- a/tools/ascat-gccorrect.nix
+++ b/tools/ascat-gccorrect.nix
@@ -33,4 +33,5 @@ stage rec {
       sed 1d $f >> $out
     done
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/bowtie-align.nix b/tools/bowtie-align.nix
index 67a1169..65d0e9f 100644
--- a/tools/bowtie-align.nix
+++ b/tools/bowtie-align.nix
@@ -32,4 +32,5 @@ in stage {
       > $out
   '';
   passthru.filetype = if bamOutput then filetype.bam {ref = ref; sorting = sort.name {};} else filetype.sam {ref = ref; sorting = sort.name {};};
+  passthru.multicore = true;
 }
diff --git a/tools/bowtie-index.nix b/tools/bowtie-index.nix
index 4554ad1..4607995 100644
--- a/tools/bowtie-index.nix
+++ b/tools/bowtie-index.nix
@@ -18,4 +18,5 @@ stage {
     mkdir $out
     bowtie2-build --seed ${toString seed} --threads $NIX_BUILD_CORES ${optionalString (flags != null) flags} ${ref} $out/ref
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/bwa-mem.nix b/tools/bwa-mem.nix
index 1427b49..e58ef03 100644
--- a/tools/bwa-mem.nix
+++ b/tools/bwa-mem.nix
@@ -36,4 +36,5 @@ in stage {
       > $out
   '';
   passthru.filetype = if bamOutput then filetype.bam {ref = ref; sorting = sort.name {};} else filetype.sam {ref = ref; sorting = sort.name {};};
+  passthru.multicore = true;
 }
diff --git a/tools/cnvkit-batch.nix b/tools/cnvkit-batch.nix
index 6fb97a4..d033dc4 100644
--- a/tools/cnvkit-batch.nix
+++ b/tools/cnvkit-batch.nix
@@ -37,4 +37,5 @@ stage {
     mkdir $out
     cp * $out
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/gridss-assemble.nix b/tools/gridss-assemble.nix
index e5cb3ca..27bbe32 100644
--- a/tools/gridss-assemble.nix
+++ b/tools/gridss-assemble.nix
@@ -57,4 +57,5 @@ stage rec {
       TMP_DIR=$TMPDIR/
   '';
   passthru.filetype = filetype.bam { ref = ref; sorting = sort.name {}; };
+  passthru.multicore = true;
 }
diff --git a/tools/gridss-callVariants.nix b/tools/gridss-callVariants.nix
index 4117126..1ec9206 100644
--- a/tools/gridss-callVariants.nix
+++ b/tools/gridss-callVariants.nix
@@ -50,4 +50,5 @@ stage rec {
       ${optionalString (blacklist != null) ("BLACKLIST=" + blacklist)} \
       ${optionalString (flags != null) flags}
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/gridss-softClipsToSplitReads.nix b/tools/gridss-softClipsToSplitReads.nix
index 315fbf9..4a73e3c 100644
--- a/tools/gridss-softClipsToSplitReads.nix
+++ b/tools/gridss-softClipsToSplitReads.nix
@@ -37,4 +37,5 @@ stage rec {
 			WORKER_THREADS=$NIX_BUILD_CORES
     '';
   passthru.filetype = filetype.bam { ref = ref; sorting = sort.none {}; };
+  passthru.multicore = true;
 }
diff --git a/tools/kallisto-quant.nix b/tools/kallisto-quant.nix
index 1deab67..9f0ab57 100644
--- a/tools/kallisto-quant.nix
+++ b/tools/kallisto-quant.nix
@@ -44,4 +44,5 @@ stage {
       -t $NIX_BUILD_CORES \
       ${concatStringsSep " " inputs}
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/mosdepth-depth.nix b/tools/mosdepth-depth.nix
index 448da11..2dbfc8d 100644
--- a/tools/mosdepth-depth.nix
+++ b/tools/mosdepth-depth.nix
@@ -17,4 +17,5 @@ stage {
     ln -s ${bionix.samtools.index indexAttrs input} input.bam.bai
     mosdepth -t $NIX_BUILD_CORES ${optionalString (flags != null) flags} $out/out input.bam
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/platypus-callVariants.nix b/tools/platypus-callVariants.nix
index a5e497e..f3b3e7c 100644
--- a/tools/platypus-callVariants.nix
+++ b/tools/platypus-callVariants.nix
@@ -37,4 +37,5 @@ stage {
       --bamFiles=${concatMapStringsSep "," (p: "${filename p}.bam") inputs}
   '';
   passthru.filetype = filetype.vcf {ref = ref;};
+  passthru.multicore = true;
 }
diff --git a/tools/samtools-flagstat.nix b/tools/samtools-flagstat.nix
index 57d8bb6..d733f2f 100644
--- a/tools/samtools-flagstat.nix
+++ b/tools/samtools-flagstat.nix
@@ -10,4 +10,5 @@ stage {
   name = "samtools-index";
   buildInputs = with pkgs; [ samtools ];
   buildCommand = "samtools flagstat -@ $NIX_BUILD_CORES ${input} > $out";
+  passthru.multicore = true;
 }
diff --git a/tools/samtools-index.nix b/tools/samtools-index.nix
index 84ebbe1..d0f8d7a 100644
--- a/tools/samtools-index.nix
+++ b/tools/samtools-index.nix
@@ -19,4 +19,5 @@ stage {
     samtools index -@ $NIX_BUILD_CORES ${optionalString (flags != null) flags} input.bam
     cp input.bam.bai $out
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/samtools-sort.nix b/tools/samtools-sort.nix
index d01c83f..e3fbff8 100644
--- a/tools/samtools-sort.nix
+++ b/tools/samtools-sort.nix
@@ -34,4 +34,5 @@ in stage {
           ${input} > $out
       '';
   passthru.filetype = if nameSort then bionix.types.nameSort outfmtR else coordSort outfmtR;
+  passthru.multicore = true;
 }
diff --git a/tools/strelka-call.nix b/tools/strelka-call.nix
index 1162558..4471d7d 100644
--- a/tools/strelka-call.nix
+++ b/tools/strelka-call.nix
@@ -40,4 +40,5 @@ stage {
 
     cp -r results $out
   '';
+  passthru.multicore = true;
 }
diff --git a/tools/strelka-callSomatic.nix b/tools/strelka-callSomatic.nix
index 5be4c0e..ed5626b 100644
--- a/tools/strelka-callSomatic.nix
+++ b/tools/strelka-callSomatic.nix
@@ -42,4 +42,5 @@ stage {
 
     cp -r results $out
   '';
+  passthru.multicore = true;
 }