From bac9248a5e08e8afdf5485a6e27cfe72e1ca5090 Mon Sep 17 00:00:00 2001 From: Justin Bedo Date: Fri, 29 Oct 2021 17:39:01 +1100 Subject: treewide reformatting and simplification - simplify with statix - reformat with nixpkgs-fmt --- default.nix | 87 +++++++------ doc/default.nix | 2 +- examples/call.nix | 16 +-- examples/default.nix | 9 +- examples/ex-nextflow/nextflow-example1.nix | 8 +- examples/ex-tnpair/cluster.nix | 14 ++- examples/ex-tnpair/default.nix | 3 +- examples/ex-tnpair/tnpair.nix | 33 ++--- examples/ex-wdl/wdl-scatter-gather.nix | 5 +- lib/google.nix | 4 +- lib/qsub.nix | 139 +++++++++++---------- lib/references.nix | 194 +++++++++++++++-------------- lib/shard-regex.nix | 64 ++++++---- lib/shard.nix | 6 +- lib/slurm.nix | 39 +++--- lib/types.nix | 32 ++--- test-tnpair.nix | 101 ++++++++------- tools/ascat-app.nix | 19 +-- tools/ascat-callCNV.nix | 6 +- tools/ascat.nix | 8 +- tools/bowtie-align.nix | 7 +- tools/bowtie.nix | 4 +- tools/bwa-mem.nix | 7 +- tools/bwa-mem2-app.nix | 2 +- tools/bwa-mem2.nix | 7 +- tools/bwa.nix | 6 +- tools/cnvkit-batch.nix | 15 +-- tools/cnvkit-scatter.nix | 5 +- tools/cnvkit.nix | 6 +- tools/compression-bunzip2.nix | 2 +- tools/compression-bzip2.nix | 2 +- tools/compression-gunzip.nix | 2 +- tools/compression-gzip.nix | 2 +- tools/compression.nix | 75 ++++++----- tools/crumble-app.nix | 2 +- tools/crumble.nix | 2 +- tools/delly-call.nix | 4 +- tools/delly.nix | 2 +- tools/facets-app.nix | 14 ++- tools/facets-call.nix | 4 +- tools/facets.nix | 6 +- tools/fastp-check.nix | 45 +++---- tools/fastp.nix | 6 +- tools/fastqc.nix | 4 +- tools/gridss-assemble.nix | 14 +-- tools/gridss-callVariants.nix | 28 ++--- tools/gridss-computeSamTags.nix | 12 +- tools/gridss-configFile.nix | 62 ++++----- tools/gridss-extractSVReads.nix | 6 +- tools/gridss-softClipsToSplitReads.nix | 8 +- tools/gridss-variants.nix | 24 ++-- tools/gridss.nix | 41 +++--- tools/hisat2-align.nix | 7 +- tools/hisat2-index.nix | 4 +- tools/hisat2.nix | 4 +- tools/infercnv-app.nix | 42 ++++--- tools/infercnv-infer.nix | 12 +- tools/infercnv.nix | 6 +- tools/kallisto-index.nix | 5 +- tools/kallisto-quant.nix | 11 +- tools/kallisto.nix | 4 +- tools/last-align.nix | 9 +- tools/last.nix | 2 +- tools/lumpy-call.nix | 5 +- tools/lumpy.nix | 2 +- tools/manta-call.nix | 6 +- tools/manta.nix | 2 +- tools/minimap2-align.nix | 5 +- tools/minimap2.nix | 2 +- tools/mosdepth-depth.nix | 2 +- tools/mosdepth-plot.nix | 3 +- tools/mosdepth.nix | 4 +- tools/mutect-app.nix | 19 +-- tools/mutect-call.nix | 11 +- tools/mutect.nix | 6 +- tools/octopus-call.nix | 4 +- tools/octopus-callSomatic.nix | 4 +- tools/octopus.nix | 6 +- tools/picard-markDuplicates.nix | 34 ++--- tools/picard.nix | 8 +- tools/platypus-callVariants.nix | 8 +- tools/platypus.nix | 2 +- tools/sambamba.nix | 25 ++-- tools/samtools-merge.nix | 4 +- tools/samtools-sort.nix | 7 +- tools/samtools-view.nix | 5 +- tools/snap-align.nix | 7 +- tools/snap-app.nix | 2 +- tools/snap.nix | 6 +- tools/snpeff-annotate.nix | 9 +- tools/snpeff-dbnsfp.nix | 9 +- tools/snpeff.nix | 8 +- tools/snver-call.nix | 2 +- tools/snver.nix | 4 +- tools/star-align.nix | 7 +- tools/star-index.nix | 4 +- tools/star.nix | 2 +- tools/strelka-call.nix | 77 ++++++------ tools/strelka-callSomatic.nix | 81 ++++++------ tools/strelka.nix | 4 +- tools/whisper-align.nix | 7 +- tools/whisper.nix | 2 +- tools/xenomapper-allocate.nix | 6 +- tools/xenomapper.nix | 6 +- 104 files changed, 936 insertions(+), 802 deletions(-) diff --git a/default.nix b/default.nix index 1944de0..59357b2 100644 --- a/default.nix +++ b/default.nix @@ -5,10 +5,11 @@ let bionix = nixpkgs.lib.makeExtensible (self: let callBionix = file: attrs: import file ({ bionix = self; } // attrs); - in with self; { - callBionix = callBionix; + in + with self; { + inherit callBionix; id = x: x; - exec = f: x: y: f x y; + exec = id; exec' = f: exec (_: f) { }; exec'' = f: exec' (_: f) { }; callBionixE = p: exec (callBionix p); @@ -52,15 +53,17 @@ let slurm-run = callPackage ./lib/slurm.nix { }; slurm-exec = f: x: y: - slurm-run x (f (builtins.removeAttrs x [ - "ppn" - "mem" - "walltime" - "partition" - "slurmFlags" - "salloc" - "srun" - ]) y); + slurm-run x (f + (builtins.removeAttrs x [ + "ppn" + "mem" + "walltime" + "partition" + "slurmFlags" + "salloc" + "srun" + ]) + y); slurm = bionix.extend (self: super: { exec = super.slurm-run; }); qsub = attrs: bionix.extend (self: super: @@ -78,7 +81,8 @@ let qsub = attrs: (callPackage ./lib/qsub.nix { }) (qsubDefs // attrs); exec = f: x: y: qsub (builtins.intersectAttrs qsubDefs x) (super.exec f - (builtins.removeAttrs x (builtins.attrNames qsubDefs)) y); + (builtins.removeAttrs x (builtins.attrNames qsubDefs)) + y); }); def = f: defs: attrs: f (defs // attrs); @@ -88,22 +92,24 @@ let name = "link-outputs"; outputs = [ "out" ] ++ attrNames x; nativeBuildInputs = [ pkgs.perl ]; - buildCommand = let - recurse = x: - if x ? type && x.type == "derivation" then - x - else if builtins.typeOf x == "set" then - linkOutputs x - else - abort "linkOutputs: unsupported type"; - link = dst: src: '' - ln -s ${recurse src} $(perl -e 'print $ENV{"${dst}"}') ; ln -s ${ - recurse src - } $out/${dst} - ''; - in '' - mkdir $out - '' + (concatStringsSep "\n" (mapAttrsToList link x)); + buildCommand = + let + recurse = x: + if x ? type && x.type == "derivation" then + x + else if builtins.typeOf x == "set" then + linkOutputs x + else + abort "linkOutputs: unsupported type"; + link = dst: src: '' + ln -s ${recurse src} $(perl -e 'print $ENV{"${dst}"}') ; ln -s ${ + recurse src + } $out/${dst} + ''; + in + '' + mkdir $out + '' + (concatStringsSep "\n" (mapAttrsToList link x)); passthru.linkInputs = x; }; @@ -127,15 +133,15 @@ let # Export nixpkgs and standard library lib pkgs = nixpkgs; lib = nixpkgs.lib // { - types = types; + inherit types; shard = callBionix ./lib/shard.nix { }; }; stage = x@{ name, stripStorePaths ? true, multicore ? false, ... }: (if stripStorePaths then strip else x: x) - (nixpkgs.stdenvNoCC.mkDerivation (x // { - name = "bionix-" + name; - inherit multicore; - })); + (nixpkgs.stdenvNoCC.mkDerivation (x // { + name = "bionix-" + name; + inherit multicore; + })); strip = drv: let stripCommand = '' @@ -156,7 +162,8 @@ let rewriteOutput $o done ''; - in drv.overrideAttrs (attrs: + in + drv.overrideAttrs (attrs: if attrs ? buildCommand then { buildCommand = attrs.buildCommand + stripCommand; } else { @@ -186,13 +193,15 @@ let overlayByType = { lambda = bionix: overlay: bionix.extend - (self: super: nixpkgs.lib.recursiveUpdate super (overlay self super)); + (self: super: nixpkgs.lib.recursiveUpdate super (overlay self super)); path = bionix: path: overlay bionix (import path); }; overlay = bionix: overlay: let overlayType = builtins.typeOf overlay; - in if overlayByType ? ${overlayType} then - overlayByType.${overlayType} bionix overlay + in + if overlayByType ? "${overlayType}" then + overlayByType."${overlayType}" bionix overlay else builtins.throw ("cannot overlay type " + overlayType); -in with nixpkgs.lib; foldl overlay bionix overlays +in +with nixpkgs.lib; foldl overlay bionix overlays diff --git a/doc/default.nix b/doc/default.nix index d68a6b1..ac8bfeb 100644 --- a/doc/default.nix +++ b/doc/default.nix @@ -1,4 +1,4 @@ -{ bionix ? import ./.. {} }: +{ bionix ? import ./.. { } }: with bionix; diff --git a/examples/call.nix b/examples/call.nix index 8a38fdb..81a112e 100644 --- a/examples/call.nix +++ b/examples/call.nix @@ -2,9 +2,10 @@ # with the Platypus variant caller. Each input is preprocessed by aligning # against a reference genome (defaults to GRCH38), fixing mate information, and # marking duplicates. Finally platypus is called over all samples. -{bionix ? import {} -,inputs -,ref ? bionix.ref.grch38.seq }: +{ bionix ? import { } +, inputs +, ref ? bionix.ref.grch38.seq +}: with bionix; with lib; @@ -13,9 +14,10 @@ let preprocess = flip pipe [ (bwa.align { inherit ref; }) (samtools.sort { nameSort = true; }) - (samtools.fixmate {}) - (samtools.sort {}) - (samtools.markdup {}) + (samtools.fixmate { }) + (samtools.sort { }) + (samtools.markdup { }) ]; -in platypus.call {} (map preprocess inputs) +in +platypus.call { } (map preprocess inputs) diff --git a/examples/default.nix b/examples/default.nix index 9aace38..ef57341 100644 --- a/examples/default.nix +++ b/examples/default.nix @@ -1,6 +1,6 @@ # This example uses the pipelines specified in the call.nix file on the # synthetic data in this directory. -{bionix ? import {}}: +{ bionix ? import { } }: with bionix; @@ -15,7 +15,7 @@ let sha256 = "0kh29i6fg14dn0fb1xj6pkpk6d83y7zg7aphkbvjrhm82braqkm8"; }; - input2 = fetchFastQ { + input2 = fetchFastQ { url = "https://github.com/PapenfussLab/bionix/raw/master/examples/sample1-2.fq"; sha256 = "0czk85km6a91y0fn4b7f9q7ps19b5jf7jzwbly4sgznps7ir2kdk"; }; @@ -28,7 +28,7 @@ let sha256 = "08gixavfklqvk1m2ic6v56z82vl00qnpsd9xb64z6zl03nz98mcy"; }; - input2 = fetchFastQ { + input2 = fetchFastQ { url = "https://github.com/PapenfussLab/bionix/raw/master/examples/sample2-2.fq"; sha256 = "1xxwm2vq52axpdhm14rh5mg5nzzpxaqnvhzrqhajm27fqksgzjjw"; }; @@ -41,4 +41,5 @@ let sha256 = "0sy9hq8n55knfkiblam50dzaiwhrx6pv8b8l1njdn6kfj4wflz2p"; }; -in import ./call.nix {inherit inputs ref bionix;} +in +import ./call.nix { inherit inputs ref bionix; } diff --git a/examples/ex-nextflow/nextflow-example1.nix b/examples/ex-nextflow/nextflow-example1.nix index 4f00431..687505c 100644 --- a/examples/ex-nextflow/nextflow-example1.nix +++ b/examples/ex-nextflow/nextflow-example1.nix @@ -1,7 +1,8 @@ # This is a translation of the Nextflow example found at # https://www.nextflow.io/example1.html -{ bionix ? import ./../.. {} -, input ? ./sample.fa}: +{ bionix ? import ./../.. { } +, input ? ./sample.fa +}: with bionix; with lib; @@ -24,7 +25,8 @@ let ''; }; -in pipe input [ +in +pipe input [ splitSequences (each reverse) ] diff --git a/examples/ex-tnpair/cluster.nix b/examples/ex-tnpair/cluster.nix index 8d54995..5d4e1de 100644 --- a/examples/ex-tnpair/cluster.nix +++ b/examples/ex-tnpair/cluster.nix @@ -4,11 +4,12 @@ let bionix = import { overlays = [ (_: super: - super."${if tmpDir == null then "slurm" else "qsub"}" { - ppn = 24; - mem = 7; - walltime = "3:00:00"; - } // super.lib.optionalAttrs (tmpDir != null) { inherit tmpDir; }) + super."${if tmpDir == null then "slurm" else "qsub"}" + { + ppn = 24; + mem = 7; + walltime = "3:00:00"; + } // super.lib.optionalAttrs (tmpDir != null) { inherit tmpDir; }) (self: super: with super; { @@ -27,4 +28,5 @@ let }) ]; }; -in import ./. { inherit bionix; } +in +import ./. { inherit bionix; } diff --git a/examples/ex-tnpair/default.nix b/examples/ex-tnpair/default.nix index f9581f8..65a2265 100644 --- a/examples/ex-tnpair/default.nix +++ b/examples/ex-tnpair/default.nix @@ -45,6 +45,7 @@ let fetch = s: mapAttrs (_: fetchFastQGZ) s.inputs; -in import ./tnpair.nix { +in +import ./tnpair.nix { inherit pair fetch bionix; } diff --git a/examples/ex-tnpair/tnpair.nix b/examples/ex-tnpair/tnpair.nix index 9e6befe..414a391 100644 --- a/examples/ex-tnpair/tnpair.nix +++ b/examples/ex-tnpair/tnpair.nix @@ -1,4 +1,4 @@ -{bionix ? import {}, pair, fetch}: +{ bionix ? import { }, pair, fetch }: with bionix; with lib; @@ -13,7 +13,7 @@ let fetch (align { preset = "sr"; ref = ref.grch38.seq; flags = "-R'@RG\\tID:${s.type}\\tSM:${s.type}'"; }) (sort { nameSort = true; }) - (fixmate {}) + (fixmate { }) (sort { }) (markdup { }) ]; @@ -28,21 +28,26 @@ let bams = mapAttrs (_: preprocess) pair; - variants = let - somatic = strelka.callSomatic { } bams; in mapAttrs (_: flip pipe [ - (compression.uncompress { }) - (snpeff.annotate { db = ref.grch38.snpeff.db; }) - dropErrors - (snpeff.dbnsfp { dbnsfp = ref.grch38.snpeff.dbnsfp; }) - ]) { - "snvs.vcf" = somatic.snvs; - "indels.vcf" = somatic.snvs; - "germline.vcf" = strelka.call { } [bams.normal]; - }; + variants = + let + somatic = strelka.callSomatic { } bams; in + mapAttrs + (_: flip pipe [ + (compression.uncompress { }) + (snpeff.annotate { inherit (ref.grch38.snpeff) db; }) + dropErrors + (snpeff.dbnsfp { inherit (ref.grch38.snpeff) dbnsfp; }) + ]) + { + "snvs.vcf" = somatic.snvs; + "indels.vcf" = somatic.snvs; + "germline.vcf" = strelka.call { } [ bams.normal ]; + }; cnvs = cnvkit.callCNV { } { normals = [ bams.normal ]; tumours = [ bams.tumour ]; }; -in linkOutputs { +in +linkOutputs { inherit variants; alignments = linkOutputs (mapAttrs' (n: nameValuePair (n + ".bam")) bams); cnvkit = cnvs; diff --git a/examples/ex-wdl/wdl-scatter-gather.nix b/examples/ex-wdl/wdl-scatter-gather.nix index 61dc441..5245662 100644 --- a/examples/ex-wdl/wdl-scatter-gather.nix +++ b/examples/ex-wdl/wdl-scatter-gather.nix @@ -1,6 +1,6 @@ # The scatter-gather example from https://github.com/openwdl/wdl # translated to bionix -{ bionix ? import ./../.. {} }: +{ bionix ? import ./../.. { } }: with bionix; with lib; @@ -30,4 +30,5 @@ let ''; }; -in gather (map analysis prepare) +in +gather (map analysis prepare) diff --git a/lib/google.nix b/lib/google.nix index 354b5c5..06b80b3 100644 --- a/lib/google.nix +++ b/lib/google.nix @@ -10,7 +10,9 @@ let "https://raw.githubusercontent.com/circulosmeos/gdown.pl/master/gdown.pl"; sha256 = "1pw3vg70bgf33akbbphpr6zn3jndv0khmsa3k0m877hgzg1v52qv"; }; -in runCommand "gdown" { +in +runCommand "gdown" +{ nativeBuildInputs = [ perl wget ]; outputHashAlgo = "sha256"; outputHash = sha256; diff --git a/lib/qsub.nix b/lib/qsub.nix index a8827a8..2528162 100644 --- a/lib/qsub.nix +++ b/lib/qsub.nix @@ -4,76 +4,87 @@ with lib; let escape = x: if builtins.typeOf x == "string" then escapeShellArg x else x; -in { ppn, mem, walltime, queue ? null, qsubFlags ? null, tmpDir, sleepTime -, qsubPath ? "/usr/bin" }: +in +{ ppn +, mem +, walltime +, queue ? null +, qsubFlags ? null +, tmpDir +, sleepTime +, qsubPath ? "/usr/bin" +}: drv: let ppnReified = if drv.multicore then ppn else 1; -in overrideDerivation drv ({ args, builder, name, ... }: { +in +overrideDerivation drv ({ args, builder, name, ... }: { builder = "/bin/bash"; - args = let - script = writeScript "qsub-script" '' - #!${stdenv.shell} - while [ ! -e ${tmpDir}/qsub-$PBS_JOBID ] ; do - sleep ${toString sleepTime} - done - set -a - . ${tmpDir}/qsub-$PBS_JOBID/nix-set - set +a - TMPDIR=${tmpDir}/qsub-$PBS_JOBID - TEMP=$TMPDIR - TMP=$TMPDIR - NIX_BUILD_TOP=$TMPDIR - cd $TMPDIR - ${builder} ${concatMapStringsSep " " escape args} &> qsub-log - echo $? > qsub-exit - ''; + args = + let + script = writeScript "qsub-script" '' + #!${stdenv.shell} + while [ ! -e ${tmpDir}/qsub-$PBS_JOBID ] ; do + sleep ${toString sleepTime} + done + set -a + . ${tmpDir}/qsub-$PBS_JOBID/nix-set + set +a + TMPDIR=${tmpDir}/qsub-$PBS_JOBID + TEMP=$TMPDIR + TMP=$TMPDIR + NIX_BUILD_TOP=$TMPDIR + cd $TMPDIR + ${builder} ${concatMapStringsSep " " escape args} &> qsub-log + echo $? > qsub-exit + ''; - qsub = writeScript "qsub" '' - #!${stdenv.shell} - PATH=${qsubPath} - SHELL=/bin/sh - NIX_BUILD_CORES=${toString ppnReified} + qsub = writeScript "qsub" '' + #!${stdenv.shell} + PATH=${qsubPath} + SHELL=/bin/sh + NIX_BUILD_CORES=${toString ppnReified} - while : ; do - qsub -l nodes=1:ppn=${toString ppnReified},mem=${ - toString mem - }gb,walltime=${walltime} \ - -N "${name}" \ - ${optionalString (queue != null) "-q ${queue}"} \ - ${optionalString (qsubFlags != null) qsubFlags} \ - ${script} 2>&1 > id - if [ $? -eq 0 ] ; then - break - fi - if ! grep "Please retry" id > /dev/null ; then - cat id >&2 - exit 1 - fi - sleep ${toString sleepTime} - done - id=$(cat id) - echo $id + while : ; do + qsub -l nodes=1:ppn=${toString ppnReified},mem=${ + toString mem + }gb,walltime=${walltime} \ + -N "${name}" \ + ${optionalString (queue != null) "-q ${queue}"} \ + ${optionalString (qsubFlags != null) qsubFlags} \ + ${script} 2>&1 > id + if [ $? -eq 0 ] ; then + break + fi + if ! grep "Please retry" id > /dev/null ; then + cat id >&2 + exit 1 + fi + sleep ${toString sleepTime} + done + id=$(cat id) + echo $id - function cleanup { - qdel $id 2>/dev/null || true - sleep ${toString sleepTime} - rm -rf ${tmpDir}/qsub-$id - } - trap cleanup INT TERM EXIT + function cleanup { + qdel $id 2>/dev/null || true + sleep ${toString sleepTime} + rm -rf ${tmpDir}/qsub-$id + } + trap cleanup INT TERM EXIT - cp -r $TMPDIR ${tmpDir}/qsub-$id - set > ${tmpDir}/qsub-$id/nix-set - until qstat -f ''${id%%.} 2>&1 | grep "\(Unknown Job\|job_state = C\)" > /dev/null ; do - sleep ${toString sleepTime} - done - cat ${tmpDir}/qsub-$id/qsub-log - if [ -e ${tmpDir}/qsub-$id/qsub-exit ]; then - exitCode=$(cat ${tmpDir}/qsub-$id/qsub-exit) - else - exitCode=1 - fi - exit $exitCode - ''; + cp -r $TMPDIR ${tmpDir}/qsub-$id + set > ${tmpDir}/qsub-$id/nix-set + until qstat -f ''${id%%.} 2>&1 | grep "\(Unknown Job\|job_state = C\)" > /dev/null ; do + sleep ${toString sleepTime} + done + cat ${tmpDir}/qsub-$id/qsub-log + if [ -e ${tmpDir}/qsub-$id/qsub-exit ]; then + exitCode=$(cat ${tmpDir}/qsub-$id/qsub-exit) + else + exitCode=1 + fi + exit $exitCode + ''; - in [ "-c" qsub ]; + in + [ "-c" qsub ]; }) diff --git a/lib/references.nix b/lib/references.nix index 23ccde3..fd8c3cc 100644 --- a/lib/references.nix +++ b/lib/references.nix @@ -26,7 +26,7 @@ rec { buildInputs = with pkgs; [ gawk ]; buildCommand = '' gunzip < $src | awk '/^[^#]/{print "chr" $0;next}{print}' > $out - ''; + ''; passthru.filetype = filetype.vcf { ref = seq; }; }; encode.blacklist = stage { @@ -42,28 +42,30 @@ rec { ''; }; - ensembl = let version = "74"; - in { - cdna = stage { - name = "ensembl-grch37-cdna-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh37.${version}.cdna.all.fa.gz"; - sha256 = "1m62hiw17zcxg3sza0aq53885wb8g202j8lc1ilhmkg2izzbyihj"; + ensembl = + let version = "74"; + in + { + cdna = stage { + name = "ensembl-grch37-cdna-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh37.${version}.cdna.all.fa.gz"; + sha256 = "1m62hiw17zcxg3sza0aq53885wb8g202j8lc1ilhmkg2izzbyihj"; + }; + buildCommand = "gunzip < $src > $out"; + passthru.filetype = filetype.fa { }; }; - buildCommand = "gunzip < $src > $out"; - passthru.filetype = filetype.fa { }; - }; - gtf = stage { - name = "ensembl-grch37-gtf-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/gtf/homo_sapiens/Homo_sapiens.GRCh37.${version}.gtf.gz"; - sha256 = "1m62hiw17zcxg3sza0aq53885wb8g202j8lc1ilhmkg2izzbyihj"; + gtf = stage { + name = "ensembl-grch37-gtf-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/gtf/homo_sapiens/Homo_sapiens.GRCh37.${version}.gtf.gz"; + sha256 = "1m62hiw17zcxg3sza0aq53885wb8g202j8lc1ilhmkg2izzbyihj"; + }; + buildCommand = "gunzip < $src > $out"; }; - buildCommand = "gunzip < $src > $out"; }; - }; snpeff = { db = pkgs.stdenv.mkDerivation rec { name = "GRCh37.75"; @@ -114,7 +116,7 @@ rec { buildInputs = with pkgs; [ gawk ]; buildCommand = '' gunzip < $src | awk '/^[^#]/{print "chr" $0;next}{print}' > $out - ''; + ''; passthru.filetype = filetype.vcf { ref = seq; }; }; encode.blacklist = stage { @@ -129,47 +131,49 @@ rec { gunzip < $bed | awk '{print $1 ":" $2 "-" $3}' > $out ''; }; - ensembl = let version = "97"; - in { - cdna = stage { - name = "ensembl-grch38-cdna-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz"; - sha256 = "1m6hvnvlrsi6bzcmq0lnv0igy3in1a7jp723yc74g4g6zjp3cy8c"; + ensembl = + let version = "97"; + in + { + cdna = stage { + name = "ensembl-grch38-cdna-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz"; + sha256 = "1m6hvnvlrsi6bzcmq0lnv0igy3in1a7jp723yc74g4g6zjp3cy8c"; + }; + buildCommand = "gunzip < $src > $out"; + passthru.filetype = filetype.fa { }; }; - buildCommand = "gunzip < $src > $out"; - passthru.filetype = filetype.fa { }; - }; - ncrna = stage { - name = "ensembl-grch38-ncrna-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz"; - sha256 = "1r0dmybn31wf6xc90z2c08ngivkv39hqa8wqg3vik6s4spwpdhj0"; + ncrna = stage { + name = "ensembl-grch38-ncrna-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz"; + sha256 = "1r0dmybn31wf6xc90z2c08ngivkv39hqa8wqg3vik6s4spwpdhj0"; + }; + buildCommand = "gunzip < $src > $out"; + passthru.filetype = filetype.fa { }; }; - buildCommand = "gunzip < $src > $out"; - passthru.filetype = filetype.fa { }; - }; - gtf = stage { - name = "ensembl-grch38-gtf-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/gtf/homo_sapiens/Homo_sapiens.GRCh38.${version}.gtf.gz"; - sha256 = "068ab5jf87il301jcr0576c4q0smv6kxpv94gnrm3qzl6kvmaawd"; + gtf = stage { + name = "ensembl-grch38-gtf-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/gtf/homo_sapiens/Homo_sapiens.GRCh38.${version}.gtf.gz"; + sha256 = "068ab5jf87il301jcr0576c4q0smv6kxpv94gnrm3qzl6kvmaawd"; + }; + buildCommand = "gunzip < $src > $out"; }; - buildCommand = "gunzip < $src > $out"; - }; - gff3 = stage { - name = "ensembl-grch38-gff-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/gff3/homo_sapiens/Homo_sapiens.GRCh38.${version}.gff3.gz"; - sha256 = "1xvhsn938mw0032qgc9dvw3k2xrhpx77b8ms03fkrs2s67f7zli7"; + gff3 = stage { + name = "ensembl-grch38-gff-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/gff3/homo_sapiens/Homo_sapiens.GRCh38.${version}.gff3.gz"; + sha256 = "1xvhsn938mw0032qgc9dvw3k2xrhpx77b8ms03fkrs2s67f7zli7"; + }; + buildCommand = "gunzip < $src > $out"; }; - buildCommand = "gunzip < $src > $out"; }; - }; snpeff = { db = pkgs.stdenv.mkDerivation rec { name = "GRCh38.86"; @@ -221,48 +225,50 @@ rec { buildCommand = "gunzip < $src > $out"; passthru.filetype = filetype.fa { }; }; - ensembl = let version = "94"; - in { - cdna = stage { - name = "ensembl-grcm38-cdna-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/cdna/Mus_musculus.GRCm38.cdna.all.fa.gz"; - sha256 = "0khp9l6s35lav2xqp7vkk6ybnz4wjihn7lapjf2lbpnbzjb4hp6d"; + ensembl = + let version = "94"; + in + { + cdna = stage { + name = "ensembl-grcm38-cdna-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/cdna/Mus_musculus.GRCm38.cdna.all.fa.gz"; + sha256 = "0khp9l6s35lav2xqp7vkk6ybnz4wjihn7lapjf2lbpnbzjb4hp6d"; + }; + buildCommand = "gunzip < $src > $out"; + passthru.filetype = filetype.fa { }; }; - buildCommand = "gunzip < $src > $out"; - passthru.filetype = filetype.fa { }; - }; - ncrna = stage { - name = "ensembl-grcm38-ncrna-${version}"; - version = "94"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/ncrna/Mus_musculus.GRCm38.ncrna.fa.gz"; - sha256 = "0d997gm8p2b89rm5d46m2x4vz9lijxarfr2lzylnbi8gyqrbagdd"; + ncrna = stage { + name = "ensembl-grcm38-ncrna-${version}"; + version = "94"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/ncrna/Mus_musculus.GRCm38.ncrna.fa.gz"; + sha256 = "0d997gm8p2b89rm5d46m2x4vz9lijxarfr2lzylnbi8gyqrbagdd"; + }; + buildCommand = "gunzip < $src > $out"; + passthru.filetype = filetype.fa { }; }; - buildCommand = "gunzip < $src > $out"; - passthru.filetype = filetype.fa { }; - }; - gtf = stage { - name = "ensembl-grcm38-gtf-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/gtf/mus_musculus/Mus_musculus.GRCm38.${version}.gtf.gz"; - sha256 = "0i61jq5i5bcini5nxqxxp3rnz2xzgychvzdn0k451f5rv053lp3v"; + gtf = stage { + name = "ensembl-grcm38-gtf-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/gtf/mus_musculus/Mus_musculus.GRCm38.${version}.gtf.gz"; + sha256 = "0i61jq5i5bcini5nxqxxp3rnz2xzgychvzdn0k451f5rv053lp3v"; + }; + buildCommand = "gunzip < $src > $out"; }; - buildCommand = "gunzip < $src > $out"; - }; - gff3 = stage { - name = "ensembl-grcm38-gff-${version}"; - src = pkgs.fetchurl { - url = - "ftp://ftp.ensembl.org/pub/release-${version}/gff3/mus_musculus/Mus_musculus.GRCm38.${version}.gff3.gz"; - sha256 = "15fmdpx6g96fygwhs10jwrb2q5p9y64bc3d4clg856k57qzzgprg"; + gff3 = stage { + name = "ensembl-grcm38-gff-${version}"; + src = pkgs.fetchurl { + url = + "ftp://ftp.ensembl.org/pub/release-${version}/gff3/mus_musculus/Mus_musculus.GRCm38.${version}.gff3.gz"; + sha256 = "15fmdpx6g96fygwhs10jwrb2q5p9y64bc3d4clg856k57qzzgprg"; + }; + buildCommand = "gunzip < $src > $out"; }; - buildCommand = "gunzip < $src > $out"; }; - }; }; mm10 = mm10-p4; diff --git a/lib/shard-regex.nix b/lib/shard-regex.nix index 06fba4d..9dc1720 100644 --- a/lib/shard-regex.nix +++ b/lib/shard-regex.nix @@ -1,54 +1,64 @@ -{bionix, n}: +{ bionix, n }: with bionix; with lib.types; input: let - re = let f = matchFiletype' "shard-regex" { + re = + let f = matchFiletype' "shard-regex" { fa = _: "^>"; fq = _: "^@"; gz = f; bz2 = f; }; - in f input.filetype; - decompress = matchFiletype "shard-regex-decompression" { + in f input.filetype; + decompress = matchFiletype "shard-regex-decompression" + { fa = _: "cat"; fq = _: "cat"; gz = _: "gunzip"; bz2 = _: "bunzip2"; - } input; - compress = matchFiletype "shard-regex-compression" { + } + input; + compress = matchFiletype "shard-regex-compression" + { fa = _: "cat"; fq = _: "cat"; gz = _: "gzip"; bz2 = _: "bzip2"; - } input; - compressPkgs = with bionix.pkgs; matchFiletype "shard-regex-compression" { - fa = _: []; - fq = _: []; + } + input; + compressPkgs = with bionix.pkgs; matchFiletype "shard-regex-compression" + { + fa = _: [ ]; + fq = _: [ ]; gz = _: [ gzip ]; bz2 = _: [ bzip2 ]; - } input; -in stage { + } + input; +in +stage { name = "shard"; outputs = [ "out" ] ++ builtins.genList (i: "out" + toString (i + 2)) (n - 1); buildInputs = [ pkgs.gawk ] ++ compressPkgs; - buildCommand = let - awkScript = pkgs.writeText "shard.awk" '' - BEGIN{cout=0} - FNR==NR{out[nout++] = $0;next} - /${re}/{cout = (cout + 1) % nout} - {print > out[cout]} + buildCommand = + let + awkScript = pkgs.writeText "shard.awk" '' + BEGIN{cout=0} + FNR==NR{out[nout++] = $0;next} + /${re}/{cout = (cout + 1) % nout} + {print > out[cout]} + ''; + in + '' + for o in $outputs ; do + echo $(basename ''${!o}) >> outputs + done + awk -f ${awkScript} outputs <(${decompress} < ${input}) + for o in $outputs ; do + ${compress} < $(basename ''${!o}) > ''${!o} + done ''; - in '' - for o in $outputs ; do - echo $(basename ''${!o}) >> outputs - done - awk -f ${awkScript} outputs <(${decompress} < ${input}) - for o in $outputs ; do - ${compress} < $(basename ''${!o}) > ''${!o} - done - ''; passthru.filetype = input.filetype; } diff --git a/lib/shard.nix b/lib/shard.nix index 95cf251..9b85027 100644 --- a/lib/shard.nix +++ b/lib/shard.nix @@ -1,9 +1,9 @@ -{bionix}: +{ bionix }: with bionix; with lib.types; { - regex = n: input: outputDrvs (callBionixE ./shard-regex.nix { n = n; } input); - fastQPair = n: {input1, input2}: lib.zipListsWith (i: j: {input1 = i; input2 = j;}) (lib.shard.regex n input1) (lib.shard.regex n input2); + regex = n: input: outputDrvs (callBionixE ./shard-regex.nix { inherit n; } input); + fastQPair = n: { input1, input2 }: lib.zipListsWith (i: j: { input1 = i; input2 = j; }) (lib.shard.regex n input1) (lib.shard.regex n input2); } diff --git a/lib/slurm.nix b/lib/slurm.nix index c050628..8c9ad50 100644 --- a/lib/slurm.nix +++ b/lib/slurm.nix @@ -4,23 +4,34 @@ with lib; let escape = x: if builtins.typeOf x == "string" then escapeShellArg x else x; -in { ppn, mem, walltime, partition ? null, slurmFlags ? null -, salloc ? "/usr/bin/salloc", srun ? "/usr/bin/srun", ... }: +in +{ ppn +, mem +, walltime +, partition ? null +, slurmFlags ? null +, salloc ? "/usr/bin/salloc" +, srun ? "/usr/bin/srun" +, ... +}: drv: let ppnReified = if drv.multicore then ppn else 1; -in overrideDerivation drv ({ args, builder, name, ... }: { +in +overrideDerivation drv ({ args, builder, name, ... }: { builder = stdenv.shell; - args = let - slurm = writeScript "slurm" '' - #!${stdenv.shell} - NIX_BUILD_CORES=${toString ppnReified} + args = + let + slurm = writeScript "slurm" '' + #!${stdenv.shell} + NIX_BUILD_CORES=${toString ppnReified} - ${salloc} -c $NIX_BUILD_CORES --mem=${toString mem}G -t ${walltime} \ - -J "${name}" \ - ${optionalString (partition != null) "-p ${partition}"} \ - ${optionalString (slurmFlags != null) slurmFlags} \ - ${srun} ${builder} ${concatMapStringsSep " " escape args} - ''; + ${salloc} -c $NIX_BUILD_CORES --mem=${toString mem}G -t ${walltime} \ + -J "${name}" \ + ${optionalString (partition != null) "-p ${partition}"} \ + ${optionalString (slurmFlags != null) slurmFlags} \ + ${srun} ${builder} ${concatMapStringsSep " " escape args} + ''; - in [ "-c" slurm ]; + in + [ "-c" slurm ]; }) diff --git a/lib/types.nix b/lib/types.nix index fbc7513..7e58ce0 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; with lib; @@ -25,30 +25,30 @@ rec { matchFiletype = sym: y: x: if x ? filetype then matchFiletype' sym y x.filetype else abort "unknown filetype for ${sym}"; matchFiletype' = sym: y: x: match x (defError (idft sym) y filetype); filetype = make-type "filetype" { - fa = {}; - fq = {}; - bam = {ref = any; sorting = sort;}; - sam = {ref = any; sorting = sort;}; - cram = {ref = any; sorting = sort;}; - vcf = {ref = any;}; - bed = {ref = any;}; + fa = { }; + fq = { }; + bam = { ref = any; sorting = sort; }; + sam = { ref = any; sorting = sort; }; + cram = { ref = any; sorting = sort; }; + vcf = { ref = any; }; + bed = { ref = any; }; gz = filetype; bz2 = filetype; }; - toCram = matchFiletype "bam2cram" { bam = filetype.cram; sam = filetype.cram; cram = filetype.cram; }; - toBam = matchFiletype "bam2cram" { bam = filetype.bam; sam = filetype.bam; cram = filetype.bam; }; - toSam = matchFiletype "bam2cram" { bam = filetype.sam; sam = filetype.sam; cram = filetype.sam; }; + toCram = matchFiletype "bam2cram" { bam = filetype.cram; sam = filetype.cram; inherit (filetype) cram; }; + toBam = matchFiletype "bam2cram" { inherit (filetype) bam; sam = filetype.bam; cram = filetype.bam; }; + toSam = matchFiletype "bam2cram" { bam = filetype.sam; inherit (filetype) sam; cram = filetype.sam; }; matchSorting = sym: y: x: match x.sorting (defError (idst sym) y sort); matchFileSorting = sym: y: let f = matchSorting sym y; in matchFiletype sym { bam = f; sam = f; cram = f; }; sort = make-type "sort" { - none = {}; - coord = {}; - name = {}; + none = { }; + coord = { }; + name = { }; }; - coordSort = f: matchFiletype "coordSort" { bam = x: filetype.bam (x // {sorting = sort.coord {};}); } {filetype = f;}; - nameSort = f: matchFiletype "nameSort" { bam = x: filetype.bam (x // {sorting = sort.name {};}); } {filetype = f;}; + coordSort = f: matchFiletype "coordSort" { bam = x: filetype.bam (x // { sorting = sort.coord { }; }); } { filetype = f; }; + nameSort = f: matchFiletype "nameSort" { bam = x: filetype.bam (x // { sorting = sort.name { }; }); } { filetype = f; }; gunzip = matchFiletype "gunzip" { gz = x: x; }; bunzip2 = matchFiletype "bunzip2" { bz2 = x: x; }; diff --git a/test-tnpair.nix b/test-tnpair.nix index 6eb2e23..9ba0e51 100644 --- a/test-tnpair.nix +++ b/test-tnpair.nix @@ -1,4 +1,4 @@ -with import {}; +with import { }; with lib; let @@ -6,28 +6,32 @@ let name = baseNameOf path; buildCommand = "ln -s ${path} $out"; }; - fetchfq = attrs: types.tagFiletype (types.filetype.fq {}) (fetchlocal attrs); - fetchfa = attrs: types.tagFiletype (types.filetype.fa {}) (fetchlocal attrs); + fetchfq = attrs: types.tagFiletype (types.filetype.fq { }) (fetchlocal attrs); + fetchfa = attrs: types.tagFiletype (types.filetype.fa { }) (fetchlocal attrs); ref = fetchfa ./examples/ref.fa; - alignWithRG = rg: x: nameSort (bwa.align { inherit ref; flags = "-R'@RG\\tID:${rg}\\tSM:${rg}'";} x); - sort = samtools.sort {}; - nameSort = samtools.sort {nameSort = true;}; - flagstat = samtools.flagstat {}; - check-fastqc = fastqc.check {}; - check-fastp = fastp.check {}; - callVariants = strelka.callSomatic {}; - markdup = samtools.markdup {}; - fixmate = samtools.fixmate {}; + alignWithRG = rg: x: nameSort (bwa.align { inherit ref; flags = "-R'@RG\\tID:${rg}\\tSM:${rg}'"; } x); + sort = sambamba.sort { }; + nameSort = sambamba.sort { nameSort = true; }; + flagstat = samtools.flagstat { }; + check-fastqc = fastqc.check { }; + check-fastp = fastp.check { }; + callVariants = strelka.callSomatic { }; + markdup = samtools.markdup { }; + fixmate = samtools.fixmate { }; tnpair = { - tumour = {name = "mysample1"; files = { + tumour = { + name = "mysample1"; + files = { input1 = fetchfq ./examples/sample1-1.fq; input2 = fetchfq ./examples/sample1-2.fq; }; }; - normal = {name = "mysample2"; files = { + normal = { + name = "mysample2"; + files = { input1 = fetchfq ./examples/sample2-1.fq; input2 = fetchfq ./examples/sample2-2.fq; }; @@ -37,36 +41,36 @@ let processPair = { tumour, normal }: rec { alignments = mapAttrs (_: x: markdup (sort (fixmate (alignWithRG x.name x.files)))) { inherit normal tumour; }; variants = callVariants alignments; - octopusSomatic = octopus.callSomatic {} {inherit (alignments) normal; tumours = [ alignments.tumour ];}; - glvariants = strelka.call {} (builtins.attrValues alignments); - platypusVars = platypus.call {} (builtins.attrValues alignments); - octopusVars = octopus.call {} (builtins.attrValues alignments); - shards = map (x: nameSort (bwa.align {inherit ref;} x)) (shard.fastQPair 2 normal.files); + octopusSomatic = octopus.callSomatic { } { inherit (alignments) normal; tumours = [ alignments.tumour ]; }; + glvariants = strelka.call { } (builtins.attrValues alignments); + platypusVars = platypus.call { } (builtins.attrValues alignments); + octopusVars = octopus.call { } (builtins.attrValues alignments); + shards = map (x: nameSort (bwa.align { inherit ref; } x)) (shard.fastQPair 2 normal.files); }; tnpairResult = processPair tnpair; cnvkitResults = rec { - cnvs = cnvkit.callCNV {} (with tnpairResult.alignments; { normals = [ normal ]; tumours = [ tumour ];}); - plot = cnvkit.scatterPlot {} cnvs.out1; + cnvs = cnvkit.callCNV { } (with tnpairResult.alignments; { normals = [ normal ]; tumours = [ tumour ]; }); + plot = cnvkit.scatterPlot { } cnvs.out1; }; alignments = { - "bowtie-normal.bam" = bowtie.align {inherit ref;} tnpair.normal.files; - "last.maf" = lastal.align {inherit ref;} tnpair.normal.files; - "whisper.bam" = whisper.align {inherit ref;} tnpair.normal.files; - "bwa-mem.bam" = bwa.mem {inherit ref;} tnpair.normal.files; - "bwa-mem2.bam" = bwa.mem2 {inherit ref;} tnpair.normal.files; - "minimap2-normal.bam" = minimap2.align {inherit ref; preset = "sr"; } tnpair.normal.files; - "snap-normal.bam" = snap.align {inherit ref; } tnpair.normal.files; + "bowtie-normal.bam" = bowtie.align { inherit ref; } tnpair.normal.files; + "last.maf" = lastal.align { inherit ref; } tnpair.normal.files; + "whisper.bam" = whisper.align { inherit ref; } tnpair.normal.files; + "bwa-mem.bam" = bwa.mem { inherit ref; } tnpair.normal.files; + "bwa-mem2.bam" = bwa.mem2 { inherit ref; } tnpair.normal.files; + "minimap2-normal.bam" = minimap2.align { inherit ref; preset = "sr"; } tnpair.normal.files; + "snap-normal.bam" = snap.align { inherit ref; } tnpair.normal.files; "${tnpair.tumour.name}.flagstat" = flagstat tnpairResult.alignments.tumour; - "hisat2-normal.bam" = hisat2.align {inherit ref;} tnpair.normal.files; + "hisat2-normal.bam" = hisat2.align { inherit ref; } tnpair.normal.files; }; testNaming = linkOutputs { - crai = samtools.index {} (samtools.view { outfmt = types.toCram; } tnpairResult.alignments.tumour); - kallisto = kallisto.quant {inherit ref;} (attrValues tnpair.tumour.files); - facets = facets.callCNV {} {vcf = tnpairResult.platypusVars; bams = with tnpairResult.alignments; [ normal tumour ];}; + crai = samtools.index { } (samtools.view { outfmt = types.toCram; } tnpairResult.alignments.tumour); + kallisto = kallisto.quant { inherit ref; } (attrValues tnpair.tumour.files); + facets = facets.callCNV { } { vcf = tnpairResult.platypusVars; bams = with tnpairResult.alignments; [ normal tumour ]; }; cnvkit = cnvkitResults.cnvs; "cnvkit.pdf" = cnvkitResults.plot; "octopus.vcf" = tnpairResult.octopusVars; @@ -74,24 +78,25 @@ let strelka-indels = tnpairResult.variants.indels; "strelka.snvs.vcf" = tnpairResult.variants.snvs; "strelka.gl.vcf" = tnpairResult.glvariants; - delly = delly.call {} (with tnpairResult.alignments; [normal tumour]); - lumpy = lumpy.call {} (with tnpairResult.alignments; [normal tumour]); - manta = manta.call {} (with tnpairResult.alignments; {normals = [normal tumour]; }); - mantaTN = manta.call {} (with tnpairResult.alignments; {normals = [normal]; tumour = tumour;}); - mantaT = manta.call {} (with tnpairResult.alignments; {tumour = tumour;}); - gridss = gridss.callVariants {} (with tnpairResult.alignments; [normal tumour]); - gridss2 = gridss.call (with tnpairResult.alignments; [normal tumour]); - gridss3 = gridss.callAndAssemble (with tnpairResult.alignments; [normal tumour]); - "merged-shards.bam" = samtools.merge {} tnpairResult.shards; - "merged.bam" = samtools.merge {} [tnpairResult.alignments.tumour tnpairResult.alignments.normal]; - "merged-namesorted.bam" = samtools.merge {} [(nameSort tnpairResult.alignments.tumour) (nameSort tnpairResult.alignments.normal)]; - "${tnpair.tumour.name}.cram" = samtools.view { outfmt = types.toCram; } (tnpairResult.alignments.tumour); + delly = delly.call { } (with tnpairResult.alignments; [ normal tumour ]); + lumpy = lumpy.call { } (with tnpairResult.alignments; [ normal tumour ]); + manta = manta.call { } (with tnpairResult.alignments; { normals = [ normal tumour ]; }); + mantaTN = manta.call { } (with tnpairResult.alignments; { normals = [ normal ]; inherit tumour; }); + mantaT = manta.call { } (with tnpairResult.alignments; { inherit tumour; }); + gridss = gridss.callVariants { } (with tnpairResult.alignments; [ normal tumour ]); + gridss2 = gridss.call (with tnpairResult.alignments; [ normal tumour ]); + gridss3 = gridss.callAndAssemble (with tnpairResult.alignments; [ normal tumour ]); + "merged-shards.bam" = samtools.merge { } tnpairResult.shards; + "merged.bam" = samtools.merge { } [ tnpairResult.alignments.tumour tnpairResult.alignments.normal ]; + "merged-namesorted.bam" = samtools.merge { } [ (nameSort tnpairResult.alignments.tumour) (nameSort tnpairResult.alignments.normal) ]; + "${tnpair.tumour.name}.cram" = samtools.view { outfmt = types.toCram; } tnpairResult.alignments.tumour; "${tnpair.tumour.name}.fastqc.1" = check-fastqc tnpair.tumour.files.input1; "${tnpair.tumour.name}.fastp" = check-fastp tnpair.tumour.files; - snver = snver.call { ploidy=1; } tnpairResult.alignments; - mosdepth = mosdepth.plot {} { inputs = mapAttrsToList (_: mosdepth.depth {}) tnpairResult.alignments; names = [ "seq1" "seq2" ]; }; - xenomapper = xenomapper.allocate {} { primary = nameSort tnpairResult.alignments.tumour; secondary = nameSort tnpairResult.alignments.tumour; }; + snver = snver.call { ploidy = 1; } tnpairResult.alignments; + mosdepth = mosdepth.plot { } { inputs = mapAttrsToList (_: mosdepth.depth { }) tnpairResult.alignments; names = [ "seq1" "seq2" ]; }; + xenomapper = xenomapper.allocate { } { primary = nameSort tnpairResult.alignments.tumour; secondary = nameSort tnpairResult.alignments.tumour; }; inherit alignments; }; -in testNaming +in +testNaming diff --git a/tools/ascat-app.nix b/tools/ascat-app.nix index 51aa677..bd6d538 100644 --- a/tools/ascat-app.nix +++ b/tools/ascat-app.nix @@ -1,4 +1,4 @@ -{stdenv, callPackage, buildPerlPackage, fetchurl, fetchFromGitHub, perlPackages, R, bwa, samtools, pkgconfig, zlib, htslib, curl, bzip2, lzma, gnutls, nettle, gmp, p11-kit, libtasn1, perl, psmisc, time, vcftools, rWrapper, rPackages}: +{ stdenv, callPackage, buildPerlPackage, fetchurl, fetchFromGitHub, perlPackages, R, bwa, samtools, pkgconfig, zlib, htslib, curl, bzip2, lzma, gnutls, nettle, gmp, p11-kit, libtasn1, perl, psmisc, time, vcftools, rWrapper, rPackages }: let ascat = fetchurl { @@ -98,7 +98,7 @@ let url = "mirror://cpan/authors/id/C/CD/CDRAUG/${name}.tar.gz"; sha256 = "0yvhgifs8g9rwdcq84zw4b005nq2jml6c75zgjscv6d2pd3lj1ss"; }; - propagatedBuildInputs = with perlPackages; [DBI DataStag Error GD Graph HTTPMessage HTTPMessage IOstringy IOString IPCRun LWP ListMoreUtils SetScalar TestMost TestRequiresInternet URI XMLDOM XMLDOMXPath XMLLibXML XMLLibXML libxml_perl XMLSAX XMLSAXBase XMLSAXWriter XMLTwig XMLWriter YAML DBFile ]; + propagatedBuildInputs = with perlPackages; [ DBI DataStag Error GD Graph HTTPMessage HTTPMessage IOstringy IOString IPCRun LWP ListMoreUtils SetScalar TestMost TestRequiresInternet URI XMLDOM XMLDOMXPath XMLLibXML XMLLibXML libxml_perl XMLSAX XMLSAXBase XMLSAXWriter XMLTwig XMLWriter YAML DBFile ]; nativeBuildInputs = with perlPackages; [ TestException TestWarn TestDifferences TestDeep ]; }; @@ -108,7 +108,7 @@ let url = "mirror://cpan/authors/id/C/CM/CMUNGALL/${name}.tar.gz"; sha256 = "0ncf4l39ka23nb01jlm6rzxdb5pqbip01x0m38bnvf1gim825caa"; }; - propagatedBuildInputs = with perlPackages; [IOString Graph XMLLibXSLT ]; + propagatedBuildInputs = with perlPackages; [ IOString Graph XMLLibXSLT ]; }; XMLDOMXPath = buildPerlPackage rec { @@ -117,7 +117,7 @@ let url = "mirror://cpan/authors/id/M/MI/MIROD/${name}.tar.gz"; sha256 = "1si9m1pqih3ibbd6jnw69fh98dd4krxpx90p65x9j4aja55afwq1"; }; - propagatedBuildInputs= with perlPackages; [XMLDOM XMLXPathEngine]; + propagatedBuildInputs = with perlPackages; [ XMLDOM XMLXPathEngine ]; doCheck = false; }; @@ -215,10 +215,10 @@ let name = "alleleCount.pl-${version}"; version = "4.0.1"; src = fetchFromGitHub { - owner = "cancerit"; - repo = "alleleCount"; - rev = "v${version}"; - sha256 = "0nkwnjqglgshzhlmz1r0khdjai9mfz4ih8bzrzg0g18d1725k6gp"; + owner = "cancerit"; + repo = "alleleCount"; + rev = "v${version}"; + sha256 = "0nkwnjqglgshzhlmz1r0khdjai9mfz4ih8bzrzg0g18d1725k6gp"; }; preConfigure = '' cd perl @@ -266,4 +266,5 @@ let ]; }; -in ascatNGS +in +ascatNGS diff --git a/tools/ascat-callCNV.nix b/tools/ascat-callCNV.nix index f74bb18..aa1a8ec 100644 --- a/tools/ascat-callCNV.nix +++ b/tools/ascat-callCNV.nix @@ -1,12 +1,12 @@ { bionix , ref , gc -, indexAttrs ? {} -, bamIndexAttrs ? {} +, indexAttrs ? { } +, bamIndexAttrs ? { } , flags ? null }: -{tumour, normal, gender}: +{ tumour, normal, gender }: with bionix; with lib; diff --git a/tools/ascat.nix b/tools/ascat.nix index 700cb4e..142e0d9 100644 --- a/tools/ascat.nix +++ b/tools/ascat.nix @@ -1,17 +1,17 @@ -{bionix}: +{ bionix }: with bionix; { - app = pkgs.callPackage ./ascat-app.nix {}; + app = pkgs.callPackage ./ascat-app.nix { }; /* Generate GC correction file for ascatNGS. - Type: gccorrect :: {ref :: fasta, chrPrefix :: string, ...} -> (snps :: VCF) -> (gc :: GC) + Type: gccorrect :: {ref :: fasta, chrPrefix :: string, ...} -> (snps :: VCF) -> (gc :: GC) */ gccorrect = callBionixE ./ascat-gccorrect.nix; /* Call CNVs using ascatNGS. Gender is a string as per ascatNGS docs (e.g., "XX"). - Type: callCNV :: {ref :: fasta, gc :: GC, ...} -> {tumour :: bam, normal :: bam, gender :: string} -> CNVs + Type: callCNV :: {ref :: fasta, gc :: GC, ...} -> {tumour :: bam, normal :: bam, gender :: string} -> CNVs */ callCNV = callBionixE ./ascat-callCNV.nix; } diff --git a/tools/bowtie-align.nix b/tools/bowtie-align.nix index c83c49c..8df3dba 100644 --- a/tools/bowtie-align.nix +++ b/tools/bowtie-align.nix @@ -2,7 +2,7 @@ , ref , bamOutput ? true , flags ? null -, indexAttrs ? {} +, indexAttrs ? { } }: { input1 @@ -18,7 +18,8 @@ let fa = f: matchFiletype "bowtie2-ref" { fa = _: f; } f; fq = f: matchFiletype "bowtie2-input" { fq = _: f; gz = matchFiletype' "bowtie2-input" { fq = _: f; }; } f; -in stage { +in +stage { name = "bowtie2-align"; buildInputs = with pkgs; [ bowtie2 bc samtools ]; buildCommand = '' @@ -32,6 +33,6 @@ in stage { ${optionalString bamOutput "| samtools view -b"} \ > $out ''; - passthru.filetype = if bamOutput then filetype.bam {ref = ref; sorting = sort.none {};} else filetype.sam {ref = ref; sorting = sort.name {};}; + passthru.filetype = if bamOutput then filetype.bam { inherit ref; sorting = sort.none { }; } else filetype.sam { inherit ref; sorting = sort.name { }; }; passthru.multicore = true; } diff --git a/tools/bowtie.nix b/tools/bowtie.nix index df97d36..2208dff 100644 --- a/tools/bowtie.nix +++ b/tools/bowtie.nix @@ -4,12 +4,12 @@ with bionix; { /* Align a sequence against a reference - Type: align :: {ref :: fasta, bamOutput :: bool, ...} -> {input1 :: fastq, input2 :: fastq} -> bam/sam + Type: align :: {ref :: fasta, bamOutput :: bool, ...} -> {input1 :: fastq, input2 :: fastq} -> bam/sam */ align = callBionixE ./bowtie-align.nix; /* Create a Bowtie index - Type: index :: {seed :: int, ...} -> fasta -> index + Type: index :: {seed :: int, ...} -> fasta -> index */ index = callBionixE ./bowtie-index.nix; } diff --git a/tools/bwa-mem.nix b/tools/bwa-mem.nix index 31c9487..52e73d1 100644 --- a/tools/bwa-mem.nix +++ b/tools/bwa-mem.nix @@ -2,7 +2,7 @@ , ref , bamOutput ? true , flags ? null -, indexAttrs ? {} +, indexAttrs ? { } }: { input1 @@ -18,7 +18,8 @@ let fa = f: matchFiletype "bwa-ref" { fa = _: f; } f; fq = f: matchFiletype "bwa-input" { fq = _: f; gz = matchFiletype' "bwa-input" { fq = _: f; }; } f; -in stage { +in +stage { name = "bwa-mem"; buildInputs = with pkgs; [ bwa bc ] ++ optional bamOutput samtools; buildCommand = '' @@ -35,6 +36,6 @@ in stage { ${optionalString bamOutput "| samtools view -b"} \ > $out ''; - passthru.filetype = if bamOutput then filetype.bam {ref = ref; sorting = sort.none {};} else filetype.sam {ref = ref; sorting = sort.name {};}; + passthru.filetype = if bamOutput then filetype.bam { inherit ref; sorting = sort.none { }; } else filetype.sam { inherit ref; sorting = sort.name { }; }; passthru.multicore = true; } diff --git a/tools/bwa-mem2-app.nix b/tools/bwa-mem2-app.nix index 49f4af3..6ee2ab4 100644 --- a/tools/bwa-mem2-app.nix +++ b/tools/bwa-mem2-app.nix @@ -1,5 +1,5 @@ { clangStdenv, fetchFromGitHub, zlib }: - clangStdenv.mkDerivation rec { +clangStdenv.mkDerivation rec { pname = "bwa-mem2"; version = "2.1"; src = fetchFromGitHub { diff --git a/tools/bwa-mem2.nix b/tools/bwa-mem2.nix index b16b7ea..843c5b0 100644 --- a/tools/bwa-mem2.nix +++ b/tools/bwa-mem2.nix @@ -2,7 +2,7 @@ , ref , bamOutput ? true , flags ? null -, indexAttrs ? {} +, indexAttrs ? { } }: { input1 @@ -18,7 +18,8 @@ let fa = f: matchFiletype "bwa-ref" { fa = _: f; } f; fq = f: matchFiletype "bwa-input" { fq = _: f; gz = matchFiletype' "bwa-input" { fq = _: f; }; } f; -in stage { +in +stage { name = "bwa-mem2"; buildInputs = with pkgs; [ bionix.bwa.app2 bc ] ++ optional bamOutput samtools; buildCommand = '' @@ -35,6 +36,6 @@ in stage { ${optionalString bamOutput "| samtools view -b"} \ > $out ''; - passthru.filetype = if bamOutput then filetype.bam {ref = ref; sorting = sort.none {};} else filetype.sam {ref = ref; sorting = sort.name {};}; + passthru.filetype = if bamOutput then filetype.bam { inherit ref; sorting = sort.none { }; } else filetype.sam { inherit ref; sorting = sort.name { }; }; passthru.multicore = true; } diff --git a/tools/bwa.nix b/tools/bwa.nix index 90a8bd3..6fb7d71 100644 --- a/tools/bwa.nix +++ b/tools/bwa.nix @@ -3,18 +3,18 @@ with bionix; rec { - app2 = pkgs.callPackage ./bwa-mem2-app.nix {}; + app2 = pkgs.callPackage ./bwa-mem2-app.nix { }; /* Align read against a reference: defaults to bwa-mem */ align = bionix.bwa.mem; /* Align reads against a reference using bwa-mem - Type: bwa-mem :: {ref :: fasta, bamOutput :: bool, ...} -> {input1, input2} -> bam/sam + Type: bwa-mem :: {ref :: fasta, bamOutput :: bool, ...} -> {input1, input2} -> bam/sam */ mem = callBionixE ./bwa-mem.nix; mem2 = callBionixE ./bwa-mem2.nix; /* Creates an reference index for BWA - Type: index :: {...} -> fasta -> BWA index + Type: index :: {...} -> fasta -> BWA index */ index = callBionixE ./bwa-index.nix; index2 = callBionixE ./bwa-index2.nix; diff --git a/tools/cnvkit-batch.nix b/tools/cnvkit-batch.nix index 113e0c9..7f7893f 100644 --- a/tools/cnvkit-batch.nix +++ b/tools/cnvkit-batch.nix @@ -1,17 +1,18 @@ -{bionix -,targets ? null -,annotations ? null -,flags ? null -,indexAttrs ? {}}: +{ bionix +, targets ? null +, annotations ? null +, flags ? null +, indexAttrs ? { } +}: -{normals ? [], tumours}: +{ normals ? [ ], tumours }: with bionix; with lib; with types; let - getref = f: matchFiletype "cnvkit-batch" { bam = {ref, ...}: ref; } f; + getref = matchFiletype "cnvkit-batch" { bam = { ref, ... }: ref; }; refs = map getref normals ++ map getref tumours; ref = head refs; sorted = matchFileSorting "cnvkit-batch" { coord = _: true; }; diff --git a/tools/cnvkit-scatter.nix b/tools/cnvkit-scatter.nix index 2899584..6cd5b96 100644 --- a/tools/cnvkit-scatter.nix +++ b/tools/cnvkit-scatter.nix @@ -1,5 +1,6 @@ -{bionix -,flags ? null}: +{ bionix +, flags ? null +}: input: diff --git a/tools/cnvkit.nix b/tools/cnvkit.nix index fc55d94..812652a 100644 --- a/tools/cnvkit.nix +++ b/tools/cnvkit.nix @@ -1,15 +1,15 @@ -{bionix}: +{ bionix }: with bionix; { /* Call CNVs - Type: callCNV :: {targets :: target file, annotations :: annotation file, ...} -> {normals :: [bam], tumours :: [bam]} -> CNVs + Type: callCNV :: {targets :: target file, annotations :: annotation file, ...} -> {normals :: [bam], tumours :: [bam]} -> CNVs */ callCNV = callBionixE ./cnvkit-batch.nix; /* Scatter plot from CNV calls - Type: scatterPlot :: {} -> CNVs -> PDF + Type: scatterPlot :: {} -> CNVs -> PDF */ scatterPlot = callBionixE ./cnvkit-scatter.nix; } diff --git a/tools/compression-bunzip2.nix b/tools/compression-bunzip2.nix index 06fe682..a36b66d 100644 --- a/tools/compression-bunzip2.nix +++ b/tools/compression-bunzip2.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; diff --git a/tools/compression-bzip2.nix b/tools/compression-bzip2.nix index 6dfb749..5332e7d 100644 --- a/tools/compression-bzip2.nix +++ b/tools/compression-bzip2.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; diff --git a/tools/compression-gunzip.nix b/tools/compression-gunzip.nix index 6c98f91..96a4e49 100644 --- a/tools/compression-gunzip.nix +++ b/tools/compression-gunzip.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; diff --git a/tools/compression-gzip.nix b/tools/compression-gzip.nix index 50fca9e..0229b94 100644 --- a/tools/compression-gzip.nix +++ b/tools/compression-gzip.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; diff --git a/tools/compression.nix b/tools/compression.nix index bfcc292..c9c8777 100644 --- a/tools/compression.nix +++ b/tools/compression.nix @@ -1,4 +1,4 @@ -{bionix}: +{ bionix }: with bionix; with types; @@ -8,42 +8,51 @@ let gunzip = callBionixE ./compression-gunzip.nix; bunzip2 = callBionixE ./compression-bunzip2.nix; -in { - uncompress = attrs: f: matchFiletype "uncompress" { - fa = _: f; - fq = _: f; - bam = _: f; - sam = _: f; - cram = _: f; - vcf = _: f; - bed = _: f; - gz = _: gunzip attrs f; - bz2 = _: bunzip2 attrs f; - } f; +in +{ + uncompress = attrs: f: matchFiletype "uncompress" + { + fa = _: f; + fq = _: f; + bam = _: f; + sam = _: f; + cram = _: f; + vcf = _: f; + bed = _: f; + gz = _: gunzip attrs f; + bz2 = _: bunzip2 attrs f; + } + f; gzip = attrs: f: let gz = gzip attrs f; - in types.matchFiletype "compressed" { - fa = _: gz; - fq = _: gz; - bam = _: gz; - sam = _: gz; - cram = _: gz; - vcf = _: gz; - bed = _: gz; - gz = x: x; - } f; + in + types.matchFiletype "compressed" + { + fa = _: gz; + fq = _: gz; + bam = _: gz; + sam = _: gz; + cram = _: gz; + vcf = _: gz; + bed = _: gz; + gz = x: x; + } + f; bzip2 = attrs: f: let bz2 = bzip2 attrs f; - in types.matchFiletype "compressed" { - fa = _: gz; - fq = _: gz; - bam = _: gz; - sam = _: gz; - cram = _: gz; - vcf = _: gz; - bed = _: gz; - bz2 = x: x; - } f; + in + types.matchFiletype "compressed" + { + fa = _: gz; + fq = _: gz; + bam = _: gz; + sam = _: gz; + cram = _: gz; + vcf = _: gz; + bed = _: gz; + bz2 = x: x; + } + f; } diff --git a/tools/crumble-app.nix b/tools/crumble-app.nix index fc30d5a..d8ed1a7 100644 --- a/tools/crumble-app.nix +++ b/tools/crumble-app.nix @@ -1,4 +1,4 @@ -{stdenv, fetchFromGitHub, autoreconfHook, htslib}: +{ stdenv, fetchFromGitHub, autoreconfHook, htslib }: stdenv.mkDerivation rec { name = "crumble-${version}"; diff --git a/tools/crumble.nix b/tools/crumble.nix index 4256f37..dfb4e31 100644 --- a/tools/crumble.nix +++ b/tools/crumble.nix @@ -3,5 +3,5 @@ with bionix; { - crumble = pkgs.callPackage ./crumble-app.nix {}; + crumble = pkgs.callPackage ./crumble-app.nix { }; } diff --git a/tools/delly-call.nix b/tools/delly-call.nix index 5378810..80084be 100644 --- a/tools/delly-call.nix +++ b/tools/delly-call.nix @@ -1,4 +1,4 @@ -{ bionix, flags ? "", faidxAttrs ? {}, indexAttrs ? {}}: +{ bionix, flags ? "", faidxAttrs ? { }, indexAttrs ? { } }: inputs: @@ -41,6 +41,6 @@ stage { ''; passthru = { multicore = true; - filetype = filetype.vcf { ref = ref; }; + filetype = filetype.vcf { inherit ref; }; }; } diff --git a/tools/delly.nix b/tools/delly.nix index d94457c..22d4d89 100644 --- a/tools/delly.nix +++ b/tools/delly.nix @@ -4,7 +4,7 @@ with bionix; { /* Call structural variants - Type: call { ... } -> [bam] -> vcf + Type: call { ... } -> [bam] -> vcf */ call = callBionixE ./delly-call.nix; } diff --git a/tools/facets-app.nix b/tools/facets-app.nix index 6553a67..fe2a31b 100644 --- a/tools/facets-app.nix +++ b/tools/facets-app.nix @@ -1,8 +1,9 @@ -{buildRPackage -,fetchFromGitHub -,R -,htslib -,zlib}: +{ buildRPackage +, fetchFromGitHub +, R +, htslib +, zlib +}: let pctGCdata = buildRPackage rec { @@ -18,7 +19,8 @@ let buildInputs = [ R ]; }; -in buildRPackage rec{ +in +buildRPackage rec{ name = "facets-${version}"; version = "0.6.1"; requireX = false; diff --git a/tools/facets-call.nix b/tools/facets-call.nix index a736627..43c4c7f 100644 --- a/tools/facets-call.nix +++ b/tools/facets-call.nix @@ -1,6 +1,6 @@ -{bionix}: +{ bionix }: -{vcf, bams}: +{ vcf, bams }: with bionix; with types; diff --git a/tools/facets.nix b/tools/facets.nix index 14110a9..b91e422 100644 --- a/tools/facets.nix +++ b/tools/facets.nix @@ -1,12 +1,12 @@ -{bionix}: +{ bionix }: with bionix; { - app = lib.callPackageWith (pkgs // pkgs.rPackages) ./facets-app.nix {}; + app = lib.callPackageWith (pkgs // pkgs.rPackages) ./facets-app.nix { }; /* Call CNVs - Type: callCnv :: {...} -> {vcf, bams :: [bams]} -> CNVs + Type: callCnv :: {...} -> {vcf, bams :: [bams]} -> CNVs */ callCNV = callBionixE ./facets-call.nix; } diff --git a/tools/fastp-check.nix b/tools/fastp-check.nix index 06f00ae..9467cd9 100644 --- a/tools/fastp-check.nix +++ b/tools/fastp-check.nix @@ -1,10 +1,10 @@ { bionix , flags ? null -} : +}: { input1 , input2 ? null -} : +}: with bionix; with lib; @@ -15,26 +15,27 @@ let out = stage { - name = "fastp"; - buildInputs = [ pkgs.fastp ]; - outputs = [ "out" "fastq1" "json" ] ++ (if input2 != null then [ "fastq2" ] else []); - buildCommand = '' - mkdir -p $out - fastp \ - ${optionalString (flags != null) flags} \ - -i ${fq input1} \ - -o fastq1.fq.gz \ - ${optionalString (input2 != null) '' - -I ${fq input2} \ - -O fastq2.fq.gz \ + name = "fastp"; + buildInputs = [ pkgs.fastp ]; + outputs = [ "out" "fastq1" "json" ] ++ (if input2 != null then [ "fastq2" ] else [ ]); + buildCommand = '' + mkdir -p $out + fastp \ +