From be187bfeb074e5c2b65f25f3dcdb82a98b15570d Mon Sep 17 00:00:00 2001 From: Justin Bedo Date: Thu, 4 Jul 2019 14:26:52 +1000 Subject: linkOutputs: refactor output linking expression The existing linkDrv only has a singular output, making it difficult to separate the various types of output. The new functionality replaces the old with attribute sets defining the outputs. The old linkDrv is deprecated. --- default.nix | 24 ++++++++--------- examples/ex-tnpair/tnpair.nix | 10 +++---- test-tnpair.nix | 63 +++++++++++++++++++++---------------------- 3 files changed, 48 insertions(+), 49 deletions(-) diff --git a/default.nix b/default.nix index 3077470..36dbe44 100644 --- a/default.nix +++ b/default.nix @@ -51,19 +51,19 @@ let def = f: defs: attrs: f (defs // attrs); pipe = let g = fs: with builtins; let h = head fs; t = tail fs; in if t != [] then x: (g t (h x)) else h; in g; - link = {src, dst}: '' - d=$(dirname ${dst}) - if [ ! -e $out/$d ] ; then - mkdir -p $out/$d - fi - ln -s ${src} $out/${dst} - ''; - mkLinks = nixpkgs.lib.concatMapStringsSep "\n" link; - linkDrv = x: nixpkgs.stdenvNoCC.mkDerivation { - name = "link"; - buildCommand = mkLinks x; + linkOutputs = x: with lib; nixpkgs.stdenvNoCC.mkDerivation { + name = "link-outputs"; + outputs = [ "out" ] ++ attrNames x; + nativeBuildInputs = [ pkgs.perl ]; + buildCommand = let + recurse = x: if x ? type && x.type == "derivation" then x else + if builtins.typeOf x == "set" then linkOutputs x + else error "linkOutputs: unsupported type"; + link = dst: src: '' + ln -s ${recurse src} $(perl -e 'print $ENV{"${dst}"}') ; ln -s ${recurse src} $out/${dst} + ''; + in "mkdir $out \n" + (concatStringsSep "\n" (mapAttrsToList link x)); }; - ln = x: y: { src = x; dst = y; }; # Fetching files of specific type fetchFastQ = attrs: with types; tagFiletype (filetype.fq {}) (fetchurl attrs); diff --git a/examples/ex-tnpair/tnpair.nix b/examples/ex-tnpair/tnpair.nix index c42fe95..ada96c8 100644 --- a/examples/ex-tnpair/tnpair.nix +++ b/examples/ex-tnpair/tnpair.nix @@ -19,8 +19,8 @@ let (samtools.markdup {}) ]; -in linkDrv [ - (ln (strelka.callSomatic {} {normal = preprocess normal; tumour = preprocess tumour;}) "strelka") - (ln (preprocess normal) "normal.bam") - (ln (preprocess tumour) "tumour.bam") -] +in linkOutputs { + strelka = strelka.callSomatic {} {normal = preprocess normal; tumour = preprocess tumour;}; + "normal.bam" = preprocess normal; + "tumour.bam" = preprocess tumour; +} diff --git a/test-tnpair.nix b/test-tnpair.nix index f90e3a9..afbdf68 100644 --- a/test-tnpair.nix +++ b/test-tnpair.nix @@ -54,37 +54,36 @@ let plot = cnvkit.scatterPlot {} cnvs; }; - testNaming = linkDrv [ - (ln (facets.callCNV {} {vcf = tnpairResult.platypusVars; bams = with tnpairResult.alignments; [ normal tumour ];}) "facets") - (ln cnvkitResults.cnvs "cnvkit") - (ln cnvkitResults.plot "cnvkit.pdf") - (ln tnpairResult.octopusVars "octopus.vcf") - (ln tnpairResult.octopusSomatic "octopus-somatic.vcf") - (ln tnpairResult.variants "strelka") - (ln tnpairResult.glvariants "strelka-gl") - (ln tnpairResult.variants.indels "strelka.indels.vcf") - (ln tnpairResult.variants.snvs "strelka.snvs.vcf") - (ln tnpairResult.glvariants.variants "strelka.gl.vcf") - (ln (bowtie.align {inherit ref;} tnpair.normal.files) "alignments/bowtie-normal.bam") - (ln (bwa.mem {inherit ref;} tnpair.normal.files) "alignments/bwa-mem.bam") - (ln (bwa.mem2 {inherit ref;} tnpair.normal.files) "alignments/bwa-mem2.bam") - (ln (minimap2.align {inherit ref; preset = "sr"; } tnpair.normal.files) "alignments/minimap2-normal.bam") - (ln (snap.align {inherit ref; } tnpair.normal.files) "alignments/snap-normal.bam") - (ln (gridss.callVariants {} (with tnpairResult.alignments; [normal tumour])) "gridss") - (ln (gridss.call (with tnpairResult.alignments; [normal tumour])) "gridss2") - (ln (gridss.callAndAssemble (with tnpairResult.alignments; [normal tumour])) "gridss3") - (ln (samtools.merge {} tnpairResult.shards) "alignments/merged-shards.bam") - (ln (samtools.merge {} [tnpairResult.alignments.tumour tnpairResult.alignments.normal]) "alignments/merged.bam") - (ln (samtools.merge {} [(nameSort tnpairResult.alignments.tumour) (nameSort tnpairResult.alignments.normal)]) "alignments/merged-namesorted.bam") - (ln (samtools.view { outfmt = types.toCram; } (tnpairResult.alignments.tumour)) "alignments/${tnpair.tumour.name}.cram") - #(ln (samtools.view { outfmt = types.toCram; } (tnpairResult.alignments.normal)) "alignments/${tnpair.normal.name}.cram") - (ln (flagstat tnpairResult.alignments.tumour) "alignments/${tnpair.tumour.name}.flagstat") - #(ln (flagstat tnpairResult.alignments.normal) "alignments/${tnpair.normal.name}.flagstat") - (ln (check-fastqc tnpair.tumour.files.input1) "fastqc/${tnpair.tumour.name}.1") - #(ln (check-fastqc tnpair.normal.files.input1) "fastqc/${tnpair.normal.name}.1") - #(ln (check-fastqc tnpair.normal.files.input2) "fastqc/${tnpair.normal.name}.2") - #(ln (check-fastqc tnpair.tumour.files.input2) "fastqc/${tnpair.tumour.name}.2") - (ln (check-fastp tnpair.tumour.files) "fastp/${tnpair.tumour.name}") - ]; + alignments = { + "bowtie-normal.bam" = bowtie.align {inherit ref;} tnpair.normal.files; + "bwa-mem.bam" = bwa.mem {inherit ref;} tnpair.normal.files; + "bwa-mem2.bam" = bwa.mem2 {inherit ref;} tnpair.normal.files; + "minimap2-normal.bam" = minimap2.align {inherit ref; preset = "sr"; } tnpair.normal.files; + "snap-normal.bam" = snap.align {inherit ref; } tnpair.normal.files; + "${tnpair.tumour.name}.flagstat" = flagstat tnpairResult.alignments.tumour; + }; + + testNaming = linkOutputs { + facets = facets.callCNV {} {vcf = tnpairResult.platypusVars; bams = with tnpairResult.alignments; [ normal tumour ];}; + cnvkit = cnvkitResults.cnvs; + "cnvkit.pdf" = cnvkitResults.plot; + "octopus.vcf" = tnpairResult.octopusVars; + "octopus-somatic.vcf" = tnpairResult.octopusSomatic; + strelka = tnpairResult.variants; + strelka-gl = tnpairResult.glvariants; + strelka-indels = tnpairResult.variants.indels; + "strelka.snvs.vcf" = tnpairResult.variants.snvs; + "strelka.gl.vcf" = tnpairResult.glvariants.variants; + gridss = gridss.callVariants {} (with tnpairResult.alignments; [normal tumour]); + gridss2 = gridss.call (with tnpairResult.alignments; [normal tumour]); + gridss3 = gridss.callAndAssemble (with tnpairResult.alignments; [normal tumour]); + "merged-shards.bam" = samtools.merge {} tnpairResult.shards; + "merged.bam" = samtools.merge {} [tnpairResult.alignments.tumour tnpairResult.alignments.normal]; + "merged-namesorted.bam" = samtools.merge {} [(nameSort tnpairResult.alignments.tumour) (nameSort tnpairResult.alignments.normal)]; + "${tnpair.tumour.name}.cram" = samtools.view { outfmt = types.toCram; } (tnpairResult.alignments.tumour); + "${tnpair.tumour.name}.fastqc.1" = check-fastqc tnpair.tumour.files.input1; + "${tnpair.tumour.name}.fastp" = check-fastp tnpair.tumour.files; + inherit alignments; + }; in testNaming -- cgit v1.2.3