diff options
author | Justin Bedo <cu@cua0.org> | 2019-01-09 16:38:42 +1100 |
---|---|---|
committer | Justin Bedo <cu@cua0.org> | 2019-01-09 16:38:42 +1100 |
commit | c3babc9eb65dc8c56d07ce8a6ee0d8c03bbd223e (patch) | |
tree | f7d88ec21635664781c7c79fe0ea6f962ae38004 | |
parent | 9fd618d0305d4927c8d86fc37238d1216e401967 (diff) |
large refactor
- use bionix.pkgs instead of nixpkgs
- replace stdenv.mkDerivation with stage
51 files changed, 186 insertions, 267 deletions
diff --git a/default.nix b/default.nix index 6ddcd1b..c50e063 100644 --- a/default.nix +++ b/default.nix @@ -4,7 +4,7 @@ let inherit (nixpkgs) fetchurl callPackage; bionix = nixpkgs.lib.makeExtensible (self: - let callBionix = file: attrs: import file ({ bionix = self; nixpkgs = nixpkgs; } // attrs); + let callBionix = file: attrs: import file ({ bionix = self; } // attrs); in with self; { callBionix = callBionix; id = x: x; @@ -60,8 +60,8 @@ let # Export nixpkgs and standard library lib pkgs = nixpkgs; - lib = nixpkgs.lib; - stage = nixpkgs.stdenvNoCC.mkDerivation; + lib = nixpkgs.lib // { types = types; }; + stage = nixpkgs.stdenv.mkDerivation; # splitting/joining splitFile = file: drv: stage { diff --git a/lib/references.nix b/lib/references.nix index 9d1b947..2e5e937 100755 --- a/lib/references.nix +++ b/lib/references.nix @@ -1,33 +1,33 @@ -{ bionix, nixpkgs }: +{ bionix }: -with nixpkgs; -with bionix.types; +with bionix; +with types; rec { grch38 = grch38-p12; grch38-p12 = rec { - seq = stdenvNoCC.mkDerivation rec { + seq = pkgs.stdenvNoCC.mkDerivation rec { name = "seq-grch38.${version}"; version = "p12"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_28/GRCh38.p12.genome.fa.gz"; sha256 = "0ji2ggpmgnbpwbhq8mirj6h3lyy02nl2rnz7n892iq5cqpsblh4z"; }; buildCommand = "gunzip < $src > $out"; passthru.filetype = filetype.fa {}; }; - blacklist = stdenvNoCC.mkDerivation { + blacklist = pkgs.stdenvNoCC.mkDerivation { name = "blacklist-grch38"; - src = fetchurl { + src = pkgs.fetchurl { url = "http://mitra.stanford.edu/kundaje/akundaje/release/blacklists/hg38-human/hg38.blacklist.bed.gz"; sha256 = "1lpnqq1mjidbdxj5i6x26rxa8x1rs8q3hlf0z1z49j3jsnkgffky"; }; buildCommand = "gunzip < $src > $out"; passthru.filetype = filetype.bed { ref = seq; }; }; - dbsnp = stdenvNoCC.mkDerivation { + dbsnp = pkgs.stdenvNoCC.mkDerivation { name = "dbsnp-b151_GRCh38p7"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606_b151_GRCh38p7/VCF/common_all_20180418.vcf.gz"; sha256 = "0r6m2yrcfw8bbdca515axjls30ssjas6x3qwi5qz07l3prjwmdd4"; }; @@ -37,7 +37,7 @@ rec { ''; passthru.filetype = filetype.vcf { ref = seq; }; }; - cosmic = {coding, noncoding}: stdenvNoCC.mkDerivation rec { + cosmic = {coding, noncoding}: pkgs.stdenvNoCC.mkDerivation rec { name = "cosmic-grch38"; buildInputs = [ gawk ]; buildCommand = '' @@ -47,20 +47,20 @@ rec { passthru.filetype = filetype.vcf { ref = seq; }; }; ensembl = { - cdna = stdenvNoCC.mkDerivation rec { + cdna = pkgs.stdenvNoCC.mkDerivation rec { name = "ensembl-grch38-cdna-${version}"; version = "94"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz"; sha256 = "1fc5d6p2wlwsm49wnmxmm3byjx5jvr6z9fpzrq7v7fpb086adl0h"; }; buildCommand = "gunzip < $src > $out"; passthru.filetype = filetype.fa {}; }; - ncrna = stdenvNoCC.mkDerivation rec { + ncrna = pkgs.stdenvNoCC.mkDerivation rec { name = "ensembl-grch38-ncrna-${version}"; version = "94"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ensembl.org/pub/release-${version}/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz"; sha256 = "1cpasykwriila52nqgvw6d3mjyh6d9qi613hvhn4h1dxkqzgnjff"; }; @@ -71,7 +71,7 @@ rec { snpeff = { db = stdenv.mkDerivation rec { name = "GRCh38.86"; - src = fetchurl { + src = pkgs.fetchurl { url = "mirror://sourceforge/project/snpeff/databases/v4_3/snpEff_v4_3_${name}.zip"; sha256 = "1rf8q7l732ayjq2lpny4s75zpij05j00151374nqblk4wri2mz0i"; }; @@ -98,10 +98,10 @@ rec { grcm38 = grcm38-p6; grcm38-p6 = { - seq = stdenvNoCC.mkDerivation rec { + seq = pkgs.stdenvNoCC.mkDerivation rec { name = "seq-grcm38.${version}"; version = "p6"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M19/GRCm38.${version}.genome.fa.gz"; sha256 = "0ryiqab5bldpzawylsk2qpjxr2j701q03ww9jqyxhkimqpn9g3mr"; }; @@ -109,20 +109,20 @@ rec { passthru.filetype = filetype.fa {}; }; ensembl = { - cdna = stdenvNoCC.mkDerivation rec { + cdna = pkgs.stdenvNoCC.mkDerivation rec { name = "ensembl-grch38-cdna-${version}"; version = "94"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/cdna/Mus_musculus.GRCm38.cdna.all.fa.gz"; sha256 = "0khp9l6s35lav2xqp7vkk6ybnz4wjihn7lapjf2lbpnbzjb4hp6d"; }; buildCommand = "gunzip < $src > $out"; passthru.filetype = filetype.fa {}; }; - ncrna = stdenvNoCC.mkDerivation rec { + ncrna = pkgs.stdenvNoCC.mkDerivation rec { name = "ensembl-grch38-ncrna-${version}"; version = "94"; - src = fetchurl { + src = pkgs.fetchurl { url = "ftp://ftp.ensembl.org/pub/release-${version}/fasta/mus_musculus/ncrna/Mus_musculus.GRCm38.ncrna.fa.gz"; sha256 = "0d997gm8p2b89rm5d46m2x4vz9lijxarfr2lzylnbi8gyqrbagdd"; }; @@ -134,10 +134,10 @@ rec { mm10 = mm10-p4; mm10-p4 = { - seq = stdenvNoCC.mkDerivation rec { + seq = pkgs.stdenvNoCC.mkDerivation rec { name = "seq-mm10.${version}"; version = "p4"; - src = fetchurl { + src = pkgs.fetchurl { url = "http://hgdownload.soe.ucsc.edu/goldenPath/mm10/mm10Patch4/mm10Patch4.fa.gz"; sha256 = "1660d6d05f3aa266c6053cfd1efef1747d9e854836917241d6f47cff7a55340c"; }; diff --git a/lib/types.nix b/lib/types.nix index 3e26b8a..f711170 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -1,10 +1,10 @@ -{bionix, nixpkgs}: +{bionix}: -with nixpkgs; +with bionix; with lib; let - nix-adt-src = fetchFromGitHub { + nix-adt-src = pkgs.fetchFromGitHub { owner = "shlevy"; repo = "nix-adt"; rev = "dd04b5d08eed65ecd73aafde56a78078e09f1c74"; diff --git a/test-tnpair.nix b/test-tnpair.nix index b897366..0851a74 100644 --- a/test-tnpair.nix +++ b/test-tnpair.nix @@ -1,49 +1,39 @@ -with (import <nixpkgs> {}); +with import <bionix> {}; with lib; let - bionix = (import <bionix> {}).extend (self: super: with self; { - bwa = with super.bwa; { - align = align; - index = def index { flags = "-a is"; }; - }; - }); - -in - -with bionix; - -let - fetchlocal = path: stdenv.mkDerivation { + fetchlocal = path: pkgs.stdenv.mkDerivation { name = baseNameOf path; buildCommand = "ln -s ${path} $out"; }; fetchfq = attrs: types.tagFiletype (types.filetype.fq {}) (fetchlocal attrs); fetchfa = attrs: types.tagFiletype (types.filetype.fa {}) (fetchlocal attrs); - ref = fetchfa ./example/ref.fa; + ref = fetchfa ./examples/ref.fa; alignWithRG = rg: bwa.align { inherit ref; flags = "-R'@RG\\tID:${rg}\\tSM:${rg}'";}; sort = samtools.sort {}; flagstat = samtools.flagstat {}; check = fastqc.check {}; callVariants = strelka.call {}; + markdup = samtools.markdup {}; + fixmate = samtools.fixmate {}; tnpair = { tumour = {name = "mysample1"; files = { - input1 = fetchfq ./example/sample1-1.fq; - input2 = fetchfq ./example/sample1-2.fq; + input1 = fetchfq ./examples/sample1-1.fq; + input2 = fetchfq ./examples/sample1-2.fq; }; }; normal = {name = "mysample2"; files = { - input1 = fetchfq ./example/sample2-1.fq; - input2 = fetchfq ./example/sample2-2.fq; + input1 = fetchfq ./examples/sample2-1.fq; + input2 = fetchfq ./examples/sample2-2.fq; }; }; }; processPair = { tumour, normal }: rec { - alignments = mapAttrs (_: x: sort (alignWithRG x.name x.files)) { inherit normal tumour; }; + alignments = mapAttrs (_: x: markdup (sort (fixmate (alignWithRG x.name x.files)))) { inherit normal tumour; }; variants = callVariants alignments; }; diff --git a/test.nix b/test.nix deleted file mode 100644 index 9851310..0000000 --- a/test.nix +++ /dev/null @@ -1,35 +0,0 @@ -{pkgs ? import <nixpkgs> {} -,bionix ? import <bionix> {}}: - -with pkgs; -with lib; -with bionix; - -let - inherit (types) filetype tagFiletype; - - fetchLocal = path: stdenv.mkDerivation { - name = baseNameOf path; - buildCommand = "ln -s ${path} $out"; - }; - tagfq = path: tagFiletype (filetype.fq {}) (fetchLocal path); - tagfa = path: tagFiletype (filetype.fa {}) (fetchLocal path); - - ref = tagfa ./example/ref.fa; - alignWithRG = rg: bwa.align { inherit ref; flags = "-R'@RG\\tID:${rg}\\tSM:${rg}'";}; - - samples = [ {name = "mysample1"; files = {input1 = tagfq ./example/sample1-1.fq; input2 = tagfq ./example/sample1-2.fq;};} - {name = "mysample2"; files = {input1 = tagfq ./example/sample2-1.fq; input2 = tagfq ./example/sample2-1.fq;};} ]; - - alignments = map (i: samtools.sort {} (alignWithRG i.name i.files)) samples; - variants = platypus.call {} alignments; - -in stdenv.mkDerivation { - name = "myproject"; - buildCommand = '' - mkdir $out - ln -s ${variants} $out/platypus.vcf - mkdir $out/alignments - ${concatStringsSep "\n" (zipListsWith (s: a: "ln -s ${a} $out/alignments/${s.name}.bam") samples alignments)} - ''; -} diff --git a/tools/bowtie-align.nix b/tools/bowtie-align.nix index 456747a..67a1169 100644 --- a/tools/bowtie-align.nix +++ b/tools/bowtie-align.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , ref , bamOutput ? true , flags ? null @@ -10,18 +9,18 @@ , input2 ? null }: -with nixpkgs; +with bionix; with lib; -with bionix.types; -with bionix.compression; +with types; +with compression; let fa = f: matchFiletype "bowtie2-ref" { fa = _: f; } f; fq = f: matchFiletype "bowtie2-input" { fq = _: f; gz = matchFiletype' "bowtie2-input" { fq = _: f; }; } f; -in stdenv.mkDerivation { +in stage { name = "bowtie2-align"; - buildInputs = [ bowtie2 bc ] ++ optional bamOutput samtools; + buildInputs = with pkgs; [ bowtie2 bc ] ++ optional bamOutput samtools; buildCommand = '' cores=$(echo $NIX_BUILD_CORES ${optionalString bamOutput "- 1"} | bc) if [[ $cores -lt 1 ]] ; then diff --git a/tools/bowtie-index.nix b/tools/bowtie-index.nix index 98e529f..4554ad1 100644 --- a/tools/bowtie-index.nix +++ b/tools/bowtie-index.nix @@ -1,20 +1,19 @@ { bionix -, nixpkgs , flags ? null , seed ? 42 }: ref: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; assert (matchFiletype "bowtie-index" { fa = _: true; } ref); -stdenv.mkDerivation { +stage { name = "bowtie-index"; - buildInputs = [ bowtie2 ]; + buildInputs = with pkgs; [ bowtie2 ]; buildCommand = '' mkdir $out bowtie2-build --seed ${toString seed} --threads $NIX_BUILD_CORES ${optionalString (flags != null) flags} ${ref} $out/ref diff --git a/tools/bowtie.nix b/tools/bowtie.nix index df8f955..a4439d3 100644 --- a/tools/bowtie.nix +++ b/tools/bowtie.nix @@ -1,4 +1,4 @@ -{ bionix, nixpkgs }: +{ bionix }: with bionix; diff --git a/tools/bwa-index.nix b/tools/bwa-index.nix index c879bc4..17b651f 100644 --- a/tools/bwa-index.nix +++ b/tools/bwa-index.nix @@ -1,19 +1,18 @@ { bionix -, nixpkgs , flags ? null }: ref: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; assert (matchFiletype "bwa-index" { fa = _: true; } ref); -stdenv.mkDerivation { +stage { name = "bwa-index"; - buildInputs = [ bwa ]; + buildInputs = with pkgs; [ bwa ]; buildCommand = '' ln -s ${ref} ref.fa bwa index ${optionalString (flags != null) flags} ref.fa diff --git a/tools/bwa-mem.nix b/tools/bwa-mem.nix index 2b3bc64..1427b49 100644 --- a/tools/bwa-mem.nix +++ b/tools/bwa-mem.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , ref , bamOutput ? true , flags ? null @@ -10,18 +9,18 @@ , input2 ? null }: -with nixpkgs; +with bionix; with lib; -with bionix.types; -with bionix.compression; +with types; +with compression; let fa = f: matchFiletype "bwa-ref" { fa = _: f; } f; fq = f: matchFiletype "bwa-input" { fq = _: f; gz = matchFiletype' "bwa-input" { fq = _: f; }; } f; -in stdenv.mkDerivation { +in stage { name = "bwa-mem"; - buildInputs = [ bwa bc ] ++ optional bamOutput samtools; + buildInputs = with pkgs; [ bwa bc ] ++ optional bamOutput samtools; buildCommand = '' ln -s ${fa ref} ref.fa for f in ${bionix.bwa.index indexAttrs ref}/* ; do diff --git a/tools/bwa.nix b/tools/bwa.nix index ea0e6a0..d0ab5ee 100644 --- a/tools/bwa.nix +++ b/tools/bwa.nix @@ -1,4 +1,4 @@ -{ bionix, nixpkgs }: +{ bionix }: with bionix; diff --git a/tools/compression.nix b/tools/compression.nix index 4184fec..6e69d7e 100644 --- a/tools/compression.nix +++ b/tools/compression.nix @@ -1,6 +1,5 @@ -{bionix, nixpkgs}: +{bionix}: -with nixpkgs; with bionix; { @@ -12,11 +11,11 @@ with bionix; cram = _: f; vcf = _: f; bed = _: f; - gz = _: types.tagFiletype (types.gunzip f.filetype) (stdenv.mkDerivation { + gz = _: types.tagFiletype (types.gunzip f.filetype) (stage { name = "gunzip"; buildCommand = "gunzip < ${f} > $out"; }); - bz2 = _: types.tagFiletype (types.bunzip2 f.filetype) (stdenv.mkDerivation { + bz2 = _: types.tagFiletype (types.bunzip2 f.filetype) (stage { name = "bunzip2"; buildCommand = "bunzip2 < ${f} > $out"; }); @@ -24,7 +23,7 @@ with bionix; gzip = f: let - gz = (stdenv.mkDerivation { + gz = (stage { name = "gzip"; buildCommand = "gzip < ${f} > $out"; passthru = { filetype = types.filetype.gz f.filetype; }; @@ -42,7 +41,7 @@ with bionix; bzip2 = f: let - bz2 = (stdenv.mkDerivation { + bz2 = (stage { name = "bzip2"; buildCommand = "bzip2 < ${f} > $out"; passthru = { filetype = types.filetype.bz2 f.filetype; }; diff --git a/tools/crumble.nix b/tools/crumble.nix index 6031fca..4256f37 100644 --- a/tools/crumble.nix +++ b/tools/crumble.nix @@ -1,8 +1,7 @@ -{ bionix, nixpkgs }: +{ bionix }: -with nixpkgs; with bionix; { - crumble = callPackage ./crumble-app.nix {}; + crumble = pkgs.callPackage ./crumble-app.nix {}; } diff --git a/tools/fastqc-check.nix b/tools/fastqc-check.nix index 17d6183..d33b987 100644 --- a/tools/fastqc-check.nix +++ b/tools/fastqc-check.nix @@ -1,14 +1,13 @@ { bionix -, nixpkgs , flags ? null }: -with nixpkgs; +with bionix; with lib; input: -stdenv.mkDerivation { +stage { name = "fastqc-check"; buildInputs = [ bionix.fastqc.fastqc ]; buildCommand = '' diff --git a/tools/fastqc.nix b/tools/fastqc.nix index 6e603b2..5632958 100644 --- a/tools/fastqc.nix +++ b/tools/fastqc.nix @@ -1,9 +1,8 @@ -{ bionix, nixpkgs }: +{ bionix }: -with nixpkgs; with bionix; { - fastqc = callPackage ./fastqc-app.nix {}; + fastqc = pkgs.callPackage ./fastqc-app.nix {}; check = callBionixE ./fastqc-check.nix; } diff --git a/tools/gridss-assemble.nix b/tools/gridss-assemble.nix index f57bc12..e5cb3ca 100644 --- a/tools/gridss-assemble.nix +++ b/tools/gridss-assemble.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , bwaIndexAttrs ? {} , faidxAttrs ? {} , indexAttrs ? {} @@ -9,9 +8,9 @@ , heapSize ? "31g" }: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; inputs: @@ -36,9 +35,9 @@ in assert (all sorted inputs); assert (homoRef); -stdenv.mkDerivation rec { +stage rec { name = "gridss-assemble"; - buildInputs = [ jre bwa ]; + buildInputs = with pkgs; [ jre bwa ]; buildCommand = '' TMPDIR=$(pwd) ln -s ${ref} ref.fa diff --git a/tools/gridss-callVariants.nix b/tools/gridss-callVariants.nix index d55db23..4117126 100644 --- a/tools/gridss-callVariants.nix +++ b/tools/gridss-callVariants.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , blacklist ? null , bwaIndexAttrs ? {} , faidxAttrs ? {} @@ -8,9 +7,9 @@ , heapSize ? "31g" }: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; inputs: @@ -22,9 +21,9 @@ in assert (length (unique refs) == 1); -stdenv.mkDerivation rec { +stage rec { name = "gridss-callVariants"; - buildInputs = [ jre R bwa ]; + buildInputs = with pkgs; [ jre R bwa ]; buildCommand = '' ln -s ${ref} ref.fa ln -s ${bionix.samtools.faidx faidxAttrs ref} ref.fa.fai diff --git a/tools/gridss-collectMetrics.nix b/tools/gridss-collectMetrics.nix index 17e3c46..37e22f1 100644 --- a/tools/gridss-collectMetrics.nix +++ b/tools/gridss-collectMetrics.nix @@ -1,14 +1,13 @@ { bionix -, nixpkgs , thresholdCoverage ? 10000 , flags ? null , config ? null , heapSize ? "1G" }: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; input: @@ -17,9 +16,9 @@ let in -stdenv.mkDerivation rec { +stage rec { name = "gridss-collectMetrics"; - buildInputs = [ jre R ]; + buildInputs = with pkgs; [ jre R ]; buildCommand = '' mkdir $out java -Xmx${heapSize} -cp ${bionix.gridss.jar} \ diff --git a/tools/gridss-computeSamTags.nix b/tools/gridss-computeSamTags.nix index c462461..5b249e7 100644 --- a/tools/gridss-computeSamTags.nix +++ b/tools/gridss-computeSamTags.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , blacklist ? null , bwaIndexAttrs ? {} , faidxAttrs ? {} @@ -8,9 +7,9 @@ , heapSize ? "1G" }: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; input: @@ -21,9 +20,9 @@ in assert(sorted); -stdenv.mkDerivation rec { +stage rec { name = "gridss-computeSamTags"; - buildInputs = [ jre ]; + buildInputs = with pkgs; [ jre ]; buildCommand = '' ln -s ${ref} ref.fa ln -s ${bionix.samtools.faidx faidxAttrs ref} ref.fa.fai diff --git a/tools/gridss-configFile.nix b/tools/gridss-configFile.nix index 15eba55..d6b600e 100644 --- a/tools/gridss-configFile.nix +++ b/tools/gridss-configFile.nix @@ -1,14 +1,15 @@ -{bionix, nixpkgs}: +{bionix}: -with nixpkgs; +with bionix; +with lib; let attrsToGridssConfigString = attrsToGridssConfigStringPrepend ""; attrsToGridssConfigStringPrepend = prepend: attrs: - lib.concatStringsSep "\n" ( - lib.attrValues ( - lib.mapAttrs + concatStringsSep "\n" ( + attrValues ( + mapAttrs (name: attr: prepend + (iniLine name attr)) attrs)); @@ -31,6 +32,6 @@ let # Allows for repeated fields (e.g. for adapters): list = name: attr: concatStringsSep "\n" (map (x: iniLine name x) attr); }; -in configAttrs: (writeText +in configAttrs: (pkgs.writeText "gridss.properties.override" ((attrsToGridssConfigString configAttrs) + "\n")) diff --git a/tools/gridss-extractSVReads.nix b/tools/gridss-extractSVReads.nix index 5fc4c59..f2fab30 100644 --- a/tools/gridss-extractSVReads.nix +++ b/tools/gridss-extractSVReads.nix @@ -1,5 +1,4 @@ { bionix -, nixpkgs , dictIndexAttrs ? {} , faidxAttrs ? {} , flags ? null @@ -9,9 +8,9 @@ , config ? null }: -with nixpkgs; +with bionix; with lib; -with bionix.types; +with types; input: @@ -20,9 +19,9 @@ let in -stdenv.mkDerivation rec { +stage rec { name = "gridss-extractSVReads"; - buildInputs = [ jre R ]; + buildInputs = with pk |