| 45 | | subgraph cluster_1 { |
| 46 | | style=filled; |
| 47 | | color=lightgrey; |
| 48 | | "flowcell_lane.1.fq.gz" -> align1 -> alignPE |
| 49 | | "chr[1-24].fasta" -> align1 |
| 50 | | "chr[1-24].fasta" -> align2 |
| 51 | | "chr[1-24].fasta" -> alignPE |
| 52 | | "flowcell_lane.2.fq.gz" -> align2 -> alignPE -> MarkDuplicates -> "IndelRealigner & \n FixMateInformation (knownsOnly)" ->"Quality Recalibration"->"flowcell_lane.aligned.bam" |
| 53 | | "realign.intervals" -> "IndelRealigner & \n FixMateInformation (knownsOnly)" |
| 54 | | label = "Per Lane (750*3=2250) "; |
| 55 | | } |
| | 24 | "reference.fasta" -> RealignerTargetCreator -> "realign.intervals" "indelcalls.vcf"-> RealignerTargetCreator "reference.fasta"->Split->"chr[1-24] .fasta" dbsnp -> RealignerTargetCreator label = "Per genome (1)"; |
| | 28 | subgraph cluster_1 { |
| | 29 | style=filled; color=lightgrey; "flowcell_lane.1.fq.gz" -> align1 -> alignPE "chr[1-24] .fasta" -> align1 "chr[1-24] .fasta" -> align2 "chr[1-24] .fasta" -> alignPE "flowcell_lane.2.fq.gz" -> align2 -> alignPE -> MarkDuplicates -> "IndelRealigner & \n FixMateInformation (knownsOnly)" ->"Quality Recalibration"->"flowcell_lane.aligned.bam" "realign.intervals" -> "IndelRealigner & \n FixMateInformation (knownsOnly)" label = "Per Lane (750*3=2250) "; |
| | 30 | } |
| | 31 | |
| | 32 | subgraph cluster_2 { |
| | 33 | style=filled; color=lightgrey; "flowcell_lane.aligned.bam" -> Merge -> "sample.aligned.bam" -> "IndelRealigner & FixMateInformation " "flowcell_lane2.aligned.bam" -> Merge "flowcell_lane3.aligned.bam" -> Merge "IndelRealigner & FixMateInformation " -> IndelGenotyperV2 -> FilterSingleCalls -> UnifiedGenotyper -> Filtration -> VariantEval -> "sample QC reports" |
| | 34 | |
| | 35 | Filtration -> "sample_chr[1-24].vcf" |
| | 36 | |
| | 37 | label = "Per Sample or Trio*Chromosome (750*24=18k)"; |
| | 38 | |
| | 39 | } |
| | 40 | |
| | 41 | subgraph cluster_3 { |
| | 42 | style=filled; color=lightgrey; |
| | 43 | |
| | 44 | "sample.aligned.bam" -> "UnifiedGenotype (without realign)"->"QC against arrays and BGI" |
| | 45 | |
| | 46 | label = "QC per sample"; |
| | 47 | |
| | 48 | } |
| | 49 | |
| | 50 | } |
| 95 | | * Recode workflow 2 to work per genome instead of per chromosome and test - Freerk (done) -> workflow3 still needs to be done |
| 96 | | * Run on pilot data (6) to evaluate timing and concurrency issues (can 6 run on one node?) - Freerk (in progress) |
| 97 | | * Complete analysis of data (60) until including merge to sample.aligned.bam - Freerk |
| 98 | | * QC pipeline - Can we get Jeroen and Yurii involved here |
| | 67 | |
| | 68 | * Recode workflow 2 to work per genome instead of per chromosome and test - Freerk (done) -> workflow3 still needs to be done |
| | 69 | * Run on pilot data (6) to evaluate timing and concurrency issues (can 6 run on one node?) - Freerk (in progress) |
| | 70 | * Complete analysis of data (60) until including merge to sample.aligned.bam - Freerk |
| | 71 | * QC pipeline - Can we get Jeroen and Yurii involved here |
| 109 | | * Port pipeline to Grid with help of Barbara |
| 110 | | * What do we need to generate exactly - Barbara |
| | 83 | * Port pipeline to Grid with help of Barbara |
| | 84 | * What do we need to generate exactly - Barbara |
| | 85 | |
| | 86 | == == |
| | 87 | == Optimization? == |
| | 88 | {{{ |
| | 89 | {| {{table}} |
| | 90 | | align="center" style="background:#f0f0f0;"|'''Step''' |
| | 91 | | align="center" style="background:#f0f0f0;"|'''Cores''' |
| | 92 | | align="center" style="background:#f0f0f0;"|'''Memory (gb)''' |
| | 93 | | align="center" style="background:#f0f0f0;"|'''Time (hh.mm)''' |
| | 94 | |- |
| | 95 | | BWA alignment||1||± 6||10.05 |
| | 96 | |- |
| | 97 | | BWA spe||1||||3.35 |
| | 98 | |- |
| | 99 | | Sam-Bam||1||||12.3 |
| | 100 | |- |
| | 101 | | Sam sort||1||||5.05 |
| | 102 | |- |
| | 103 | | Mark Duplicates||1||4||1.55 |
| | 104 | |- |
| | 105 | | Realignment (knowns only)||1||8 (*can be lowered)||5.2 |
| | 106 | |- |
| | 107 | | Fix mates||1||6 (*)||3.05 |
| | 108 | |- |
| | 109 | | Covariates bef.||1||2||12.35 |
| | 110 | |- |
| | 111 | | Recalibrate||1||4||7.3 |
| | 112 | |- |
| | 113 | | Sam sort||1||||4.5 |
| | 114 | |- |
| | 115 | | Covariates aft.||1||2||11.2 |
| | 116 | |- |
| | 117 | | Analyze Covar.||1||4||< 00.01 |
| | 118 | |- |
| | 119 | | Total||||||± 90 (< 4 days) |
| | 120 | |- |
| | 121 | | |
| | 122 | |} |
| | 123 | }}} |
| | 124 | === Disk === |
| | 125 | * Option 1, If it is possible to let a node guarantee certain amount of disk space (/tmp), we should use the entire cluster. Before start running a pipeline, we can just ask the node to reserve that amount of disk space. |
| | 126 | * Option 2, If we can cut a dedicate part of the cluster, we can use our own scheduler to share the nodes/disks. E.g, depending on the disk space usage pattern and how we can remove the data, we can decide which jobs run at which node and when. |
| | 127 | |
| | 128 | === Memory/CPU time === |
| | 129 | * Can multiple samples use the same reference genome in memory during the BWA alignment. I.e. 1 sample->6GB, 3 samples->6GB. |
| | 130 | * NO? |
| | 131 | * Can we parallelize the Markduplicate? |
| | 132 | * YES! |
| | 133 | * !MarkDuplicates finds sequence pairs that map to the same position, marking or removing the duplicates so you can work with unique pairs in downstream analyses. If you want them removed, use the REMOVE_DUPLICATES=true flag when running the program. |
| | 134 | * Can we parallelize covariate before/after, recalibration? |
| | 135 | * Don't know |
| | 136 | |
