From 7c3e4486551735cca7a93c465b27d9160da336aa Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Wed, 30 Oct 2024 14:19:12 +0100 Subject: [PATCH 01/12] Removed .nextflow directory when updating checksums & disabled publishing md5sum files in demultiplex pipeline --- roles/arteria-sequencing-report-ws/files/hiseq.rsync | 3 +++ .../templates/nextflow_configs/demultiplex.config.j2 | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/arteria-sequencing-report-ws/files/hiseq.rsync b/roles/arteria-sequencing-report-ws/files/hiseq.rsync index 6de43274..1a38d5c3 100644 --- a/roles/arteria-sequencing-report-ws/files/hiseq.rsync +++ b/roles/arteria-sequencing-report-ws/files/hiseq.rsync @@ -37,6 +37,9 @@ # Exclude all .sentinel files - .sentinel +# Exclude all .nextflow directory +- .nextflow + # Include full tree of some subdirs # This works if rsync is called with src dest, but not src/ dest + /*/Sisyphus/*** diff --git a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 index 4209150e..32db9f5b 100644 --- a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 +++ b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 @@ -51,7 +51,7 @@ process { } withName: MD5SUM { - publishDir = [] + publishDir enabled:false } } From 754247bdbb6c8cbdbfe49b17dce17f6422c9ef1c Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Mon, 4 Nov 2024 11:25:40 +0100 Subject: [PATCH 02/12] Skipping the MD5SUM process in the demultiplex pipeline --- .../templates/nextflow_configs/demultiplex.config.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 index 32db9f5b..d2da130a 100644 --- a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 +++ b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 @@ -14,6 +14,10 @@ plugins { id 'nf-validation@{{ nf_validation_version }}' } +params { + skip_tools = 'MD5SUM' +} + process { withName: BCL2FASTQ { @@ -50,8 +54,4 @@ process { ] } - withName: MD5SUM { - publishDir enabled:false - } } - From 42f6f02d5487b622392dc7c1c2bb15dfeb3545f4 Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Mon, 4 Nov 2024 13:20:15 +0100 Subject: [PATCH 03/12] Changed case for module to skip according to documentation here https://nf-co.re/demultiplex/dev/parameters/#:~:text=Comma%2Dseparated%20list%20of%20tools%20to%20skip%20(fastp%2Cfastqc%2Ckraken%2Cmultiqc%2Ccheckqc%2Cfalco%2Cmd5sum%2Csamshee) --- .../templates/nextflow_configs/demultiplex.config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 index d2da130a..8c657f43 100644 --- a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 +++ b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 @@ -15,7 +15,7 @@ plugins { } params { - skip_tools = 'MD5SUM' + skip_tools = 'md5sum' } process { From bf93882400a57baed44693b6f1502e77934c2e9f Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Mon, 11 Nov 2024 14:11:37 +0100 Subject: [PATCH 04/12] Moved skip_tools md5sum to the main demultiplex pipeline config --- .../templates/nextflow_configs/demultiplex.config.j2 | 4 ---- .../templates/pipeline_configs/demultiplex.yml.j2 | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 index 8c657f43..13391552 100644 --- a/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 +++ b/roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2 @@ -14,10 +14,6 @@ plugins { id 'nf-validation@{{ nf_validation_version }}' } -params { - skip_tools = 'md5sum' -} - process { withName: BCL2FASTQ { diff --git a/roles/arteria-sequencing-report-ws/templates/pipeline_configs/demultiplex.yml.j2 b/roles/arteria-sequencing-report-ws/templates/pipeline_configs/demultiplex.yml.j2 index 886ef574..39d9fa0f 100644 --- a/roles/arteria-sequencing-report-ws/templates/pipeline_configs/demultiplex.yml.j2 +++ b/roles/arteria-sequencing-report-ws/templates/pipeline_configs/demultiplex.yml.j2 @@ -20,7 +20,7 @@ pipeline_parameters: outdir: "{runfolder_path}" project: {{uppmax_project}} demultiplexer: "bcl2fastq" - skip_tools: "fastp,falco,multiqc" + skip_tools: "fastp,falco,multiqc,md5sum" input_samplesheet_content: | id,samplesheet,lane,flowcell {runfolder_name},{runfolder_path}/SampleSheet.csv,,{runfolder_path} From 2eeaea9988526c50cff047d5ae25774c8b0a50db Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Mon, 18 Nov 2024 15:04:43 +0100 Subject: [PATCH 05/12] Update multiqc repo path, Upgrade multiqc_ngi to 0.9.0 to use cloudant_python_sdk --- roles/multiqc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/multiqc/defaults/main.yml b/roles/multiqc/defaults/main.yml index c0c6a735..b06d05bf 100644 --- a/roles/multiqc/defaults/main.yml +++ b/roles/multiqc/defaults/main.yml @@ -1,7 +1,7 @@ -multiqc_repo: https://github.com/ewels/MultiQC.git +multiqc_repo: https://github.com/MultiQC/MultiQC.git multiqc_dest: "{{ sw_path }}/multiqc" multiqc_version: "v1.25.1" multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi" -multiqc_ngi_version: "0.8.2" +multiqc_ngi_version: "0.9.0" From 9e1b237c19f3e6486cdf55d0fca566d816432f04 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Mon, 18 Nov 2024 15:05:14 +0100 Subject: [PATCH 06/12] Bump TACA to latest --- roles/taca/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taca/defaults/main.yml b/roles/taca/defaults/main.yml index 0a135f3f..16710ce7 100644 --- a/roles/taca/defaults/main.yml +++ b/roles/taca/defaults/main.yml @@ -9,7 +9,7 @@ flowcell_parser_version: cfe18abb6b758a6389a9d6aa75f1866d517dc36e taca_repo: https://github.com/SciLifeLab/TACA.git taca_dest: "{{ sw_path }}/TACA" -taca_version: c08d3f3ca4ed2f0f8ef7e669b6d7313b746e33f0 +taca_version: 43b763b44be324c9d8ef6841d9c56e2a85cd2d62 ngi_pipeline_analysisdir: "{{ ngi_pipeline_workdir }}/ANALYSIS" ngi_pipeline_datadir: "{{ ngi_pipeline_workdir }}/DATA" From 9c04660737766291df205916487134dd145685b5 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Mon, 18 Nov 2024 16:31:27 +0100 Subject: [PATCH 07/12] Update TACA repo link --- roles/taca/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taca/defaults/main.yml b/roles/taca/defaults/main.yml index 16710ce7..7380f1e0 100644 --- a/roles/taca/defaults/main.yml +++ b/roles/taca/defaults/main.yml @@ -7,7 +7,7 @@ flowcell_parser_repo: https://github.com/NationalGenomicsInfrastructure/flowcell flowcell_parser_dest: "{{ sw_path }}/flowcell_parser" flowcell_parser_version: cfe18abb6b758a6389a9d6aa75f1866d517dc36e -taca_repo: https://github.com/SciLifeLab/TACA.git +taca_repo: https://github.com/NationalGenomicsInfrastructure/TACA.git taca_dest: "{{ sw_path }}/TACA" taca_version: 43b763b44be324c9d8ef6841d9c56e2a85cd2d62 From ecf06c4bea0005b427cfbfb31aa317b227111087 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Tue, 19 Nov 2024 12:31:30 +0100 Subject: [PATCH 08/12] Fix bug in multiqc_ngi --- roles/multiqc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/multiqc/defaults/main.yml b/roles/multiqc/defaults/main.yml index b06d05bf..91709cb6 100644 --- a/roles/multiqc/defaults/main.yml +++ b/roles/multiqc/defaults/main.yml @@ -4,4 +4,4 @@ multiqc_version: "v1.25.1" multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi" -multiqc_ngi_version: "0.9.0" +multiqc_ngi_version: "0.9.1" From 3363453b1cae15e6a1dc695b62c6dee9f6090f17 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Thu, 21 Nov 2024 11:51:17 +0100 Subject: [PATCH 09/12] Upgrade multiqc to fix kaleido issues and multiqc_ngi for import issues --- roles/multiqc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/multiqc/defaults/main.yml b/roles/multiqc/defaults/main.yml index 91709cb6..16f9f900 100644 --- a/roles/multiqc/defaults/main.yml +++ b/roles/multiqc/defaults/main.yml @@ -1,7 +1,7 @@ multiqc_repo: https://github.com/MultiQC/MultiQC.git multiqc_dest: "{{ sw_path }}/multiqc" -multiqc_version: "v1.25.1" +multiqc_version: "v1.25.2" multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi" -multiqc_ngi_version: "0.9.1" +multiqc_ngi_version: "0.9.2" From d2425197f36815e4eb0dcc464baef4b2a8fb2d0c Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Mon, 25 Nov 2024 16:32:37 +0100 Subject: [PATCH 10/12] Changed path of .nextflow folder to be excluded to - /*/.nextflow/*** --- roles/arteria-sequencing-report-ws/files/hiseq.rsync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/arteria-sequencing-report-ws/files/hiseq.rsync b/roles/arteria-sequencing-report-ws/files/hiseq.rsync index 1a38d5c3..a9bb765a 100644 --- a/roles/arteria-sequencing-report-ws/files/hiseq.rsync +++ b/roles/arteria-sequencing-report-ws/files/hiseq.rsync @@ -38,7 +38,7 @@ - .sentinel # Exclude all .nextflow directory -- .nextflow +- /*/.nextflow/*** # Include full tree of some subdirs # This works if rsync is called with src dest, but not src/ dest From 375d5ea89db3bd5e09f5a3dd619f73fa97899aa4 Mon Sep 17 00:00:00 2001 From: nelnk861 Date: Fri, 29 Nov 2024 11:05:36 +0100 Subject: [PATCH 11/12] Fix exclude .nextflow Co-authored-by: Adrien Coulier --- roles/arteria-sequencing-report-ws/files/hiseq.rsync | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/roles/arteria-sequencing-report-ws/files/hiseq.rsync b/roles/arteria-sequencing-report-ws/files/hiseq.rsync index a9bb765a..502e3e45 100644 --- a/roles/arteria-sequencing-report-ws/files/hiseq.rsync +++ b/roles/arteria-sequencing-report-ws/files/hiseq.rsync @@ -5,6 +5,13 @@ # rsync -av --prune-empty-dirs --include-from hiseq.rsync $PWD/ biologin:~/incoming/runfolders/$runfolder > $PWD/rsync.log # +# Exclude all .nextflow directory +# This works if rsync is called with src dest +- /*/.nextflow/*** + +# This works if rsync is called with src/ dest +- .nextflow + + */ # Include some files regardless of location in tree @@ -37,9 +44,6 @@ # Exclude all .sentinel files - .sentinel -# Exclude all .nextflow directory -- /*/.nextflow/*** - # Include full tree of some subdirs # This works if rsync is called with src dest, but not src/ dest + /*/Sisyphus/*** From c9595944e16d6dfa51a7a2846f1791b0368f66a9 Mon Sep 17 00:00:00 2001 From: matrulda Date: Tue, 10 Dec 2024 08:19:13 +0100 Subject: [PATCH 12/12] Update Uppsalas crontab to check disk usage every Monday --- roles/func_accounts/templates/crontab.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/func_accounts/templates/crontab.j2 b/roles/func_accounts/templates/crontab.j2 index 956661c2..1e0eb298 100644 --- a/roles/func_accounts/templates/crontab.j2 +++ b/roles/func_accounts/templates/crontab.j2 @@ -6,6 +6,9 @@ SHELL=/bin/bash # restart supervisord if it has died for some reason 11 * * * * bash {{ ngi_resources }}/start_supervisord_{{ site }}.sh &> /dev/null +# Check disk usage every Monday +0 7 * * 1 echo "Disk usage Miarka:" && /vulpes/ngi/production/latest/sw/standalone_scripts/du_miarka.py /proj/ngi2016001 + {% endif %} # updates charon with locally stored run info for sarek