Merge branch 'develop' into enhancement/multiqc

mskcc · Apr 10, 2023 · c7d92fb · c7d92fb
2 parents e33f7fe + 33aa15b
commit c7d92fb
Show file tree

Hide file tree

Showing 32 changed files with 364 additions and 165 deletions.
diff --git a/.github/workflows/code-linting.yml b/.github/workflows/code-linting.yml
@@ -3,7 +3,7 @@ on:
   push:
     branches: [master]
   pull_request:
-    branches: [main]
+    branches: [main, develop]
 
 # Cancel if a newer run is started
 concurrency:

diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml
@@ -0,0 +1,79 @@
+name: Pytest-workflow
+on:
+  pull_request:
+    branches: [main, develop]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    name: ${{ matrix.tags }} ${{ matrix.profile }}
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["test_profile"]
+        profile: ["singularity"]
+    env:
+      NXF_ANSI_LOG: false
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install Python dependencies
+        run: python -m pip install --upgrade pip pytest-workflow
+
+      - uses: actions/cache@v2
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-
+      - name: Install Nextflow
+        env:
+          CAPSULE_LOG: none
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      # Test the module
+      - name: Run pytest-workflow
+        # only use one thread for pytest-workflow to avoid race condition on conda cache.
+        run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt-get update > /dev/null
+          sudo apt-get install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.profile }}
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/work
+            !/home/runner/pytest_workflow_*/*/work/singularity
diff --git a/README.md b/README.md
@@ -56,10 +56,12 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool
    | umi2    | `str`/`int` | `NNNXX`/`3`          | `''`     |
    | strand  | `str`       | `yes`/`no`/`reverse` | `no`     |
    | fastq_1 | `str`       | `/path/to/*fastq.gz` | (none)   |
-   | fastq_1 | `str`       | `/path/to/*fastq.gz` | (none)   |
+   | fastq_2 | `str`       | `/path/to/*fastq.gz` | (none)   |
 
    If you are running on juno, chain the `juno` profile (i.e. `-profile singularity,juno`) to take advantage of local resources on juno.
 
+5. For more information on running the pipeline, please see the [detailed usage documentation](docs/usage.md).
+
 ## Credits
 
 anoronh4/forte was originally written by Anne Marie Noronha <noronhaa@mskcc.org>.

diff --git a/conf/juno.config b/conf/juno.config
@@ -12,6 +12,30 @@ process {
     beforeScript   = "module load singularity/3.1.1; unset R_LIBS; catch_term () { echo 'caught USR2/TERM signal'; set +e; false; on_exit ; } ; trap catch_term USR2 TERM"
     errorStrategy  = { task.attempt <= 3 ? 'retry' : 'ignore' }
     maxRetries     = 3
+
+    withLabel:process_single {
+        cpus   = { check_max( 1                  , 'cpus'    )                    }
+        memory = { round_memory( check_max( 6.GB, 'memory'   )/task.cpus, "down") }
+        time   = { check_max( 4.h  * task.attempt, 'time'    )                    }
+    }
+    withLabel:process_low {
+        cpus   = { check_max( 2     * task.attempt, 'cpus'   )                    }
+        memory = { round_memory( check_max( 12.GB, 'memory'  )/task.cpus, "down") }
+        time   = { check_max( 4.h   * task.attempt, 'time'   )                    }
+    }
+    withLabel:process_medium {
+        cpus   = { check_max( 6     * task.attempt, 'cpus'   )                    }
+        memory = { round_memory( check_max( 36.GB, 'memory'  )/task.cpus, "down") }
+        time   = { check_max( 8.h   * task.attempt, 'time'   )                    }
+    }
+    withLabel:process_high {
+        cpus   = { check_max( 12    * task.attempt, 'cpus'   )                    }
+        memory = { round_memory( check_max( 72.GB, 'memory'  )/task.cpus, "down") }
+        time   = { check_max( 16.h  * task.attempt, 'time'   )                    }
+    }
+    withLabel:process_high_memory {
+        memory = { round_memory( check_max( 200.GB, 'memory' )/task.cpus, "down") }
+    }
 }
 
 params {

diff --git a/conf/modules.config b/conf/modules.config
@@ -122,10 +122,14 @@ process {
     }
 
     withName: FUSIONREPORT_DOWNLOAD {
+        ext.when = params.run_fusion_report
+        secret = ["COSMIC_PASSWD"]
         storeDir = { "${params.reference_base}/fusionreport" }
+        ext.args = "--cosmic_usr ${params.cosmic_usr} --cosmic_passwd \$COSMIC_PASSWD"
     }
 
     withName: FUSIONREPORT {
+        ext.when = params.run_fusion_report
         ext.args = {
             [
                 "-t ${params.fusion_report_cutoff}",
@@ -235,7 +239,7 @@ process {
         ext.prefix = { "$meta.sample" }
         ext.args = {
             [
-                "--outSAMattrRGline ${meta.read_group.collect{"ID:${it} SM:${meta.sample} PL:Illumina"}.join(" , ")}",
+                "--outSAMattrRGline ${meta.read_group.split(",").collect{"ID:${it} SM:${meta.sample} PL:Illumina"}.join(" , ")}",
                 '--quantMode GeneCounts',
                 '--twopassMode Basic',
                 '--outSAMtype BAM SortedByCoordinate',

diff --git a/conf/test.config b/conf/test.config
@@ -27,5 +27,6 @@ params {
     outdir = "results"
 
     genome = 'smallGRCh37'
+    run_fusion_report = false
 
 }
diff --git a/docs/usage.md b/docs/usage.md
@@ -82,6 +82,14 @@ When you run the above command, Nextflow automatically pulls the pipeline code f
 nextflow pull anoronh4/forte
 ```
 
+### Fusion-report annotation
+
+Annotation with the fusion-report module uses a reference database that is built within the Forte pipeline. Building the reference database requires access to COSMIC with a username and password. Once you have these two items, you can pass in the username on the command-line like so: `--cosmic_usr <yourusername>` and you must set up the password using nextflow's `secrets` functionality, which allows for the secure transmission of sensitive information within the pipeline:
+
+```bash
+nextflow secrets set COSMIC_PASSWD 'mycosmicpw'
+```
+
 ### OncoKB annotation
 
 To enable OncoKB fusion annotation, you must have an [API token to access data from OncoKB](https://www.oncokb.org/apiAccess). Once you have obtained a token, it needs to be registered as a Nextflow Secret, which allows for the secure transmission of sensitive information within the pipeline:

diff --git a/modules.json b/modules.json
@@ -7,7 +7,7 @@
                 "nf-core": {
                     "arriba": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
                         "installed_by": ["modules"]
                     },
                     "cat/fastq": {
@@ -22,12 +22,12 @@
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "8022c68e7403eecbd8ba9c49496f69f8c49d50f0",
+                        "git_sha": "b6d4d476aee074311c89d82a69c1921bd70c8180",
                         "installed_by": ["modules"]
                     },
                     "fastp": {
                         "branch": "master",
-                        "git_sha": "1e49f31e93c56a3832833eef90a02d3cde5a3f7e",
+                        "git_sha": "20a508676f40d0fd3f911ac595af91ec845704c4",
                         "installed_by": ["modules"]
                     },
                     "gatk4/bedtointervallist": {
@@ -127,12 +127,12 @@
                     },
                     "umitools/dedup": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "git_sha": "6d9c7e43404e20a97d2f6f88548456afe78282e6",
                         "installed_by": ["modules"]
                     },
                     "umitools/extract": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "git_sha": "6d9c7e43404e20a97d2f6f88548456afe78282e6",
                         "installed_by": ["modules"]
                     }
                 }

diff --git a/modules/local/fusioncatcher/detect/main.nf b/modules/local/fusioncatcher/detect/main.nf
@@ -2,7 +2,7 @@ process FUSIONCATCHER_DETECT {
     tag "$meta.id"
     label 'process_high'
 
-    conda (params.enable_conda ? "bioconda::fusioncatcher=1.33" : null)
+    conda "bioconda::fusioncatcher=1.33"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'docker.io/clinicalgenomics/fusioncatcher:1.33' :
         'docker.io/clinicalgenomics/fusioncatcher:1.33' }"

diff --git a/modules/local/fusioncatcher/download/main.nf b/modules/local/fusioncatcher/download/main.nf
@@ -2,7 +2,7 @@ process FUSIONCATCHER_DOWNLOAD {
     tag 'fusioncatcher_download'
     label 'process_medium'
 
-    conda (params.enable_conda ? "bioconda::fusioncatcher=1.33" : null)
+    conda "bioconda::fusioncatcher=1.33"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'docker.io/clinicalgenomics/fusioncatcher:1.33' :
         'docker.io/clinicalgenomics/fusioncatcher:1.33' }"

diff --git a/modules/local/fusionreport/download/main.nf b/modules/local/fusionreport/download/main.nf
@@ -3,25 +3,26 @@ process FUSIONREPORT_DOWNLOAD {
     label 'process_medium'
 
     // Note: 2.7X indices incompatible with AWS iGenomes.
-    conda (params.enable_conda ? 'bioconda::star=2.7.9a' : null)
+    conda 'bioconda::star=2.7.9a'
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'cmopipeline/fusion-report:0.0.1' :
         'cmopipeline/fusion-report:0.0.1' }"
         //'docker.io/rannickscilifelab/fusion-report:2.1.5updated' :
         //'docker.io/rannickscilifelab/fusion-report:2.1.5updated' }"
 
-
-    input:
-    val(username)
-    val(passwd)
-
     output:
     path "db"                , emit: reference
-    path "versions.yml"     , emit: versions
+    path "versions.yml"      , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
 
     script:
+    def args = task.ext.args ?: ''
     """
-    fusion_report download --cosmic_usr $username --cosmic_passwd $passwd db
+    fusion_report download \\
+        $args \\
+        db
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

diff --git a/modules/local/fusionreport/run/main.nf b/modules/local/fusionreport/run/main.nf
@@ -3,7 +3,7 @@ process FUSIONREPORT {
     label 'process_low'
 
     // Note: 2.7X indices incompatible with AWS iGenomes.
-    conda (params.enable_conda ? 'bioconda::star=2.7.9a' : null)
+    conda 'bioconda::star=2.7.9a'
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'cmopipeline/fusion-report:0.0.1' :
         'cmopipeline/fusion-report:0.0.1' }"

diff --git a/modules/local/htseq/count/main.nf b/modules/local/htseq/count/main.nf
@@ -2,7 +2,7 @@ process HTSEQ_COUNT {
     tag "$meta.id"
     label 'process_medium'
 
-    conda (params.enable_conda ? "bioconda::htseq=2.0.2" : null)
+    conda "bioconda::htseq=2.0.2"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/htseq:2.0.2--py39h919a90d_0' :
         'quay.io/biocontainers/htseq:2.0.2--py39h919a90d_0' }"

diff --git a/modules/local/star/align/main.nf b/modules/local/star/align/main.nf
@@ -2,7 +2,7 @@ process STAR_ALIGN {
     tag "$meta.id"
     label 'process_high'
 
-    conda (params.enable_conda ? "bioconda::star=2.7.10a bioconda::samtools=1.16.1 conda-forge::gawk=5.1.0" : null)
+    conda "bioconda::star=2.7.10a bioconda::samtools=1.16.1 conda-forge::gawk=5.1.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' :
         'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' }"

diff --git a/modules/local/starfusion/build/main.nf b/modules/local/starfusion/build/main.nf
@@ -1,7 +1,7 @@
 process STARFUSION_BUILD {
     tag 'star-fusion'
 
-    conda (params.enable_conda ? "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a" : null)
+    conda "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a"
     container "docker.io/trinityctat/starfusion:1.10.1"
 
     input:

diff --git a/modules/local/starfusion/detect/main.nf b/modules/local/starfusion/detect/main.nf
@@ -2,7 +2,7 @@ process STARFUSION {
     tag "$meta.id"
     label 'process_high'
 
-    conda (params.enable_conda ? "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a" : null)
+    conda "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a"
     container "docker.io/trinityctat/starfusion:1.10.1"
 
     input:
@@ -17,13 +17,14 @@ process STARFUSION {
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def fasta = meta.single_end ? "--left_fq ${reads[0]}" : "--left_fq ${reads[0]} --right_fq ${reads[1]}"
+    def reads_in = reads ? ( meta.single_end ? "--left_fq ${reads[0]}" : "--left_fq ${reads[0]} --right_fq ${reads[1]}" ) : ''
+    def junction_in = junction ? "-J $junction" : ''
     def args = task.ext.args ?: ''
     """
     STAR-Fusion \\
         --genome_lib_dir $reference \\
-        $fasta \\
-        -J $junction \\
+        $reads_in \\
+        $junction_in \\
         --CPU $task.cpus \\
         --examine_coding_effect \\
         --output_dir . \\

diff --git a/modules/local/starfusion/download/main.nf b/modules/local/starfusion/download/main.nf
@@ -1,7 +1,7 @@
 process STARFUSION_DOWNLOAD {
     tag 'star-fusion'
 
-    conda (params.enable_conda ? "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a" : null)
+    conda "bioconda::dfam=3.3 bioconda::hmmer=3.3.2 bioconda::star-fusion=1.10.0 bioconda::trinity=date.2011_11_2 bioconda::samtools=1.9 bioconda::star=2.7.8a"
     container "docker.io/trinityctat/starfusion:1.10.1"
 
     input:

diff --git a/modules/nf-core/arriba/main.nf b/modules/nf-core/arriba/main.nf
diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf