COVID19 PickSeq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Rename Files by removing the first 23 characters**\n",
    "\n",
    "```\n",
    "for file in *; do mv $file `echo $file | cut -c23-`; done\n",
    "```\n",
    "\n",
    "**Unzip all the files**\n",
    "```\n",
    "bunzip2 *.bz2\n",
    "```\n",
    "\n",
    "**Create the merge.csv file**\n",
    "\n",
    "```\n",
    "(echo 'samplename,description'; for f in raw/*fastq; do readlink -f $f | perl -pe 's/(.*?_(S[0-9]+)_.*)/\\1,\\2/'; done) > to_merge.csv\n",
    "```\n",
    "\n",
    "\n",
    "\n",
    "**Merge multiple files into a single file**\n",
    "\n",
    "```\n",
    "#!/bin/sh\n",
    "#SBATCH -p priority\n",
    "#SBATCH -J merge\n",
    "#SBATCH -o run.o\n",
    "#SBATCH -e run.e\n",
    "#SBATCH -t 0-12:00\n",
    "#SBATCH --cpus-per-task=20\n",
    "#SBATCH --mail-type=END         # Type of email notification- BEGIN,END,FAIL,ALL\n",
    "#SBATCH --mail-user=ajitj_nirmal@dfci.harvard.edu   # Email to which notifications will be sent\n",
    "\n",
    "module load bcbio/latest\n",
    "cd raw\n",
    "bcbio_prepare_samples.py --memory-per-job 2G --out merged --csv tomerge.csv \n",
    "\n",
    "sbatch merge.sh\n",
    "```\n",
    "\n",
    "\n",
    "**Create the alignment.csv file**\n",
    "\n",
    "```\n",
    "(echo 'samplename,description'; for f in merged/*fastq.gz; do readlink -f $f | perl -pe 's/(.*?_(S[0-9]+)_.*)/\\1,\\2/'; done) > alignment.csv\n",
    "```\n",
    "\n",
    "**Prepare the O2.yaml file**\n",
    "\n",
    "```\n",
    "cd ..\n",
    "vim O2.yaml\n",
    "\n",
    "details:\n",
    "  - analysis: RNA-seq\n",
    "    genome_build: mmul10\n",
    "    algorithm:\n",
    "      aligner: hisat2\n",
    "      strandedness: unstranded\n",
    "upload:\n",
    "  dir: ../final\n",
    "```\n",
    "\n",
    "**Intiate bcBio**\n",
    "\n",
    "```\n",
    "module load bcbio/latest\n",
    "unset PYTHONPATH\n",
    "bcbio_nextgen.py -w template O2.yaml alignment.csv merged/\n",
    "```\n",
    "\n",
    "**Create Submission script to O2**\n",
    "\n",
    "```\n",
    "vim submit_bcbio.sh\n",
    "\n",
    "#!/bin/sh\n",
    "#SBATCH -p priority\n",
    "#SBATCH -J pickseq\n",
    "#SBATCH -o run.o\n",
    "#SBATCH -e run.e\n",
    "#SBATCH -t 10-00:00\n",
    "#SBATCH --cpus-per-task=20\n",
    "#SBATCH --mem=150G\n",
    "#SBATCH --mail-type=END         # Type of email notification- BEGIN,END,FAIL,ALL\n",
    "#SBATCH --mail-user=ajitj_nirmal@dfci.harvard.edu   # Email to which notifications will be sent\n",
    "\n",
    "export PATH=/n/app/bcbio/tools/bin:$PATH\n",
    "bcbio_nextgen.py ../config/alignment.yaml \\\n",
    "    -n 24 -t local\n",
    "```\n",
    "\n",
    "\n",
    "**Submit job to O2 for processing**\n",
    "\n",
    "```\n",
    "cp submit_bcbio.sh alignment/work\n",
    "cd alignment/work\n",
    "sbatch submit_bcbio.sh\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align viral genome: Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Extract the unaligned reads from the BAM fiels**\n",
    "\n",
    "```\n",
    "# copy all the bam files\n",
    "find alignment/final/S* -name '*-ready.bam' -exec cp -prv '{}' '/n/scratch3/users/a/ajn16/covid/bamfiles/' ';'\n",
    "\n",
    "```\n",
    "\n",
    "**Loop through all the files to extract the unmapped files**\n",
    "\n",
    "\n",
    "```\n",
    "module load sambamba/0.7.1\n",
    "\n",
    "mkdir unmapped\n",
    "\n",
    "for f in *.bam\n",
    "do\n",
    " echo \"Processing $f\"\n",
    " sambamba view -f bam $f -F \"(unmapped or mate_is_unmapped)\" > unmapped/$f\n",
    "done\n",
    "\n",
    "```\n",
    "\n",
    "**Convert the bam files to FASTQ**\n",
    "\n",
    "**step:1 Sort all the bam files**\n",
    "```\n",
    "# Sort all the bam files\n",
    "\n",
    "mkdir sorted\n",
    "\n",
    "sort=_sorted.bam\n",
    "for f in *.bam\n",
    "do\n",
    " ff=`echo $f | cut -d \\- -f 1`\n",
    " samtools sort -n $f > sorted/$ff$sort\n",
    "done\n",
    "\n",
    "```\n",
    "\n",
    "**step:2 Convert sorted bamfile to fastq files**\n",
    "\n",
    "```\n",
    "# convert sorted bamfile to fastq files\n",
    "module load gcc/6.2.0 bedtools/2.27.1\n",
    "\n",
    "R1=_R1.fq\n",
    "R2=_R2.fq\n",
    "for f in *.bam\n",
    "do\n",
    " ff=`echo $f | cut -d \\_ -f 1`\n",
    " echo \"Processing $ff\"\n",
    " bedtools bamtofastq -i $f -fq unmapped_fastq/$ff$R1 -fq2 unmapped_fastq/$ff$R2\n",
    "done\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align the Viral Genome to the left over reads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create the alignment.csv file**\n",
    "\n",
    "```\n",
    "(echo 'samplename,description'; for f in raw/*fq; do readlink -f $f | perl -pe 's/(.*?_(S[0-9]+)_.*)/\\1,\\2/'; done) > alignment.csv\n",
    "```\n",
    "\n",
    "**Prepare the O2.yaml file**\n",
    "\n",
    "```\n",
    "cd ..\n",
    "vim O2.yaml\n",
    "\n",
    "details:\n",
    "  - analysis: RNA-seq\n",
    "    genome_build: MN908947.3\n",
    "    algorithm:\n",
    "      aligner: star\n",
    "      strandedness: unstranded\n",
    "upload:\n",
    "  dir: ../final\n",
    "```\n",
    "\n",
    "**Intiate bcBio**\n",
    "\n",
    "```\n",
    "module load bcbio/latest\n",
    "unset PYTHONPATH\n",
    "bcbio_nextgen.py -w template O2.yaml alignment.csv raw/\n",
    "```\n",
    "\n",
    "**Create Submission script to O2**\n",
    "\n",
    "```\n",
    "vim submit_bcbio.sh\n",
    "\n",
    "#!/bin/sh\n",
    "#SBATCH -p priority\n",
    "#SBATCH -J pickseq\n",
    "#SBATCH -o run.o\n",
    "#SBATCH -e run.e\n",
    "#SBATCH -t 10-00:00\n",
    "#SBATCH --cpus-per-task=20\n",
    "#SBATCH --mem=150G\n",
    "#SBATCH --mail-type=END         # Type of email notification- BEGIN,END,FAIL,ALL\n",
    "#SBATCH --mail-user=ajitj_nirmal@dfci.harvard.edu   # Email to which notifications will be sent\n",
    "\n",
    "export PATH=/n/app/bcbio/tools/bin:$PATH\n",
    "bcbio_nextgen.py ../config/alignment.yaml \\\n",
    "    -n 24 -t local\n",
    "```\n",
    "\n",
    "\n",
    "**Submit job to O2 for processing**\n",
    "\n",
    "```\n",
    "cp submit_bcbio.sh alignment/work\n",
    "cd alignment/work\n",
    "sbatch submit_bcbio.sh\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}