Skip to content

Deploy Config

Varun Patil edited this page Jun 26, 2019 · 1 revision

git repos

https://github.com/wncc/IITBapp cloned at /home/ubuntu/instiapp
https://github.com/go-xunk/iitb-app-deploy cloned at /var/www/instiapp

virtualenv

Located at /home/ubuntu/instiapp/instienv. Dependencies installed initially with pip.

uwsgi

uwsgi is installed globally with nginx.

/etc/rc.local

/usr/local/bin/uwsgi --emperor /etc/uwsgi/vassals --uid www-data --gid www-data --daemonize /var/log/uwsgi/uwsgi-emperor.log

/etc/uwsgi/vassals/uwsgi.ini

This file is a link to /home/ubuntu/instiapp/uwsgi.ini

# uwsgi.ini file
[uwsgi]

# Django-related settings
# the base directory (full path)
chdir           = /home/ubuntu/instiapp
# Django's wsgi file
module          = backend.wsgi
# the virtualenv (full path)
home            = /home/ubuntu/instiapp/instienv

# process-related settings
# master
master          = true
# maximum number of worker processes
processes       = 6
# the socket (use the full path to be safe
socket          = /var/www/socket/instiapp.sock
# ... with appropriate permissions - may be needed
# chmod-socket    = 664
# clear environment on exit
vacuum          = true

celery

Celery is automatically installed in virtualenv. RabbitMQ to be installed beforehand with apt on Ubuntu or similar.

/etc/systemd/system/celery.service

[Unit]
Description=Celery Service
After=network.target

[Service]
Type=forking
User=ubuntu
Group=ubuntu
EnvironmentFile=/etc/conf.d/celery
WorkingDirectory=/home/ubuntu/instiapp
ExecStart=/bin/bash -c '${CELERY_BIN} multi start ${CELERYD_NODES} \
  -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
  --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'
ExecStop=/bin/bash -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \
  --pidfile=${CELERYD_PID_FILE}'
ExecReload=/bin/bash -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \
  -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
  --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'

[Install]
WantedBy=multi-user.target

/etc/conf.d/celery

CELERYD_NODES="w1"

# Absolute or relative path to the 'celery' command:
CELERY_BIN="/home/ubuntu/instiapp/instienv/bin/celery"

# App instance to use
CELERY_APP="backend"

# Django production settings
DJANGO_SETTINGS_MODULE="backend.settings_prod"

# How to call manage.py
CELERYD_MULTI="multi"

# Extra command-line arguments to the worker
CELERYD_OPTS="--time-limit=300 --concurrency=4"

# - %n will be replaced with the first part of the nodename.
# - %I will be replaced with the current child process index
#   and is important when using the prefork pool to avoid race conditions.
CELERYD_PID_FILE="/var/run/celery/%n.pid"
CELERYD_LOG_FILE="/var/log/celery/%n%I.log"
CELERYD_LOG_LEVEL="INFO"

PostgreSQL

Installed locally with any username-password combination and one database set in settings_prod.py. Run migrations after setting database initially or restore from old dump.

Webhook

/home/ubuntu/webhook/redeploy-iitb.sh

cd /home/ubuntu/instiapp
sudo bash << EOF
source instienv/bin/activate
git pull
pip install -r requirements.txt
python manage.py migrate --settings=backend.settings_prod
python manage.py collectstatic --settings=backend.settings_prod --noinput
touch uwsgi.ini
service celery restart
service nginx restart
EOF

/home/ubuntu/webhook/angular-redeploy.sh

cd /var/www/instiapp
sudo git pull

nginx

Custom built with image-filter and pagespeed modules.

/etc/nginx/sites-available/api.insti.app.conf

upstream instiapi {
    server unix:///var/www/socket/instiapp.sock;
}

# configuration of the server
server {
    server_name api.insti.app;
    charset     utf-8;

    client_max_body_size 2M;   # adjust to taste

    root /var/www/instiapp;

    index prerender.html;

     location = / {
        uwsgi_pass  instiapi;
        include     /home/ubuntu/instiapp/uwsgi_params;
     }

     location = /robots.txt {
        return 200 'User-agent: *\nAllow: /static/\nDisallow: /';
        add_header Content-Type text/plain;
     }

     location / {
        try_files $uri $uri/ @apiproxy;
     }

    # Enable CORS for locations
    location = /api/locations {
        add_header 'Access-Control-Allow-Origin' '*';
        uwsgi_pass  instiapi;
        include     /home/ubuntu/instiapp/uwsgi_params;
    }

    location /static/upload/ {
        expires 30d;
        add_header Pragma public;
        add_header Cache-Control "public, immutable";
    }

    location @apiproxy {
        expires -1;
        pagespeed off;
        uwsgi_pass  instiapi;
        include     /home/ubuntu/instiapp/uwsgi_params;
    }

    listen 443 ssl http2; # managed by Certbot
    listen [::]:443 ssl;
    ssl_certificate /etc/letsencrypt/live/api.insti.app/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/api.insti.app/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}


server {
    if ($host = api.insti.app) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


    listen      80;
    listen [::]:80;
    server_name api.insti.app;
    return 404; # managed by Certbot


}

/etc/nginx/sites-available/insti.app.conf

server {
    server_name insti.app;
    root /var/www/instiapp;
    include     /home/ubuntu/instiapp/uwsgi_params;

    location = /robots.txt {
        return 200 'User-agent: *\nAllow: /';
        add_header Content-Type text/plain;
    }

    location = /feedback {
        return 301 https://goo.gl/forms/ne3uMishA9zcn6aN2;
    }

    location = /android {
        return 301 https://play.google.com/store/apps/details?id=app.insti;
    }

    location /.git {
        return 404;
    }

    location = /sitemap.xml {
        uwsgi_pass  instiapi;
    }

    location /api/ {
        expires -1;
        pagespeed off;
        uwsgi_pass  instiapi;
        include     /home/ubuntu/instiapp/uwsgi_params;
    }

    location = /index.html {
        http2_push /assets/petal.svg;
    }

    location / {
        if ($http_user_agent ~* "googlebot|twitterbot|bingbot|msnbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator|WhatsApp") {
            set $bot T;
        }

        if ($request_uri !~* "/assets/") {
            set $bot  "${bot}U";
        }

        if ($bot = TU) {
            uwsgi_pass  instiapi;
            break;
        }

        root /var/www/instiapp;
        expires 7d;
        error_page 404 =200 /index.html;
    }

    index index.html;

    listen 443 ssl; # managed by Certbot
    listen [::]:443 ssl;
    ssl_certificate /etc/letsencrypt/live/api.insti.app/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/api.insti.app/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}

server {
    if ($host = insti.app) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


    server_name insti.app;

    listen 80;
    listen [::]:80;
    return 404; # managed by Certbot


}

server {
    server_name www.insti.app;
    return 301 https://insti.app$request_uri;


    listen 443 ssl; # managed by Certbot
    listen [::]:443 ssl;
    ssl_certificate /etc/letsencrypt/live/www.insti.app/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/www.insti.app/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}



server {
    if ($host = www.insti.app) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


    server_name www.insti.app;
    listen 80;
    listen [::]:80;
    return 404; # managed by Certbot


}

/etc/nginx/sites-available/img.insti.app.conf

proxy_cache_path /var/www/cache/resized levels=1:2 keys_zone=resizedimages:50m max_size=1G inactive=15d;
proxy_cache_path /var/www/cache/profile levels=1:2 keys_zone=profilepics:10m max_size=200M inactive=30d;

resolver 8.8.8.8;

server {
    server_name img.insti.app;
    charset     utf-8;

    proxy_cache resizedimages;
    proxy_cache_valid 200 301 302 15d;
    proxy_cache_valid any 120m;
    proxy_cache_use_stale error timeout invalid_header updating;

    expires 30d;
    add_header Pragma public;
    add_header Cache-Control "public";
    add_header Cache-Control "immutable";
    add_header X-Cache-Status $upstream_cache_status;

    root /var/www/instiapp;

    image_filter_jpeg_quality   80;
    image_filter_buffer         20M;
    image_filter_interlace      on;

    location ~ ^/static/crop/(.*) {
        proxy_pass            https://img.insti.app/internal-static-crop/$1;
    }

    location ~ ^/static/(.*) {
        proxy_pass            https://img.insti.app/internal-static/$1;
    }

    location ~ ^/profile/(.*) {
        proxy_pass            https://img.insti.app/internal-profile/$1;
    }

    location ~ ^/yt/(.*) {
        proxy_pass            https://img.insti.app/internal-youtube/$1;
    }

    location ~ ^/internal-static/(\d+)/(.*) {
        try_files /static/$2 =401;
        image_filter                resize $1 $1;
    }

    location ~ ^/internal-static-crop/(\d+)/(\d+)/(.*) {
        try_files /static/$3 =401;
        image_filter                crop $1 $2;
    }

    location ~ ^/internal-profile/(.*) {
        proxy_pass                  https://gymkhana.iitb.ac.in/sso/media/profile_picture/$1;
        proxy_cache profilepics;
        proxy_cache_valid 200 301 302 30d;
        image_filter                crop 256 256;
    }

    location ~ ^/internal-youtube/(.*) {
        proxy_pass                  https://img.youtube.com/vi/$1/maxresdefault.jpg;
        proxy_intercept_errors on;
        error_page 404 = @ythq;
        proxy_cache profilepics;
        proxy_cache_valid 200 301 302 30d;
        image_filter                crop 512 288;
    }

    location @ythq {
        proxy_pass                  https://img.youtube.com/vi/$1/hqdefault.jpg;
        proxy_cache profilepics;
        proxy_cache_valid 200 301 302 30d;
        image_filter                crop 512 288;
    }

    listen 443 ssl http2; # managed by Certbot
    listen [::]:443 ssl http2;

    ssl_certificate /etc/letsencrypt/live/img.insti.app/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/img.insti.app/privkey.pem; # managed by Certbot
}


server {
    if ($host = img.insti.app) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


    listen      80;
    listen [::]:80;
    server_name img.insti.app;
}

cron

crontab for ubuntu

49 * * * * /home/ubuntu/cron/instiapp_chores >> /home/ubuntu/cron/log 2>&1
*/2 * * * * /home/ubuntu/cron/instiapp_chores_5 >> /home/ubuntu/cron/log 2>&1
30 2 * * * /home/ubuntu/cron/instiapp_chores_daily >> /home/ubuntu/cron/log 2>&1
0 */6 * * * /home/ubuntu/cron/ping_google >> /home/ubuntu/cron/log 2>&1
3 3 * * * /home/ubuntu/cron/flush

/home/ubuntu/cron/instiapp_chores

#!/bin/bash
cd /home/ubuntu/instiapp
LDAP_USERNAME=XYZ && export LDAP_USERNAME
LDAP_PASSWORD=XYZ && export LDAP_PASSWORD
source instienv/bin/activate

echo "Slow chores started at `date`"
python manage.py news_chore --settings backend.settings_prod
python manage.py mess_chore --settings backend.settings_prod
python manage.py clearsessions --settings backend.settings_prod
echo -e "Slow chores completed at `date`\n"

/home/ubuntu/cron/instiapp_chores_5

#!/bin/bash
cd /home/ubuntu/instiapp
LDAP_USERNAME=XYZ && export LDAP_USERNAME
LDAP_PASSWORD=XYZ && export LDAP_PASSWORD
source instienv/bin/activate

echo "Fast chores started at `date`"
python manage.py placement_blog_chore --settings backend.settings_prod
python manage.py notify-event-starting --settings backend.settings_prod
echo -e "Fast chores completed at `date`\n"

/home/ubuntu/cron/instiapp_chores_daily

#!/bin/bash
cd /home/ubuntu/instiapp
source instienv/bin/activate

echo "Daily chores started at `date`"
python manage.py clearsessions --settings backend.settings_prod
python manage.py clean-devices --settings backend.settings_prod
python manage.py clean-images --settings backend.settings_prod
python manage.py make-map-thumbs --settings backend.settings_prod
echo -e "Daily chores completed at `date`\n"

/home/ubuntu/cron/ping_google

#!/bin/bash
echo "Trying ping at `date`"
curl "https://google.com/ping?sitemap=https://insti.app/sitemap.xml"
echo -e "Ping completed at `date`\n"