tasks.py

import json
import os
import subprocess

import git
from invoke import task
# from slacker import Slacker

from webservices.env import env
from jdbc_utils import to_jdbc_url


DEFAULT_FRACTION = 0.5
FULL_TABLES = [
    'cand_inactive',
]
EXCLUDE_TABLES = [
    '*_mv',
    '*_tmp',
    '*_old',
    'sched_b2',
    'sched_e2',
    'pacronyms',
    'ofec_*',
    'dimcandproperties',
    'f_rpt_or_form_sub',
]
# Include records used in integration tests
# FORCE_INCLUDE = [
#     ('dimcand', 10025229),  # Nancy Pelosi
#     ('dimcand', 10012694),  # John Boehner
#     ('dimcmte', 10031117),  # Raul Grijalva (committee)
# ]


@task
def fetch_schemas(ctx, source, dest):
    cmd = 'pg_dump {0} --format c --schema-only --no-acl --no-owner'.format(source)
    for table in (FULL_TABLES + EXCLUDE_TABLES):
        cmd += ' --exclude-table {0}'.format(table)
    cmd += ' | pg_restore --dbname {0} --no-acl --no-owner'.format(dest)
    ctx.run(cmd, echo=True)


@task
def fetch_full(ctx, source, dest):
    cmd = 'pg_dump {0} --format c --no-acl --no-owner'.format(source)
    for table in FULL_TABLES:
        cmd += ' --table {0}'.format(table)
    cmd += ' | pg_restore --dbname {0} --no-acl --no-owner'.format(dest)
    ctx.run(cmd, echo=True)


@task
def fetch_subset(ctx, source, dest, fraction=DEFAULT_FRACTION, log=True):
    cmd = 'rdbms-subsetter {source} {dest} {fraction}'.format(**locals())
    if log:
        cmd += ' --logarithmic'
    for table in (FULL_TABLES + EXCLUDE_TABLES):
        cmd += ' --exclude-table {0}'.format(table)
    for table, key in FORCE_INCLUDE:
        cmd += ' --force {0}:{1}'.format(table, key)
    cmd += ' --config data/subset-config.json'
    cmd += ' --yes'
    ctx.run(cmd, echo=True)


@task
def build_test(ctx, source, dest, fraction=DEFAULT_FRACTION, log=True):
    fetch_full(ctx, source, dest)
    fetch_schemas(ctx, source, dest)
    fetch_subset(ctx, source, dest, fraction=fraction, log=log)


@task
def dump(ctx, source, dest):
    cmd = 'pg_dump {source} --format c --no-acl --no-owner -f {dest}'.format(**locals())
    for table in EXCLUDE_TABLES:
        cmd += ' --exclude-table {0}'.format(table)
    ctx.run(cmd, echo=True)


@task
def add_hooks(ctx):

    ctx.run('ln -s ../../bin/post-merge .git/hooks/post-merge')
    ctx.run('ln -s ../../bin/post-checkout .git/hooks/post-checkout')


@task
def remove_hooks(ctx):
    ctx.run('rm .git/hooks/post-merge')
    ctx.run('rm .git/hooks/post-checkout')


def _detect_prod(repo, branch):
    """Deploy to production if master is checked out and tagged."""
    if branch != 'master':
        return False
    try:
        # Equivalent to `git describe --tags --exact-match`
        repo.git().describe('--tags', '--exact-match')
        return True
    except git.exc.GitCommandError:
        return False


def _resolve_rule(repo, branch):
    """Get space associated with first matching rule."""
    for space, rule in DEPLOY_RULES:
        if rule(repo, branch):
            return space
    return None


def _detect_branch(repo):
    try:
        return repo.active_branch.name
    except TypeError:
        return None


def _detect_space(repo, branch=None, yes=False):
    """Detect space from active git branch.

    :param str branch: Optional branch name override
    :param bool yes: Skip confirmation
    :returns: Space name if space is detected and confirmed, else `None`
    """
    space = _resolve_rule(repo, branch)
    if space is None:
        print('No space detected')
        return None
    print('Detected space {space}'.format(**locals()))
    if not yes:
        run = input(
            'Deploy to space {space} (enter "yes" to deploy)? > '.format(**locals())
        )
        if run.lower() not in ['y', 'yes']:
            return None
    return space


DEPLOY_RULES = (
    ('prod', _detect_prod),
    ('stage', lambda _, branch: branch.startswith('release')),
    ('dev', lambda _, branch: branch == 'develop'),
)


SPACE_URLS = {
    'dev': [('app.cloud.gov', 'fec-dev-api')],
    'stage': [('app.cloud.gov', 'fec-stage-api')],
    'prod': [('app.cloud.gov', 'fec-prod-api')],
}


@task
def deploy(ctx, space=None, branch=None, login=None, yes=False):
    """Deploy app to Cloud Foundry. Log in using credentials stored per environment
    like `FEC_CF_USERNAME_DEV` and `FEC_CF_PASSWORD_DEV`; push to either `space` or t
    he space detected from the name and tags of the current branch. Note: Must pass `space`
    or `branch` if repo is in detached HEAD mode, e.g. when running on Travis.
    """
    # Detect space
    repo = git.Repo('.')
    branch = branch or _detect_branch(repo)
    space = space or _detect_space(repo, branch, yes)
    if space is None:
        return

    # Set api
    api = 'https://api.fr.cloud.gov'
    ctx.run('cf api {0}'.format(api), echo=True)

    # Log in if necessary
    if login == 'True':
        login_command = 'cf auth "$FEC_CF_USERNAME_{0}" "$FEC_CF_PASSWORD_{0}"'.format(space.upper())
        ctx.run(login_command, echo=True)

    # Target space
    ctx.run('cf target -o fec-beta-fec -s {0}'.format(space), echo=True)

    print("\nMigrating database...")
    jdbc_url = to_jdbc_url(os.getenv('FEC_MIGRATOR_SQLA_CONN_{0}'.format(space.upper())))
    run_migrations(ctx, jdbc_url)
    print("Database migrated\n")

    # Set deploy variables
    with open('.cfmeta', 'w') as fp:
        json.dump({'user': os.getenv('USER'), 'branch': branch}, fp)

    # Deploy API and worker applications
    for app in ('api', 'celery-worker', 'celery-beat'):
        deployed = ctx.run('cf app {0}'.format(app), echo=True, warn=True)
        cmd = 'zero-downtime-push' if deployed.ok else 'push'
        ctx.run('cf {cmd} {app} -f manifests/manifest_{file}_{space}.yml'.format(
            cmd=cmd,
            app=app,
            file=app.replace('-','_'),
            space=space
        ), echo=True)


@task
def notify(ctx):
    try:
        meta = json.load(open('.cfmeta'))
    except OSError:
        meta = {}
    slack = Slacker(env.get_credential('FEC_SLACK_TOKEN'))
    slack.chat.post_message(
        env.get_credential('FEC_SLACK_CHANNEL', '#fec'),
        'deploying branch {branch} of app {name} to space {space} by {user}'.format(
            name=env.name,
            space=env.space,
            user=meta.get('user'),
            branch=meta.get('branch'),
        ),
        username=env.get_credential('FEC_SLACK_BOT', 'fec-bot'),
    )

@task
def create_sample_db(ctx):
    """
    Load schema and data into the empty database pointed to by $SQLA_SAMPLE_DB_CONN
    """

    print("Loading schema...")
    db_conn = os.getenv('SQLA_SAMPLE_DB_CONN')
    jdbc_url = to_jdbc_url(db_conn)
    run_migrations(ctx, jdbc_url)
    print("Schema loaded")
    print("Loading sample data...")
    subprocess.check_call(
        ['psql', '-v', 'ON_ERROR_STOP=1', '-f', 'data/sample_db.sql', db_conn],
    )
    print("Sample data loaded")

@task
def run_migrations(ctx, jdbc_url):
    ctx.run('flyway migrate -q -url="{0}" -locations=filesystem:data/migrations'.format(jdbc_url))