diff --git a/instance-data/vault.sh.tpl b/instance-data/vault.sh.tpl new file mode 100644 index 0000000..5c330ea --- /dev/null +++ b/instance-data/vault.sh.tpl @@ -0,0 +1,372 @@ +#!/bin/bash +set -x +source /opt/ivy/bash_functions.sh + +set_ivy_tag '__IVY_TAG__' + +### +### TEMP and should be in ami-bakery ## +### + +function trust_sysenv_ca() { + local DISTRO="$(grep '^NAME=' /etc/os-release | cut -d '"' -f2)" + local SSM_CA_CERTIFICATE="/$(get_ivy_tag)/$(get_environment)/CA/ca.pem" + local REGION="${1:-$(get_region)}" + case "${DISTRO}" in + Amazon Linux) + local CA_TRUST_DIR='/etc/pki/ca-trust/source/anchors/' + local UPDATE_CA_COMMAND='update-ca-trust extract' + ;; + Ubuntu) + local CA_TRUST_DIR='/usr/local/share/ca-certificates/' + local UPDATE_CA_COMMAND='update-ca-certificates' + ;; + *) + echo "Only Amazon Linux and Ubuntu are supported at the moment" >&2 + return 1 + ;; + esac + local CA_CRT="${CA_TRUST_DIR}/ivy.pem" + get_ssm_param "${SSM_CA_CERTIFICATE}" "${REGION}" > "${CA_CRT}" + sudo ${UPDATE_CA_COMMAND} +} + +### +### CONFIG ### +### +SERVICE='Vault' +AWS_REGION="$(get_region)" +SLEEP=20 +SPLAY=$(shuf -i 1-10 -n 1) +INSTANCE_ID="$(get_instance_id)" +SSM_PREFIX="/$(get_ivy_tag)/$(get_environment)" +SSM_CA_KEY="${SSM_PREFIX}/CA/ca-key.pem" +# Filled by Cloudformation +ENI_ID='{#CFN_ENI_ID}' +KMS_KEY='{#VaultKMSUnseal}' +VAULT_CLIENT_ROLE_NAME='{#VAULT_CLIENT_ROLE_NAME}' +VAULT_CLIENT_ROLE='{#VAULT_CLIENT_ROLE}' +# Filled by Rain +ENI_IP="__ENI_IP__" +SERVER_ID="__SERVER_ID__" +HOSTS_ENTRIES="__HOSTS_ENTRIES__" +SSM_CA_REGION="__CA_REGION__" +VAULT_SECRET="__VAULT_SECRET__" +VAULT_NUMBER_OF_KEYS=5 +VAULT_NUMBER_OF_KEYS_FOR_UNSEAL=3 +NODE_NAME="vault-master-${SERVER_ID}.node.$(get_environment).$(get_ivy_tag)" + +function setup_vault_systemctl() { + systemctl daemon-reload + systemctl enable vault + systemctl start vault +} + +function generate_keys_certs() { + local DISTRO="$(grep '^NAME=' /etc/os-release | cut -d '"' -f2)" + case "${DISTRO}" in + Amazon Linux) + local CA_KEY_DIR='/etc/pki/CA/private/' + local UPDATE_CA_COMMAND='update-ca-trust extract' + ;; + Ubuntu) + local CA_TRUST_DIR='/etc/ssl/private/' + local UPDATE_CA_COMMAND='update-ca-certificates' + ;; + *) + echo "Only Amazon Linux and Ubuntu are supported at the moment" >&2 + return 1 + ;; + esac + local CA_KEY="${CA_KEY_DIR}/ca.pem" + get_ssm_param "${SSM_CA_KEY}" '--with-decryption' "${SSM_CA_REGION}" > "${CA_KEY}" + cd "${CA_KEY_DIR}" + openssl genrsa -out "${NODE_NAME}.key" 2048 + openssl req -new -key "${NODE_NAME}.key" -out "${NODE_NAME}.csr" +} + +function setup_vault_leader() { + # Do leader stuff + echo 'I was elected leader doing leader stuff' + sleep ${SLEEP} + echo 'done' + + setup_vault_systemctl + + until curl -fs -o /dev/null localhost:8200/v1/sys/init; do + echo 'Waiting for Vault to start...' + sleep 1 + done + + init=$(curl -fs localhost:8200/v1/sys/init | jq -r .initialized) + + if [ "${init}" == "false" ]; then + echo 'Initializing Vault' + install -d -m 0755 -o vault -g vault /etc/vault + SECRET_VALUE=$(vault operator init -recovery-shares=${VAULT_NUMBER_OF_KEYS} -recovery-threshold=${VAULT_NUMBER_OF_KEYS_FOR_UNSEAL}) + echo 'storing vault init values in secrets manager' + aws secretsmanager put-secret-value --region ${AWS_REGION} --secret-id ${VAULT_SECRET} --secret-string "${SECRET_VALUE}" + else + echo "Vault is already initialized" + fi + + sealed=$(curl -fs localhost:8200/v1/sys/seal-status | jq -r .sealed) + + VAULT_SECRET_VALUE=$(get_secret ${VAULT_SECRET}) + + root_token=$(echo ${VAULT_SECRET_VALUE} | awk '{ if (match($0,/Initial Root Token: (.*)/,m)) print m[1] }' | cut -d " " -f 1) + # Handle a variable number of unseal keys + for UNSEAL_KEY_INDEX in {1..${VAULT_NUMBER_OF_KEYS_FOR_UNSEAL}}; do + unseal_key+=($(echo ${VAULT_SECRET_VALUE} | awk '{ if (match($0,/Recovery Key '${UNSEAL_KEY_INDEX}': (.*)/,m)) print m[1] }'| cut -d " " -f 1)) + done + + # Should Auto unseal using KMS but this is for demonstration for manual unseal + if [ "$sealed" == "true" ]; then + echo "Unsealing Vault" + # Handle variable number of unseal keys + for UNSEAL_KEY_INDEX in {1..${VAULT_NUMBER_OF_KEYS_FOR_UNSEAL}}; do + vault operator unseal $unseal_key[${UNSEAL_KEY_INDEX}] + done + else + echo "Vault is already unsealed" + fi + + sleep ${SLEEP} + + # Login to Vault + vault login token=$root_token 2>&1 > /dev/null # Hide this output from the console + + # Enable Vault audit logs + vault audit enable file file_path=/var/log/vault/vault-audit.log + + # Enable AWS Auth + vault auth enable aws + + # Enable pki secrets engine + vault secrets enable pki + + # pki secrets engine to issue certificates with a maximum time-to-live (TTL) of 87600 hours + vault secrets tune -max-lease-ttl=87600h pki + + # Create client-role-iam role + vault write auth/aws/role/${VAULT_CLIENT_ROLE_NAME} auth_type=iam \ + bound_iam_principal_arn=${VAULT_CLIENT_ROLE} \ + policies=vaultclient \ + ttl=24h + + # Take a consul snapshot + consul snapshot save postinstall-consul.snapshot +} + +function setup_vault_member() { + while true; do + echo "Sleeping ${SLEEP} seconds to allow leader to bootstrap: " + sleep ${SLEEP} + echo 'done' + + echo -n 'Checking the cluster members to see if I am allowed to bootstrap: ' + # Check if my instance id exists in the list + echo "${HOSTS_ENTRIES}" + HOSTS_IPS=( $(awk '{ print $1 }' <<< "${HOSTS_ENTRIES}") ) + echo "${HOSTS_ENTRIES}" | grep "${ENI_IP}" + I_CAN_BOOTSTRAP=$? # Check exit status of grep command + if [ $I_CAN_BOOTSTRAP -eq 0 ]; then + UNHEALTHY_COUNT=0 + # Check each node in the cluster is okay (Except myself) + for i in "${HOSTS_IPS[@]}"; do + if [ ${i} != ${ENI_IP} ]; then # Don't check ourselves since we have not joined; then + status=$(curl -s "http://${i}:8200/v1/sys/init" | jq -r .initialized) + # increment counter if a node is initialized + if [ "$status" != true ]; then + ((++UNHEALTHY_COUNT)) + fi + fi + done + + #if [ $UNHEALTHY_COUNT -eq 0 ]; then + echo "I am a cluster member now and all nodes healthy start vault" + break + #fi + fi + echo "I am NOT a cluster member or other nodes unhealthy trying again..." + done + + setup_vault_systemctl + + # Don't signal until we report that we have started + until curl -fs -o /dev/null localhost:8200/v1/sys/init; do + echo "Waiting for Vault to start..." + sleep 2 + done + + # Don't signal until we are unsealed + while true; do + sealed=$(curl -fs localhost:8200/v1/sys/seal-status | jq -r .sealed) + echo -n "Making sure vault is unsealed..." + if [ $sealed != "false" ]; then + echo " sealed sleep 2" + sleep 2 + else + echo " unsealed signal success" + break + fi + done +} + +function setup_vault() { + # hard set hosts for vault to prevent DNS failure from exploding the world + echo "${HOSTS_ENTRIES}" >> /etc/hosts + + cat << EOF > /etc/vault.d/vault.hcl +storage "consul" { + address = "127.0.0.1:8500" + path = "vault/" +} + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + tls_disable = true +} + +seal "awskms" { + region = "${AWS_REGION}" + kms_key_id = "${KMS_KEY}" +} + +api_addr = "http://${ENI_IP}:8200" +cluster_addr = "http://${ENI_IP}:8201" +ui = true +EOF + + cat << EOF > /etc/environment +export VAULT_ADDR=http://127.0.0.1:8200 +export VAULT_SKIP_VERIFY=true +EOF + + chown vault: /etc/vault.d/vault.hcl + + . /etc/environment + + # Start consul (as a master) first! + bash /opt/ivy/configure_consul.sh master + + # So each node doesn't start same time spread out the starts + echo "Sleeping for a splay time: ${SPLAY}" + sleep ${SPLAY} + + # Check for leader + while true; do + echo -n "Sleeping for ${SLEEP} seconds to allow for election:" + sleep ${SLEEP} + echo "done" + + echo -n 'Checking if leader election has happened:' + LEADER_ELECTED=$(consul operator raft list-peers 2>&1 | grep leader) + echo "${LEADER_ELECTED}" + if consul operator raft list-peers 2>&1 | grep leader &> /dev/null; then + echo "Leader has been elected continue bootstrapping" + break + fi + echo "No leader elected trying again..." + done + + echo -n 'Am I the elected leader:' + LEADER="$(consul info | grep 'leader =' | awk '{ print $3 }')" + echo "${LEADER}" + + # If I am the leader do the leader bootstrap stuff + if [ "${LEADER}" = "true" ]; then + # Note: function below exits this process + setup_vault_leader + fi + + # Only Vault cluster members are here so + sleep ${SLEEP} + echo "Checking if I am able to bootstrap further: " + setup_vault_member +} + +function setup_kubernetes_master() { + +} + +function setup_datadog() { + # setup datadog + cat < /etc/datadog-agent/conf.d/vault.d/conf.yaml +init_config: + +instances: + - api_url: http://vault.service.$(get_ivy_tag):8200/v1 +EOF + + service datadog-agent restart +} + +function generate_consul_tokens() { + echo 'Disabling bash tracing mode to avoid logging token values' + set +x + declare -A CONSUL_TOKENS + for token in 'CONSUL_ADMIN_TOKEN' 'CONSUL_AGENT_TOKEN' 'CONSUL_ENCRYPT_KEY' 'CONSUL_MASTER_TOKEN' 'CONSUL_REGISTRATOR_TOKEN' 'CONSUL_VAULT_TOKEN'; do + "${CONSUL_TOKENS[${token}]}"="$(uuidgen | tr '[:upper:]' '[:lower:]')" + aws secretsmanager put-secret-value --region ${AWS_REGION} --secret-id "${token}" --secret-string "${CONSUL_TOKENS[${token}]}" + done + set -x +} + +function bootstrap_consul_acl() { + until curl 'http://localhost:8500/v1/status/leader' -s --fail; do + sleep `shuf -i 2-15 -n 1` + done + + if [ $(curl --retry 10 --silent --fail "http://localhost:8500/v1/acl/info/${CONSUL_MASTER_TOKEN}?token=${CONSUL_ADMIN_TOKEN}") == "[]" ]; then + # MesosMaster token doesn't exist, add it. + curl "http://localhost:8500/v1/acl/create?token=${CONSUL_ADMIN_TOKEN}" -X PUT --data '{"ID":"'${CONSUL_MASTER_TOKEN}'","Name":"MesosMaster Token","Type":"client","Rules":"# Mesos Master token\n\n# Write to all KV\nkey \"\" {\n policy = \"write\"\n}\n\n# No access to vault (protected) secrets\nkey \"vault/\" {\n policy = \"deny\"\n}\n\n# Write access to reply to exec commands\nkey \"_rexec/\" {\n policy = \"write\"\n}\n\n# Register any service (semi insecure, but necessary)\nservice \"\" {\n policy = \"write\"\n}\n\n# Allow vault service registration\nservice \"vault\" {\n policy = \"write\"\n}\n\n# Broadcast any event\nevent \"\" {\n policy = \"write\"\n}\n\n# Read exec commands, but not launch them\nevent \"_rexec\" {\n policy = \"read\"\n}"}' + fi + + if [ $(curl --retry 10 --silent --fail "http://localhost:8500/v1/acl/info/${CONSUL_VAULT_TOKEN}?token=${CONSUL_ADMIN_TOKEN}") == "[]" ]; then + # Vault token doesn't exist, add it. + curl "http://localhost:8500/v1/acl/create?token=${CONSUL_ADMIN_TOKEN}" -X PUT --data '{"ID":"'${CONSUL_VAULT_TOKEN}'","Name":"Vault Token","Type":"client","Rules":"# Token used by Vault itself to store secure data\n\n# Read/write access to vault (protected) data\nkey \"vault/\" {\n policy = \"write\"\n}"}' + fi + + if [ $(curl --retry 10 --silent --fail "http://localhost:8500/v1/acl/info/${CONSUL_AGENT_TOKEN}?token=${CONSUL_ADMIN_TOKEN}") == "[]" ]; then + # Agent token doesn't exist, add it. + curl "http://localhost:8500/v1/acl/create?token=${CONSUL_ADMIN_TOKEN}" -X PUT --data '{"ID":"'${CONSUL_AGENT_TOKEN}'","Name":"Agent Token","Type":"client","Rules":"# Generic agent token, used by all consul agents (mesos agents, etc)\n\n# Write to all KV\nkey \"\" {\n policy = \"write\"\n}\n\n# No access to vault (protected) secrets\nkey \"vault/\" {\n policy = \"deny\"\n}\n\n# Write access to reply to exec commands\nkey \"_rexec/\" {\n policy = \"write\"\n}\n\n# Register any service (semi insecure, but necessary)\nservice \"\" {\n policy = \"write\"\n}\n\n# Read only Vault service\nservice \"vault\" {\n policy = \"read\"\n}\n\n# Broadcast any event\nevent \"\" {\n policy = \"write\"\n}\n\n# Read exec commands, but not launch them\nevent \"_rexec\" {\n policy = \"read\"\n}"}' + fi + + if [ $(curl --retry 10 --silent --fail "http://localhost:8500/v1/acl/info/${CONSUL_REGISTRATOR_TOKEN}?token=${CONSUL_ADMIN_TOKEN}") == "[]" ]; then + curl "http://localhost:8500/v1/acl/create?token=${CONSUL_ADMIN_TOKEN}" -X PUT --data '{"ID":"'${CONSUL_REGISTRATOR_TOKEN}'","Name":"Registrator Token","Type":"client","Rules":"# ACL token for Registrator on Mesos Agents\n\n# Allow any service registration, except for secured services\nservice \"\" {\n policy = \"write\"\n}\n\n# Registrator cannot register vault\nservice \"vault\" {\n policy = \"read\"\n}\n\n# Registrator can only read KV\nkey \"\" {\n policy = \"read\"\n}\n\n# Deny access to vault from registrator.\n# If registrator tries to read vault, we have a problem.\nkey \"vault/\" {\n policy = \"deny\"\n}\n\n# Deny read access to rexec replies, might include sensitive information\nkey \"_rexec/\" {\n policy = \"deny\"\n}\n\n# Deny access to fire the rexec event too\nevent \"_rexec\" {\n policy = \"deny\"\n}"}' + fi +} + +function setup_consul() { + CONSUL_MASTER_TOKEN_VALUE=$(get_secret ${CONSUL_MASTER_TOKEN}) + CONSUL_AGENT_TOKEN_VALUE=$(get_secret ${CONSUL_AGENT_TOKEN}) + cat < /etc/consul.d/master.json +{ + "performance": { + "raft_multiplier": 1 + }, + "dns_config": { + "allow_stale": true + }, + "tokens": { + "master": "${CONSUL_MASTER_TOKEN_VALUE}", + "agent": "${CONSUL_AGENT_TOKEN_VALUE}", + } +} +EOF + +} + +# Let 'er rip! +attach_eni $(get_instance_id) ${ENI_ID} +set_hostname "vault-master-${SERVER_ID}" +set_prompt_color "__PROMPT_COLOR__" +trust_sysenv_ca "${SSM_CA_REGION}" +setup_datadog +generate_consul_tokens +setup_consul +setup_vault +setup_kubernetes_master + diff --git a/requirements.txt b/requirements.txt index 326312d..80bc189 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ json_tools troposphere==2.4.5 awacs==0.9.6 netaddr +cryptography==3.0 diff --git a/scripts/create-ca/README.md b/scripts/create-ca/README.md new file mode 100644 index 0000000..3217c7c --- /dev/null +++ b/scripts/create-ca/README.md @@ -0,0 +1,42 @@ +# Creates CA and stores it in SSM + +## Requirements for bash version + +- cfssl/cfssljson +- awscli +- AWS permissions: + - SSM read/write + +## Overview + +Here you will find scripts to setup the necessary secrets/certificates for standing up an Ivy environment + +## How to run it + +```shell +$ AWS_PROFILE=sandbox ./generate-seed-files.sh -s sandbox -t ivy -e 10 +Parameter /ivy/sandbox/CA/ca-key.pem does not exist in any region where ssm is available +I will create directories ./ivy/sandbox/CA, CA key and certificate and push them to ssm +2020/08/25 12:18:39 [INFO] generating a new CA key and certificate from CSR +2020/08/25 12:18:39 [INFO] generate received request +2020/08/25 12:18:39 [INFO] received CSR +2020/08/25 12:18:39 [INFO] generating key: rsa-2048 +2020/08/25 12:18:40 [INFO] encoded CSR +2020/08/25 12:18:40 [INFO] signed certificate with serial number 8039204297172209663615822211768421487651832388 +/Users/ricardo/src/infrastructure-ivy-rain/scripts/create-ca +CA_KEY_FILE is at ./ivy/sandbox/CA/ca-key.pem and CA_CERTIFICATE_FILE is at ./ivy/sandbox/CA/ca.pem +{ + "Version": 1, + "Tier": "Standard" +} +{ + "Version": 1, + "Tier": "Standard" +} +``` + +## Related links + +- [Digital Ocean's Vault and Kubernetes](https://www.digitalocean.com/blog/vault-and-kubernetes) +- [Vault's Build Your Own Certificate Authority (CA)](https://learn.hashicorp.com/vault/secrets-management/sm-pki-engine) +- [Be your own certificate authority](https://opensource.com/article/19/4/certificate-authority) diff --git a/scripts/create-ca/generate-seed-files.sh b/scripts/create-ca/generate-seed-files.sh new file mode 100644 index 0000000..e5b767c --- /dev/null +++ b/scripts/create-ca/generate-seed-files.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +THIS_SCRIPT=$(basename $0) +PADDING=$(printf %-${#THIS_SCRIPT}s " ") + +function usage () { + echo "Usage:" + echo "${THIS_SCRIPT} -s -t " + echo "${PADDING} -b " + echo "${PADDING} -e " + echo + echo 'Setup Ivy seed files (Right now only Certificate Authority, privat key and public key)' + exit 1 +} + +function is_parameter_in_ssm() { + local PARAMETER="${1}" + local REGIONS_IN_SSM=( $(aws ssm get-parameters-by-path --path '/aws/service/global-infrastructure/services/ssm/regions' --query 'Parameters[*].Value' --output='text') ) + for region in "${REGIONS_IN_SSM[@]}"; do + if aws --region="${region}" ssm get-parameter --name "${PARAMETER}" &> /dev/null; then + echo "Parameter ${PARAMETER} already exists in region ${region}" >&2 + return 0 + fi + done + echo "Parameter ${PARAMETER} does not exist in any region where ssm is available" >&2 + return 1 +} + +function generate_certificate_authority() { + local CA_DIRECTORY="${1}" + local VALID_IN_HOURS="${2}" + local SYSENV_SHORT_NAME="${3}" + cd "${CA_DIRECTORY}" + + cat > ca-config.json < ca-csr.json <&2 + exit 1 +fi + +while getopts ":s:t:b:e:" opt; do + case ${opt} in + s) + SYSENV_SHORT_NAME="${OPTARG}" ;; + t) + IVY_TAG="${OPTARG}" ;; + b) + BASE_DIRECTORY="${OPTARG}" ;; + e) + VALID_IN_YEARS="${OPTARG}" ;; + \?) + usage ;; + :) + usage ;; + esac +done + +if [[ -z ${SYSENV_SHORT_NAME:-""} ]]; then + usage +fi + +IVY_TAG="${IVY_TAG:-ivy}" +BASE_DIRECTORY="${BASE_DIRECTORY:-.}" +SSM_PREFIX="${IVY_TAG}/${SYSENV_SHORT_NAME}" +SYSENV_DIRECTORY="${BASE_DIRECTORY}/${SSM_PREFIX}" + +# Default to 10 years +VALID_IN_YEARS="${VALID_IN_YEARS:-10}" +DAYS_IN_YEAR='365' +HOURS_IN_DAY='24' +HOURS_IN_YEAR='8760' + +let "VALID_IN_HOURS = ${VALID_IN_YEARS} * ${HOURS_IN_YEAR}" + +CA_DIRECTORY="${SYSENV_DIRECTORY}/CA" +CA_KEY_SSM="/${SSM_PREFIX}/CA/ca-key.pem" +CA_CERTIFICATE_SSM="/${SSM_PREFIX}/CA/ca.pem" +echo "I will check if ${CA_KEY_SSM} is in ssm already or not" +if is_parameter_in_ssm "${CA_KEY_SSM}"; then + echo 'Nothing to do here' +else + echo "I will create directories ${CA_DIRECTORY}, CA key and certificate and push them to ssm" + mkdir -p "${CA_DIRECTORY}" + generate_certificate_authority "${CA_DIRECTORY}" "${VALID_IN_HOURS}" "${SYSENV_SHORT_NAME}" + + aws ssm put-parameter \ + --name "${CA_KEY_SSM}" \ + --type SecureString \ + --value "$(cat ${CA_KEY_FILE})" + + + aws ssm put-parameter \ + --name "${CA_CERTIFICATE_SSM}" \ + --type String \ + --value "$(cat ${CA_CERTIFICATE_FILE})" +fi diff --git a/templates/__init__.py b/templates/__init__.py index 14e357a..d2d4b8b 100644 --- a/templates/__init__.py +++ b/templates/__init__.py @@ -1,4 +1,4 @@ -from templates import vpc, vpn, security_groups, rds, elasticache, cassandra, kafka, pritunl, nexus, mesos_masters, mesos_agents +from templates import vpc, vpn, security_groups, rds, elasticache, cassandra, kafka, pritunl, nexus, mesos_masters, mesos_agents, vault TEMPLATES = { 'VPC': vpc.VPCTemplate, @@ -11,5 +11,6 @@ 'Pritunl': pritunl.PritunlTemplate, 'Nexus': nexus.NexusTemplate, 'MesosMasters': mesos_masters.MesosMastersTemplate, - 'MesosAgents': mesos_agents.MesosAgentsTemplate + 'MesosAgents': mesos_agents.MesosAgentsTemplate, + 'Vault': vault.VaultTemplate } diff --git a/templates/base.py b/templates/base.py index 4b882b6..70722e5 100644 --- a/templates/base.py +++ b/templates/base.py @@ -26,6 +26,7 @@ def __init__(self, template_name, env, params): self.region = constants.ENVIRONMENTS[self.env]['region'] self.sysenv = constants.ENVIRONMENTS[self.env]['sysenv'] self.ec2_conn = boto3.client('ec2', region_name=self.region) + self.sts_conn = boto3.client('sts', region_name=self.region) self.name = self.env + template_name self.template_name = template_name self.tpl_name = template_name.lower() @@ -143,17 +144,39 @@ def get_standard_policies(self): iam.Policy( PolicyName='DescribePermissions', PolicyDocument={ - 'Statement': [{ - 'Effect': 'Allow', - 'Action': [ - 'ec2:DescribeDhcpOptions', - 'ec2:DescribeInstances', - 'ec2:DescribeNetworkInterfaces', - 'ec2:DescribeRegions', - 'ec2:DescribeVpcs' - ], - 'Resource': '*' - }] + 'Statement': [ + { + 'Effect': 'Allow', + 'Action': [ + 'ec2:DescribeDhcpOptions', + 'ec2:DescribeInstances', + 'ec2:DescribeNetworkInterfaces', + 'ec2:DescribeRegions', + 'ec2:DescribeVpcs' + ], + 'Resource': '*' + }, + { + 'Effect': 'Allow', + 'Action': [ + 'ssm:DescribeParameters' + ], + 'Resource': '*' + }, + { + 'Effect': 'Allow', + 'Action': [ + 'ssm:GetParameters' + ], + 'Resource': 'arn:{}:ssm:{}:{}:parameter/{}/{}/CA/ca.pem'.format( + self.get_partition(), + constants.ENVIRONMENTS[self.env].get('ca_region', self.region), + self.get_account_id(), + constants.TAG, + self.env + ) + } + ] } ) ) @@ -250,6 +273,9 @@ def generate_docker_roles(self): def get_partition(self): return self.ec2_conn.meta.partition + def get_account_id(self): + return self.sts_conn.get_caller_identity()['Account'] + def default_sg_name(self, name): return '{}-{}-DefaultSecurityGroup'.format(self.env, name) @@ -313,7 +339,7 @@ def get_subnets(self, _filter=None, _preferred_only=False): """ if _filter not in [None, 'private', 'public']: raise RuntimeError('Filter not one of None, "public", or "private": {}'.format(_filter)) - filter_is_public = True if _filter is 'public' else False + filter_is_public = True if _filter == 'public' else False all_subnets = self.ec2_conn.describe_subnets( Filters=[{'Name': 'vpc-id', 'Values': [self.vpc_id]}])['Subnets'] if _filter: diff --git a/templates/vault.py b/templates/vault.py new file mode 100644 index 0000000..1448c2a --- /dev/null +++ b/templates/vault.py @@ -0,0 +1,283 @@ +from troposphere import autoscaling, ec2, iam, kms, secretsmanager, Base64, GetAtt, Parameter, Ref, Sub + +import netaddr +from config import constants +from .base import IvyTemplate +from utils.ec2 import get_block_device_mapping, get_latest_ami_id + + +class VaultTemplate(IvyTemplate): + CAPABILITIES = ['CAPABILITY_IAM'] + + def configure(self): + """ + This template creates a vault and consul master per subnet in the VPC + """ + config = constants.ENVIRONMENTS[self.env]['vault'] + self.defaults = { + 'instance_type': config.get('instance_type', 't3.large') + } + + self.set_description('Sets up Vault and Consul Masters in all Zones') + self.get_eni_policies() + self.get_default_security_groups() + self.get_standard_parameters() + self.get_standard_policies() + + _global_config = constants.ENVIRONMENTS[self.env] + + self.ami = self.add_parameter( + Parameter( + 'AMI', + Type='String', + Description='AMI ID for instances', + Default=get_latest_ami_id(self.region, 'ivy-vault', _global_config.get('ami_owner', 'self')) + ) + ) + _vault_security_group = self.add_resource( + ec2.SecurityGroup( + 'VaultSecurityGroup', + VpcId=self.vpc_id, + GroupDescription='Security Group for Vault Instances', + SecurityGroupIngress=[ + {'IpProtocol': 'tcp', 'FromPort': 8200, 'ToPort': 8201, 'CidrIp': self.vpc_cidr}, # vault rpc/lan serf + {'IpProtocol': 'udp', 'FromPort': 8200, 'ToPort': 8201, 'CidrIp': self.vpc_cidr}, # vault rpc/lan serf (udp) + {'IpProtocol': 'tcp', 'FromPort': 8500, 'ToPort': 8500, 'CidrIp': self.vpc_cidr}, # consul ui + {'IpProtocol': 'tcp', 'FromPort': 8300, 'ToPort': 8301, 'CidrIp': self.vpc_cidr}, # consul rpc/lan serf + {'IpProtocol': 'tcp', 'FromPort': 8302, 'ToPort': 8302, 'CidrIp': constants.SUPERNET}, # consul wan serf + {'IpProtocol': 'udp', 'FromPort': 8300, 'ToPort': 8301, 'CidrIp': self.vpc_cidr}, # consul rpc/lan serf (udp) + {'IpProtocol': 'udp', 'FromPort': 8302, 'ToPort': 8302, 'CidrIp': constants.SUPERNET}, # consul wan serf (udp) + ], + SecurityGroupEgress=[ + {'IpProtocol': '-1', 'FromPort': 0, 'ToPort': 65535, 'CidrIp': '0.0.0.0/0'} + ] + ) + ) + self.add_resource( + ec2.SecurityGroupIngress( + 'VaultIngressSecurityGroup', + GroupId=Ref(_vault_security_group), + IpProtocol='-1', + FromPort=-1, + ToPort=-1, + SourceSecurityGroupId=Ref(_vault_security_group) + # this allows members all traffic (for replication) + ) + ) + self.add_security_group(Ref(_vault_security_group)) + + _vault_kms_key = kms.Key( + 'VaultKMSUnseal', + Description='Vault unseal key', + PendingWindowInDays=10, + KeyPolicy={ + 'Version': '2012-10-17', + 'Id': 'key-default-1', + 'Statement': [ + { + 'Sid': 'Enable IAM User Permissions', + 'Effect': 'Allow', + 'Principal': { + 'AWS': Sub('arn:${AWS::Partition}:iam::${AWS::AccountId}:root') + }, + 'Action': 'kms:*', + 'Resource': '*' + }, + { + 'Sid': 'Allow administration of the key', + 'Effect': 'Allow', + 'Principal': { + 'AWS': Sub('arn:${{AWS::Partition}}:iam::${{AWS::AccountId}}:role/${{{0}InstanceRole}}'.format(self.name)) + }, + 'Action': [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion' + ], + 'Resource': '*' + }, + { + 'Sid': 'Allow use of the key', + 'Effect': 'Allow', + 'Principal': { + 'AWS': Sub('arn:${{AWS::Partition}}:iam::${{AWS::AccountId}}:role/${{{0}InstanceRole}}'.format(self.name)) + }, + 'Action': [ + 'kms:DescribeKey', + 'kms:Encrypt', + 'kms:Decrypt', + 'kms:ReEncrypt*', + 'kms:GenerateDataKey', + 'kms:GenerateDataKeyWithoutPlaintext' + ], + 'Resource': '*' + } + ] + }, + Tags=self.get_tags( + service_override="Vault" + ) + [ec2.Tag('Name', 'VaultKMSUnseal')] + ) + + self.add_resource(_vault_kms_key) + + _vault_secretsmanager_secret = secretsmanager.Secret( + 'VaultSecret{}'.format(self.env), + Description='Vault Root/Recovery key', + Name='VaultSecret-{}'.format(self.env), + KmsKeyId=Ref(_vault_kms_key), + Tags=self.get_tags( + service_override="Vault" + ) + [ec2.Tag('Name', 'VaultSecret-{}'.format(self.env))] + ) + + self.add_resource(_vault_secretsmanager_secret) + + # Add support for creating/updating secretsmanager entries + # You may need more permissions if use a customer-managed AWS KMS key to encrypt the secret. + # - kms:GenerateDataKey + # - kms:Decrypt + self.add_iam_policy(iam.Policy( + PolicyName='VaultSecretsManagerAccess', + PolicyDocument={ + 'Statement': [ + { + 'Effect': 'Allow', + 'Resource': Sub('arn:${{AWS::Partition}}:secretsmanager:${{AWS::Region}}:${{AWS::AccountId}}:secret:VaultSecret-{0}*'.format(self.env)), + 'Action': [ + 'secretsmanager:UpdateSecretVersionStage', + 'secretsmanager:UpdateSecret', + 'secretsmanager:PutSecretValue', + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + 'secretsmanager:TagResource' + ] + }, + { + 'Effect': 'Allow', + 'Resource': '*', + 'Action': [ + 'iam:GetRole' + ] + } + ] + } + )) + + _vault_client_role = iam.Role( + '{}ClientRole'.format(self.name), + AssumeRolePolicyDocument={ + 'Statement': [{ + 'Effect': 'Allow', + 'Principal': { + 'Service': ['ec2.amazonaws.com'] + }, + 'Action': ['sts:AssumeRole'] + }] + }, + Path='/', + Policies=[iam.Policy( + PolicyName='VaultClientPolicy', + PolicyDocument={ + 'Statement': [ + { + 'Effect': 'Allow', + 'Resource': '*', + 'Action': [ + 'ec2:DescribeInstances', + 'iam:GetInstanceProfile', + 'iam:GetUser', + 'iam:GetRole' + ] + } + ] + } + )] + ) + + self.add_resource(_vault_client_role) + + masters = [(index, ip) for index, ip in enumerate(config['masters'], 1)] + subnets = self.get_subnets('private') + for master in masters: + zone_index, master_ip = master + subnet = [s for s in subnets if netaddr.IPAddress(master_ip) in netaddr.IPNetwork(s['CidrBlock'])][0] + + _vault_eni = ec2.NetworkInterface( + 'VaultInstanceENI{}'.format(subnet['AvailabilityZone'][-1]), + Description='ENI for Vault ENV: {0} PrivateSubnet {1}'.format(self.env, subnet['SubnetId']), + GroupSet=self.security_groups, + PrivateIpAddress=master_ip, + SourceDestCheck=True, + SubnetId=subnet['SubnetId'], + Tags=self.get_tags(service_override="Vault", + role_override='Vault-{}'.format(subnet['AvailabilityZone'])) + ) + self.add_resource(_vault_eni) + + _user_data_template = self.get_cloudinit_template( + replacements=( + ('__PROMPT_COLOR__', self.prompt_color()), + ('__IVY_TAG__', constants.TAG), + ('__ENI_IP__', master_ip), + ('__SERVER_ID__', zone_index), + ('__CA_REGION__', _global_config.get('ca_region', self.region)), + ('__VAULT_SECRET__', 'VaultSecret-{}'.format(self.env)), + ('__HOSTS_ENTRIES__', '\n'.join( + ['{0} vault-master-{1}.node.{2}.{3} vault-master-{1}'. + format(ip, index, self.env, constants.TAG) for index, ip in masters] + )) + ) + ) + + _user_data = Sub( + _user_data_template + .replace('${', '${!') # Replace bash brackets with CFN escaped style + .replace('{#', '${'), # Replace rain-style CFN escapes with proper CFN brackets + { + 'CFN_ENI_ID': Ref(_vault_eni), + 'VAULT_CLIENT_ROLE_NAME': Ref(_vault_client_role), + 'VAULT_CLIENT_ROLE': GetAtt(_vault_client_role, 'Arn'), + } + ) + + _vault_launch_configuration = self.add_resource( + autoscaling.LaunchConfiguration( + 'VaultLaunchConfiguration{}'.format(subnet['AvailabilityZone'][-1]), + AssociatePublicIpAddress=False, + BlockDeviceMappings=get_block_device_mapping(self.parameters['InstanceType'].resource['Default']), + SecurityGroups=self.security_groups, + KeyName=Ref(self.keypair_name), + ImageId=Ref(self.ami), + InstanceType=Ref(self.instance_type), + InstanceMonitoring=False, + IamInstanceProfile=Ref(self.instance_profile), + UserData=Base64(_user_data) + ) + ) + self.add_resource( + autoscaling.AutoScalingGroup( + 'VaultASGroup{}'.format(subnet['AvailabilityZone'][-1]), + AvailabilityZones=[subnet['AvailabilityZone']], + HealthCheckType='EC2', + LaunchConfigurationName=Ref(_vault_launch_configuration), + MinSize=0, + MaxSize=1, + VPCZoneIdentifier=[subnet['SubnetId']], + Tags=self.get_autoscaling_tags( + service_override="Vault", + role_override='Vault-{}'.format(subnet['AvailabilityZone'])) + [ + autoscaling.Tag('Name', '{}Vault-{}'.format(self.env, subnet['AvailabilityZone']), + True) + ] + ) + )