user_data_vm

From nobody Wed Nov 18 09:59:29 2015
Comments:
 #
 # Generic DIRAC Cloud Init user_data template file for use with VMs, and 
 # containing the following ##user_data___## substitutions:
 #
 # user_data_file_hostkey
 # user_data_file_hostcert
 # user_data_option_cvmfs_proxy
 # user_data_space
 #
 # Each substitution pattern may occur more than once in this template. If you
 # are reading a processed file, then these substitutions will already have 
 # been made below.
 #
 # This file should normally be processed by Vac (version 0.20.0 onwards) or
 # Vcycle (0.3.0 onwards) internally. 
 #
 # Andrew.McNab@cern.ch January 2016
 #
Content-Type: multipart/mixed; boundary="===============3141592653589793238=="
MIME-Version: 1.0

--===============3141592653589793238==
MIME-Version: 1.0
Content-Type: text/cloud-config; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cloud-config-file"

# cloud-config

cvmfs:
    local:
        CVMFS_REPOSITORIES: grid
        CVMFS_HTTP_PROXY: ##user_data_option_cvmfs_proxy##

--===============3141592653589793238==
MIME-Version: 1.0
Content-Type: text/ucernvm; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="ucernvm-file"

[ucernvm-begin]
resize_rootfs=off
cvmfs_http_proxy='##user_data_option_cvmfs_proxy##'
[ucernvm-end]

--===============3141592653589793238==
MIME-Version: 1.0
Content-Type: text/x-shellscript; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="user_data_script"

#!/bin/sh

mkdir -p /var/spool/joboutputs
(
# Set the hostname if available; display otherwise
hostname ##user_data_machine_hostname##
date --utc +"%Y-%m-%d %H:%M:%S %Z user_data_script Start user_data on `hostname`"

echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6
date --utc +"%Y-%m-%d %H:%M:%S %Z Disable IPv6"

# Record MJFJO if substituted here by VM lifecycle manager
export MACHINEFEATURES='##user_data_machinefeatures_url##'
export JOBFEATURES='##user_data_jobfeatures_url##'
export JOBOUTPUTS='##user_data_joboutputs_url##'

# Save whatever we use by other scripts
/bin/echo -e "export MACHINEFEATURES=$MACHINEFEATURES\nexport JOBFEATURES=$JOBFEATURES\nexport JOBOUTPUTS=$JOBOUTPUTS" > /etc/profile.d/mjf.sh
/bin/echo -e "setenv MACHINEFEATURES $MACHINEFEATURES\nsetenv JOBFEATURES $JOBFEATURES\nsetenv JOBOUTPUTS $JOBOUTPUTS" > /etc/profile.d/mjf.csh

export VO='##user_data_option_vo##'
if [ "$VO" == "" ] ; then
  VO=lhcb
fi

export VM_UUID='##user_data_uuid##'
if [ "$VM_UUID" = "" -a "$JOBFEATURES" != "" ] ; then
  export VM_UUID=`python -c "import urllib ; print urllib.urlopen('$JOBFEATURES/job_id').read().strip()"`
fi

if [ "$VM_UUID" = "" ] ; then
  # If still unset then just use date and hostname from the VM lifecycle manager
  export VM_UUID=`date +'%s.##user_data_machine_hostname##'`
fi

export JOB_ID="##user_data_space##:$VM_UUID:##user_data_machinetype##"
export PILOT_UUID="vm://##user_data_space##/$JOB_ID"

# Create a shutdown_message if ACPI shutdown signal received
/bin/echo -e 'echo "100 VM received ACPI shutdown signal from hypervisor" > /var/spool/joboutputs/shutdown_message\n/sbin/shutdown -h now' >/etc/acpi/actions/power.sh
chmod +x /etc/acpi/actions/power.sh

# Disable TCP offloading etc - done by default
if [ "##user_data_option_network_offload##" != "on" ] ; then
  ethtool -K eth0 tso off gso off gro off
fi

# We never let VMs send emails (likely to be annoying errors from root)
/sbin/iptables -A OUTPUT -p tcp --dport 25 -j DROP

# Once we have finished with the metadata, stop any user process reading it later
/sbin/iptables -A OUTPUT -d 169.254.169.254 -p tcp --dport 80 -j DROP 

machine_syslog=`python -c "import urllib ; print urllib.urlopen('$MACHINEFEATURES/syslog').read().strip()"`
if [ "$machine_syslog" ] ; then
 echo "*.* @$machinesyslog" > /etc/rsyslog.d/vm.conf
 /sbin/service rsyslog restart
fi

# Get the big 40GB+ logical partition as /scratch
mkdir -p /scratch
if [ -b /dev/vdb1 -a -b /dev/vdb2 ] ; then
  # Openstack at CERN with cvm* flavor? 
  # vda1 is boot image, vdb1 is root partition, vdb2 is unformatted
  mkfs -q -t ext4 /dev/vdb2
  mount /dev/vdb2 /scratch 
elif [ -b /dev/sda1 -a -b /dev/sda2 -a -b /dev/sda3 ] ; then
  # GCE: sda1 is boot image, sda2 is root partition, sda3 is unformatted
  mkfs -q -t ext4 /dev/sda3
  mount /dev/sda3 /scratch
elif [ -b /dev/vdb1 ] ; then
  # Openstack at CERN with hep* flavor?
  # vda1 is boot image, vdb1 is root partition, and no vdb2
  # Since boot image is small, can use rest of vda for /scratch
  echo -e 'n\np\n2\n\n\nw\n'| fdisk /dev/vda
  mkfs -q -t ext4 /dev/vda2
  mount /dev/vda2 /scratch 
elif [ -b /dev/vdb ] ; then
  # Efficient virtio device
  mkfs -q -t ext4 /dev/vdb
  mount /dev/vdb /scratch
elif [ -b /dev/vda1 -a -b /dev/vda2 ] ; then
  # We just have a big vda with unused space in vda2
  mkfs -q -t ext4 /dev/vda2
  mount /dev/vda2 /scratch
elif [ -b /dev/sdb ] ; then
  # Virtual SCSI
  mkfs -q -t ext4 /dev/sdb
  mount /dev/sdb /scratch
elif [ -b /dev/hdb ] ; then
  # Virtual IDE
  mkfs -q -t ext4 /dev/hdb
  mount /dev/hdb /scratch
elif [ -b /dev/xvdb ] ; then
  # Xen virtual disk device
  mkfs -q -t ext4 /dev/xvdb
  mount /dev/xvdb /scratch
else
  date --utc +'%Y-%m-%d %H:%M:%S %Z user_data_script Missing vdb/hdb/sdb/xvdb block device for /scratch'
  echo "500 Missing vdb/hdb/sdb block device for /scratch" > /var/spool/joboutputs/shutdown_message
  /sbin/shutdown -h now
  sleep 1234567890
fi  

if [ -b /dev/vda ] ; then
  # We rely on the hypervisor's disk I/O scheduling
  echo 'noop' > /sys/block/vda/queue/scheduler
  echo 'noop' > /sys/block/vdb/queue/scheduler
fi

# anyone can create directories there
chmod ugo+rwxt /scratch

# Scratch tmp for TMPDIR
mkdir -p /scratch/tmp
chmod ugo+rwxt /scratch/tmp

mkdir -p /scratch/plt/etc/grid-security
export X509_USER_PROXY=/scratch/plt/etc/grid-security/x509proxy.pem

if [ ! -z "##user_data_option_x509_proxy##" ] ; then
  # Simple if we are given an X.509 Proxy
  cat <<X5_EOF > $X509_USER_PROXY
##user_data_option_x509_proxy##
X5_EOF

  cp $X509_USER_PROXY /scratch/plt/etc/grid-security/hostkey.pem
  cp $X509_USER_PROXY /scratch/plt/etc/grid-security/hostcert.pem

elif [ ! -z "##user_data_file_hostkey##" -a ! -z "##user_data_file_hostcert##" ] ; then
  # Given full host cert/key pair
  # Old versions of Vac/Vcycle call this user_data_file_ rather than user_data_option_

  cat <<X5_EOF > /scratch/plt/etc/grid-security/hostkey.pem
##user_data_file_hostkey##
X5_EOF

  cat <<X5_EOF > /scratch/plt/etc/grid-security/hostcert.pem
##user_data_file_hostcert##
X5_EOF
q
  cat /scratch/plt/etc/grid-security/hostkey.pem /scratch/plt/etc/grid-security/hostcert.pem > $X509_USER_PROXY
else
  date --utc +"%Y-%m-%d %H:%M:%S %Z Neither user_data_option_x509_proxy or user_data_option_hostkey/_hostcert defined!"
fi

cp $X509_USER_PROXY /root/x509proxy.pem # Save a copy which will stay owned by root, for root to use
chmod 0400 /scratch/plt/etc/grid-security/*.pem /root/x509proxy.pem

# Get CA certs from cvmfs
rm -Rf /etc/grid-security
ln -sf /cvmfs/grid.cern.ch/etc/grid-security /etc/grid-security
export X509_CERT_DIR=/etc/grid-security/certificates

# These will be picked up by login etc shells
cat <<EOF > /etc/profile.d/grid-paths.sh
export X509_CERT_DIR=/etc/grid-security/certificates
export X509_VOMS_DIR=/etc/grid-security/vomsdir
EOF

# make a first heartbeat
echo 0.0 0.0 0.0 0.0 0.0 > /var/spool/joboutputs/heartbeat
/usr/bin/curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file /var/spool/joboutputs/heartbeat "$JOBOUTPUTS/heartbeat"
/usr/bin/curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file /var/spool/joboutputs/heartbeat "$JOBOUTPUTS/vm-heartbeat"

# put heartbeat on MJF server every 5 minutes with a random offset
echo -e "*/5 * * * * root sleep `expr $RANDOM / 110` ; echo \`cut -f1-3 -d' ' /proc/loadavg\` \`cat /proc/uptime\` >/var/spool/joboutputs/heartbeat ; /usr/bin/curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file /var/spool/joboutputs/heartbeat $JOBOUTPUTS/vm-heartbeat ; /usr/bin/curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file /var/spool/joboutputs/heartbeat $JOBOUTPUTS/heartbeat >/var/log/heartbeat.log 2>&1" >/etc/cron.d/heartbeat

# We swap on the logical partition (cannot on CernVM 3 aufs filesystem)
# Since ext4 we can use fallocate:
fallocate -l 4g /scratch/swapfile
chmod 0600 /scratch/swapfile
mkswap /scratch/swapfile 
swapon /scratch/swapfile

# Swap as little as possible
sysctl vm.swappiness=1
       
# Don't want to be doing this at 4 or 5am every day!
rm -f /etc/cron.daily/mlocate.cron

# Log proxies used for cvmfs
attr -g proxy /mnt/.ro
for i in /cvmfs/*
do
  attr -g proxy $i
done

# Avoid age-old sudo problem
echo 'Defaults !requiretty' >>/etc/sudoers
echo 'Defaults visiblepw'   >>/etc/sudoers

# The pilot user account plt
/usr/sbin/useradd -b /scratch plt

chown plt.plt /var/spool/joboutputs
chmod 0755 /var/spool/joboutputs

# Add plt0102 etc accounts for the payloads that plt can sudo to

# At most one jobagent per logical processor
processors=`grep '^processor[[:space:]]' /proc/cpuinfo | wc --lines`
for ((m=0; m < processors; m++))
do
  # Up to 100 successive payloads per jobagent
  for ((n=0; n < 100; n++))
  do
    payloaduser=`printf 'plt%02dp%02d' $m $n`
    payloaduserid=`printf '1%02d%02d' $m $n`

    # Payload user home directory and dot files
    mkdir /scratch/$payloaduser
    cp -f /etc/skel/.*shrc /scratch/$payloaduser
    cp -f /etc/skel/.bash* /scratch/$payloaduser

    # Add to /etc/passwd and /etc/group
    echo "$payloaduser:x:$payloaduserid:$payloaduserid::/scratch/$payloaduser:/bin/bash" >>/etc/passwd
    echo "$payloaduser:x:$payloaduserid:plt" >>/etc/group

    # Add the plt group as a secondary group
    if [ "$payloaduser" = "plt00p00" ] ; then
      sed -i "s/^plt:.*/&$payloaduser/" /etc/group
    else
      sed -i "s/^plt:.*/&,$payloaduser/" /etc/group
    fi

    # Ownership and permissions of payload home directory
    chown -R $payloaduser.$payloaduser /scratch/$payloaduser
    chmod 0775 /scratch/$payloaduser

    # plt user can sudo to any payload user
    echo "Defaults>$payloaduser !requiretty"           >>/etc/sudoers
    echo "Defaults>$payloaduser visiblepw"             >>/etc/sudoers
    echo "Defaults>$payloaduser !env_reset"            >>/etc/sudoers
    echo "plt ALL = ($payloaduser) NOPASSWD: ALL"      >>/etc/sudoers
  done
done

cd /scratch/plt
# Fetch the DIRAC pilot scripts
if [ '##user_data_option_dirac_pilot_url##' != '' ] ; then
  wget --no-directories --recursive --no-parent --execute robots=off --reject 'index.html*' --ca-directory=$X509_CERT_DIR '##user_data_option_dirac_pilot_url##'
elif [ '##user_data_url##' != '' ] ; then
  # Remove user_data file name back to final slash
  user_data_dir=`echo '##user_data_url##' | sed 's:[^/]*$::'`
  wget --no-directories --recursive --no-parent --execute robots=off --reject 'index.html*' --ca-directory=$X509_CERT_DIR "$user_data_dir"
else
  wget --no-directories --recursive --no-parent --execute robots=off --reject 'index.html*' --ca-directory=$X509_CERT_DIR https://lhcb-portal-dirac.cern.ch/pilot/
fi    

# So payload accounts can create directories here, but not interfere
chown -R plt.plt /scratch/plt
chmod 1775 /scratch/plt

SUBMIT_POOL_OPTS="-o '/LocalSite/SubmitPool=Test'"

# Source any include scripts we fetched
for i in include_vm_*.sh
do
 if [ -r "$i" ] ; then
   . ./$i
 fi
done

if [ '##user_data_option_dirac_queue##' != '' ] ; then
  QUEUE='##user_data_option_dirac_queue##'
else
  QUEUE=default
fi

# Now run the pilot script
/usr/bin/sudo -i -n -u plt \
 X509_USER_PROXY=$X509_USER_PROXY \
 MACHINEFEATURES="$MACHINEFEATURES" JOBFEATURES="$JOBFEATURES" \
 JOB_ID="$JOB_ID" PILOT_UUID="$PILOT_UUID" \
 python /scratch/plt/dirac-pilot.py \
 --debug \
 $SUBMIT_POOL_OPTS \
 --Name '##user_data_space##' \
 --Queue $QUEUE \
 --MaxCycles 1 \
 --CEType Sudo \
 --cert --certLocation /scratch/plt/etc/grid-security \
 ##user_data_option_dirac_opts## \
 >/var/spool/joboutputs/dirac-pilot.log 2>&1

# Save JobAgent and System logs
cp -f /scratch/plt/bashrc /scratch/plt/jobagent.*.log /scratch/plt/shutdown_message* /var/log/boot.log /var/log/dmesg /var/log/secure /var/log/messages* /etc/cvmfs/default.* \
  /var/spool/joboutputs/

(
  cd /var/spool/joboutputs
  for i in *
  do
   if [ -f $i ] ; then 
    curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file "$i" \
     "$JOBOUTPUTS/"

    if [ "$DEPO_BASE_URL" != "" ] ; then
      # This will be replaced by extended pilot logging??
      curl --capath $X509_CERT_DIR --cert /root/x509proxy.pem --cacert /root/x509proxy.pem --location --upload-file "$i" \
        "$DEPO_BASE_URL/##user_data_space##/##user_data_machinetype##/##user_data_machine_hostname##/##user_data_uuid##/"
    fi
   fi
  done
)

# Try conventional shutdown
date --utc +'%Y-%m-%d %H:%M:%S %Z user_data_script Run /sbin/shutdown -h now'
/sbin/shutdown -h now
sleep 60

# If that fails, do an instant reboot
date --utc +'%Y-%m-%d %H:%M:%S %Z user_data_script Run echo o > /proc/sysrq-trigger'
echo o > /proc/sysrq-trigger

) >/var/spool/joboutputs/user_data_script.log 2>&1 &
--===============3141592653589793238==--