Merge pull request cu-csc#4 from alal3177/staged-deployment

skeleton of the staged deployment

deployment/common.py

@@ -0,0 +1,85 @@
+"""
+Set of functions that are common among deployment classes.
+"""
+
+import os
+
+from lib import util
+
+def get_run_levels(dir_path):
+    """Return sorted list of directory content
+
+    Args:
+        dir_path ( string ) : directory path
+
+    Return:
+        list or bool
+    """
+    try:
+        contents = []
+        folder_contents = sorted(os.listdir(dir_path))
+        for item in folder_contents:
+            item_first_chr = item.split("-")[0]
+            try:
+                if os.path.isdir(os.path.join(dir_path,item)) and item_first_chr.isdigit():
+                    contents.append(item)
+            except:
+                continue
+
+        return contents
+    except OSError:
+        return False
+
+def get_executable_files(run_level_dir):
+    """get executable files from a directory
+
+    Given a directory, walk into it and return absolute path of files that are executable
+
+    Args:
+        run_level_dir ( string ) : directory path
+
+    Return :
+        scripts_list ( list ) : contains all executable files
+
+    """
+    scripts_list = []
+    for root, dirs, files in os.walk(run_level_dir):
+        for afile in files:
+            file_abs_path = os.path.join(root,afile)
+            if util.is_executable_file(file_abs_path):
+                scripts_list.append(os.path.join(root,afile))
+    return scripts_list
+
+
+def get_stages(mode, levels_dir, remote_dir=""):
+    """ Get the stages of execution in a dict format
+
+    Given a root directory of the stages, loop over those levels and extract all executable scripts
+    based on given mode, i.e : client or server.
+
+    Args:
+        mode (string) : client or server
+        levels_dir (string) : deployment stage root dir
+
+    return:
+        stages_dict (dict) : every key represent an execution level, the value of that key is list of all
+        executable scripts in that level.
+
+    """
+    stages_dict = {}
+    levels = get_run_levels(levels_dir)
+    if levels:
+        for level in levels:
+            abs_path = os.path.join(levels_dir, level)
+            if level.startswith("0-"):
+                tmp_exec_files = get_executable_files(abs_path)
+                if tmp_exec_files:
+                    stages_dict[level] = get_executable_files(abs_path)
+            else:
+                abs_path_w_mode = os.path.join(abs_path, mode)
+                stages_dict[level] = get_executable_files(abs_path_w_mode)
+
+    for key, value in stages_dict.iteritems():
+        stages_dict[key] = [ x.replace(levels_dir,remote_dir,1) for x in value ]
+
+    return stages_dict

deployment/engine.py

@@ -0,0 +1,9 @@
+"""
+implements the staged deployment engine which contains the logic and
+order of execution
+"""
+
+class StagedDeploymentEngine(object):
+
+    def __init__(self):
+        raise NotImplementedError

deployment/executor.py

@@ -0,0 +1,27 @@
+"""
+Module that handle the staged deployment execution
+"""
+
+from lib import util
+
+
+class Executor(object):
+
+    def __init__(self, hostname, private_key, staged_dict):
+        self.hostname = hostname
+        self.private_key = private_key
+        self.staged_dict = staged_dict
+
+    def execute_one_level(self, run_level):
+
+        result_dict = {}
+        cmds_in_run_level = self.staged_dict[run_level]
+
+        for command in cmds_in_run_level:
+            remote_command = util.RemoteCommand(self.hostname, self.private_key, command)
+            return_code = remote_command.execute()
+            result_dict[command] = (return_code, remote_command.stdout, remote_command.stderr)
+
+            if return_code != 0:
+                break
+        return result_dict

etc/global.conf

@@ -1,3 +1,6 @@
 [DEFAULT]
 key_name = automaton
-key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub
+key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub
+ssh_priv_key = /Users/ali/.ssh/ali_alzabarah_fg.priv
+git_repo_home = /home/staged-deployment-scripts
+git_repo_location = https://github.com/alal3177/staged-deployment-scripts.git

examples/engine.py

@@ -0,0 +1,50 @@
+"""
+This entire file is just an example to demonstrate functionality of staged deployment.
+We will implement the execution workflow here when we have more time.
+"""
+
+import shutil
+
+from lib import util
+from deployment import common
+from deployment.executor import Executor
+
+# config file path
+config_file = "../etc/global.conf"
+
+# clone the repo locally
+my_local_folder = util.clone_git_repo(util.read_config(config_file).get("DEFAULT","git_repo_location"))
+
+# we fill a dict with our stages
+stages = common.get_stages("client", my_local_folder, util.read_config(config_file).get("DEFAULT","git_repo_home"))
+
+
+# remove the directory since it is not needed anymore
+shutil.rmtree(my_local_folder,ignore_errors=True)
+
+# clone the repo to the vm
+remote_clone_result = util.RemoteCommand("vm-148-120.uc.futuregrid.org",\
+    util.read_config(config_file).get("DEFAULT", "ssh_priv_key"),
+    "git clone %s %s" % (util.read_config(config_file).get("DEFAULT","git_repo_location") ,
+                         util.read_config(config_file).get("DEFAULT","git_repo_home"))).execute()
+
+# initiate executor class with the stages
+exec_obj = Executor("vm-148-120.uc.futuregrid.org",
+    util.read_config(config_file).get("DEFAULT", "ssh_priv_key"),
+    stages)
+
+# loop over all available stages that has a script with execution bit set and execute it
+# if any of the commands at stage 0 failed for example, then we abort the execution and do not go
+# to next stage
+
+abort = False
+for each_stage in stages:
+    if not abort:
+        all_commands_result = exec_obj.execute_one_level(each_stage)
+        print "done with %s and all commands results are : %s" % (each_stage, str(all_commands_result))
+        for command_result in all_commands_result:
+            result, stdout, stderr =  all_commands_result[command_result]
+            if result != 0:
+                abort = True
+                break
+

lib/util.py

@@ -1,8 +1,12 @@
 import logging
 import subprocess
+import socket
+import tempfile
+import os
 
 from ConfigParser import SafeConfigParser
 from optparse import OptionParser
+from fabric import api as fabric_api
 
 
 LOG = logging.getLogger(__name__)
@@ -23,6 +27,66 @@ def execute(self):
         return process.returncode
 
 
+class RemoteCommand(object):
+    """Run a command in a remote machine.
+
+    Given a machine address, a none interactive command and ssh key, the function uses fabric
+    to execute the command in the remote machines.
+
+    Args:
+
+        hostname (string) : address of the machine
+
+        ssh_private_key (string) : absolute path to ssh private key
+
+        command ( string ) : command to execute
+
+
+    Return:
+
+        bool
+
+    """
+
+    def __init__(self, hostname, ssh_private_key, command):
+        self.stdout = None
+        self.stderr = None
+        self.command = command
+        self.hostname = hostname
+        self.ssh_private_key = ssh_private_key
+
+    def execute(self):
+        if os.path.isfile(self.ssh_private_key):
+            context = fabric_api.settings(fabric_api.hide('running', 'stdout', 'stderr', 'warnings'),
+                user="root",
+                key_filename=[].append(self.ssh_private_key),
+                disable_known_hosts=True,
+                linewise=True,
+                warn_only=True,
+                abort_on_prompts=True,
+                always_use_pty=True,
+                timeout=5)
+
+        else:
+            LOG.debug("Path to ssh private key is invalid")
+            return None
+
+        if context:
+            with context:
+                fabric_api.env.host_string = self.hostname
+                try:
+                    results = fabric_api.run(self.command)
+                    self.stdout = results.stdout
+                    self.stderr = results.stderr
+                    return results.return_code
+                except Exception as expt:
+                    LOG.debug("Exception in running remote command: %s" % str(expt))
+                    return None
+        else:
+            LOG.debug("issue initializing fabric context")
+            return None
+
+
 def read_config(file):
     config = SafeConfigParser()
     config.read(file)
@@ -50,3 +114,63 @@ def parse_options():
 
     (options, args) = parser.parse_args()
     return (options, args)
+
+def check_port_status(address, port=22, timeout=2):
+    """Check weather a remote port is accepting connection.
+
+    Given a port and an address, we establish a socket connection
+    to determine the port state
+
+    Args :
+        address (string): address of the machine, ip or hostname
+        port (int) : port number to connect to
+        timeout (int) : time to wait for a response
+
+    return :
+        bool
+            True: if port is accepting connection
+            False : if port is not accepting
+
+    """
+
+    default_timeout = socket.getdefaulttimeout()
+    socket.setdefaulttimeout(timeout)
+    remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        remote_socket.connect((address, port))
+    except Exception as inst:
+        LOG.debug("Exception in check_port_status : %s" % (str(inst)))
+        return False
+    finally:
+        remote_socket.close()
+        socket.setdefaulttimeout(default_timeout)
+    return True
+
+def clone_git_repo(repo_src):
+    """Clone a git repo
+
+    given a repo location, clone it locally and return the directory path
+
+    Args:
+        repo_src (string): git repo src location
+
+    Return:
+        repo_dest (string): directory that contains the cloned repo
+
+    """
+    repo_dest = tempfile.mkdtemp(dir="/tmp")
+    clone_cmd_obj = Command("git clone %s %s" % (repo_src, repo_dest))
+    if clone_cmd_obj.execute() == 0:
+        return repo_dest
+
+def is_executable_file(file_path):
+    """Check if a given file is executable
+
+    Args:
+        file_path (string) : file absolute path
+
+    Return:
+        bool
+
+    """
+    return os.path.isfile(file_path) and os.access(file_path, os.X_OK)

requirements.txt

@@ -0,0 +1 @@
+Fabric==1.4.3

tests/__init__.py

@@ -0,0 +1 @@
+

tests/deployment_tests.py

@@ -0,0 +1,59 @@
+"""
+Module that tests various deployment functionality
+To run me from command line:
+cd automaton/tests
+export PYTHONPATH=$PYTHONPATH:../
+python -m unittest -v deployment_tests
+unset PYTHONPATH
+
+I should have used nose but
+"""
+
+import unittest
+
+
+from lib import util
+from deployment import common
+
+class test_deployment_functions(unittest.TestCase):
+
+    def setUp(self):
+        self.testing_machine = "vm-148-120.uc.futuregrid.org"
+        self.bad_machine_name = "Idonotexistwallah.wrong"
+        self.key_filename = "/Users/ali/.ssh/ali_alzabarah_fg.priv"
+
+    def test_port_status_check(self):
+        # ssh port
+        self.assertFalse(util.check_port_status("google.com"))
+        # ssh port
+        self.assertTrue(util.check_port_status("research.cs.colorado.edu"))
+        # http port
+        self.assertTrue(util.check_port_status("google.com",80,2))
+        # wrong domain
+        self.assertFalse(util.check_port_status("Idonotexistwallah.wrong"))
+        # wrong ip
+        self.assertFalse(util.check_port_status("256.256.256.256"))
+
+
+    def test_run_remote_command(self):
+        result = util.RemoteCommand(self.testing_machine,
+            self.key_filename, "grep ewrqwerasdfqewr /etc/passwd").execute()
+        self.assertNotEqual(result,0)
+
+        result = util.RemoteCommand(self.testing_machine,
+            self.key_filename, "ls -al /etc/passwd").execute()
+        self.assertEqual(result,0)
+
+    def test_clone_git_repo(self):
+        self.assertIsNotNone(util.clone_git_repo("https://github.com/alal3177/automaton.git"))
+
+    def test_is_executable(self):
+        self.assertFalse(util.is_executable_file("wrong/path"))
+        self.assertTrue(util.is_executable_file("/bin/echo"))
+        self.assertFalse(util.is_executable_file("/tmp"))
+
+    def test_get_executable_files(self):
+        self.assertIsNotNone(common.get_executable_files("/bin"))
+
+if __name__ == '__main__':
+    unittest.main()