From b9cdcfb4b1e72b22d1bcab3fdabc2fe837aa1ca9 Mon Sep 17 00:00:00 2001 From: Ali Alzabarah Date: Wed, 7 Nov 2012 23:39:32 -0700 Subject: [PATCH] add skeleton of the staged deployment add the core functionality of the staged deployment add tests cases for the new functionality add demonstration of staged deployment add requirements.txt for pip # updates : move the engine to examples folder add not implemented exception for deployment/engine add debug to check_port_status convert run_remote_command from function to a class create new common module in deployment move common deployment functionality to deployment package documented how to run tests cases for non-PyCharm users :) changed the RemoteCommand to return return_code or None --- deployment/__init__.py | 0 deployment/common.py | 85 ++++++++++++++++++++++++++ deployment/engine.py | 9 +++ deployment/executor.py | 27 +++++++++ etc/global.conf | 5 +- examples/__init__.py | 0 examples/engine.py | 50 +++++++++++++++ lib/util.py | 124 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/__init__.py | 1 + tests/deployment_tests.py | 59 ++++++++++++++++++ 11 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 deployment/__init__.py create mode 100644 deployment/common.py create mode 100644 deployment/engine.py create mode 100644 deployment/executor.py create mode 100644 examples/__init__.py create mode 100644 examples/engine.py create mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/deployment_tests.py diff --git a/deployment/__init__.py b/deployment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deployment/common.py b/deployment/common.py new file mode 100644 index 0000000..978f6eb --- /dev/null +++ b/deployment/common.py @@ -0,0 +1,85 @@ +""" +Set of functions that are common among deployment classes. +""" + +import os + +from lib import util + +def get_run_levels(dir_path): + """Return sorted list of directory content + + Args: + dir_path ( string ) : directory path + + Return: + list or bool + """ + try: + contents = [] + folder_contents = sorted(os.listdir(dir_path)) + for item in folder_contents: + item_first_chr = item.split("-")[0] + try: + if os.path.isdir(os.path.join(dir_path,item)) and item_first_chr.isdigit(): + contents.append(item) + except: + continue + + return contents + except OSError: + return False + +def get_executable_files(run_level_dir): + """get executable files from a directory + + Given a directory, walk into it and return absolute path of files that are executable + + Args: + run_level_dir ( string ) : directory path + + Return : + scripts_list ( list ) : contains all executable files + + """ + scripts_list = [] + for root, dirs, files in os.walk(run_level_dir): + for afile in files: + file_abs_path = os.path.join(root,afile) + if util.is_executable_file(file_abs_path): + scripts_list.append(os.path.join(root,afile)) + return scripts_list + + +def get_stages(mode, levels_dir, remote_dir=""): + """ Get the stages of execution in a dict format + + Given a root directory of the stages, loop over those levels and extract all executable scripts + based on given mode, i.e : client or server. + + Args: + mode (string) : client or server + levels_dir (string) : deployment stage root dir + + return: + stages_dict (dict) : every key represent an execution level, the value of that key is list of all + executable scripts in that level. + + """ + stages_dict = {} + levels = get_run_levels(levels_dir) + if levels: + for level in levels: + abs_path = os.path.join(levels_dir, level) + if level.startswith("0-"): + tmp_exec_files = get_executable_files(abs_path) + if tmp_exec_files: + stages_dict[level] = get_executable_files(abs_path) + else: + abs_path_w_mode = os.path.join(abs_path, mode) + stages_dict[level] = get_executable_files(abs_path_w_mode) + + for key, value in stages_dict.iteritems(): + stages_dict[key] = [ x.replace(levels_dir,remote_dir,1) for x in value ] + + return stages_dict \ No newline at end of file diff --git a/deployment/engine.py b/deployment/engine.py new file mode 100644 index 0000000..e382287 --- /dev/null +++ b/deployment/engine.py @@ -0,0 +1,9 @@ +""" +implements the staged deployment engine which contains the logic and +order of execution +""" + +class StagedDeploymentEngine(object): + + def __init__(self): + raise NotImplementedError diff --git a/deployment/executor.py b/deployment/executor.py new file mode 100644 index 0000000..691f781 --- /dev/null +++ b/deployment/executor.py @@ -0,0 +1,27 @@ +""" +Module that handle the staged deployment execution +""" + +from lib import util + + +class Executor(object): + + def __init__(self, hostname, private_key, staged_dict): + self.hostname = hostname + self.private_key = private_key + self.staged_dict = staged_dict + + def execute_one_level(self, run_level): + + result_dict = {} + cmds_in_run_level = self.staged_dict[run_level] + + for command in cmds_in_run_level: + remote_command = util.RemoteCommand(self.hostname, self.private_key, command) + return_code = remote_command.execute() + result_dict[command] = (return_code, remote_command.stdout, remote_command.stderr) + + if return_code != 0: + break + return result_dict diff --git a/etc/global.conf b/etc/global.conf index ee7dc1f..85c0f0c 100644 --- a/etc/global.conf +++ b/etc/global.conf @@ -1,3 +1,6 @@ [DEFAULT] key_name = automaton -key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub \ No newline at end of file +key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub +ssh_priv_key = /Users/ali/.ssh/ali_alzabarah_fg.priv +git_repo_home = /home/staged-deployment-scripts +git_repo_location = https://github.com/alal3177/staged-deployment-scripts.git \ No newline at end of file diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/engine.py b/examples/engine.py new file mode 100644 index 0000000..c966cf9 --- /dev/null +++ b/examples/engine.py @@ -0,0 +1,50 @@ +""" +This entire file is just an example to demonstrate functionality of staged deployment. +We will implement the execution workflow here when we have more time. +""" + +import shutil + +from lib import util +from deployment import common +from deployment.executor import Executor + +# config file path +config_file = "../etc/global.conf" + +# clone the repo locally +my_local_folder = util.clone_git_repo(util.read_config(config_file).get("DEFAULT","git_repo_location")) + +# we fill a dict with our stages +stages = common.get_stages("client", my_local_folder, util.read_config(config_file).get("DEFAULT","git_repo_home")) + + +# remove the directory since it is not needed anymore +shutil.rmtree(my_local_folder,ignore_errors=True) + +# clone the repo to the vm +remote_clone_result = util.RemoteCommand("vm-148-120.uc.futuregrid.org",\ + util.read_config(config_file).get("DEFAULT", "ssh_priv_key"), + "git clone %s %s" % (util.read_config(config_file).get("DEFAULT","git_repo_location") , + util.read_config(config_file).get("DEFAULT","git_repo_home"))).execute() + +# initiate executor class with the stages +exec_obj = Executor("vm-148-120.uc.futuregrid.org", + util.read_config(config_file).get("DEFAULT", "ssh_priv_key"), + stages) + +# loop over all available stages that has a script with execution bit set and execute it +# if any of the commands at stage 0 failed for example, then we abort the execution and do not go +# to next stage + +abort = False +for each_stage in stages: + if not abort: + all_commands_result = exec_obj.execute_one_level(each_stage) + print "done with %s and all commands results are : %s" % (each_stage, str(all_commands_result)) + for command_result in all_commands_result: + result, stdout, stderr = all_commands_result[command_result] + if result != 0: + abort = True + break + diff --git a/lib/util.py b/lib/util.py index 2382637..76f957e 100644 --- a/lib/util.py +++ b/lib/util.py @@ -1,8 +1,12 @@ import logging import subprocess +import socket +import tempfile +import os from ConfigParser import SafeConfigParser from optparse import OptionParser +from fabric import api as fabric_api LOG = logging.getLogger(__name__) @@ -23,6 +27,66 @@ def execute(self): return process.returncode +class RemoteCommand(object): + """Run a command in a remote machine. + + Given a machine address, a none interactive command and ssh key, the function uses fabric + to execute the command in the remote machines. + + Args: + + hostname (string) : address of the machine + + ssh_private_key (string) : absolute path to ssh private key + + command ( string ) : command to execute + + + Return: + + bool + + """ + + def __init__(self, hostname, ssh_private_key, command): + self.stdout = None + self.stderr = None + self.command = command + self.hostname = hostname + self.ssh_private_key = ssh_private_key + + def execute(self): + if os.path.isfile(self.ssh_private_key): + context = fabric_api.settings(fabric_api.hide('running', 'stdout', 'stderr', 'warnings'), + user="root", + key_filename=[].append(self.ssh_private_key), + disable_known_hosts=True, + linewise=True, + warn_only=True, + abort_on_prompts=True, + always_use_pty=True, + timeout=5) + + else: + LOG.debug("Path to ssh private key is invalid") + return None + + if context: + with context: + fabric_api.env.host_string = self.hostname + try: + results = fabric_api.run(self.command) + self.stdout = results.stdout + self.stderr = results.stderr + return results.return_code + except Exception as expt: + LOG.debug("Exception in running remote command: %s" % str(expt)) + return None + else: + LOG.debug("issue initializing fabric context") + return None + + def read_config(file): config = SafeConfigParser() config.read(file) @@ -50,3 +114,63 @@ def parse_options(): (options, args) = parser.parse_args() return (options, args) + +def check_port_status(address, port=22, timeout=2): + """Check weather a remote port is accepting connection. + + Given a port and an address, we establish a socket connection + to determine the port state + + Args : + address (string): address of the machine, ip or hostname + port (int) : port number to connect to + timeout (int) : time to wait for a response + + return : + bool + True: if port is accepting connection + False : if port is not accepting + + """ + + default_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(timeout) + remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + remote_socket.connect((address, port)) + except Exception as inst: + LOG.debug("Exception in check_port_status : %s" % (str(inst))) + return False + finally: + remote_socket.close() + socket.setdefaulttimeout(default_timeout) + return True + +def clone_git_repo(repo_src): + """Clone a git repo + + given a repo location, clone it locally and return the directory path + + Args: + repo_src (string): git repo src location + + Return: + repo_dest (string): directory that contains the cloned repo + + """ + repo_dest = tempfile.mkdtemp(dir="/tmp") + clone_cmd_obj = Command("git clone %s %s" % (repo_src, repo_dest)) + if clone_cmd_obj.execute() == 0: + return repo_dest + +def is_executable_file(file_path): + """Check if a given file is executable + + Args: + file_path (string) : file absolute path + + Return: + bool + + """ + return os.path.isfile(file_path) and os.access(file_path, os.X_OK) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5ca8cf1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Fabric==1.4.3 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/deployment_tests.py b/tests/deployment_tests.py new file mode 100644 index 0000000..02c67ff --- /dev/null +++ b/tests/deployment_tests.py @@ -0,0 +1,59 @@ +""" +Module that tests various deployment functionality +To run me from command line: +cd automaton/tests +export PYTHONPATH=$PYTHONPATH:../ +python -m unittest -v deployment_tests +unset PYTHONPATH + +I should have used nose but +""" + +import unittest + + +from lib import util +from deployment import common + +class test_deployment_functions(unittest.TestCase): + + def setUp(self): + self.testing_machine = "vm-148-120.uc.futuregrid.org" + self.bad_machine_name = "Idonotexistwallah.wrong" + self.key_filename = "/Users/ali/.ssh/ali_alzabarah_fg.priv" + + def test_port_status_check(self): + # ssh port + self.assertFalse(util.check_port_status("google.com")) + # ssh port + self.assertTrue(util.check_port_status("research.cs.colorado.edu")) + # http port + self.assertTrue(util.check_port_status("google.com",80,2)) + # wrong domain + self.assertFalse(util.check_port_status("Idonotexistwallah.wrong")) + # wrong ip + self.assertFalse(util.check_port_status("256.256.256.256")) + + + def test_run_remote_command(self): + result = util.RemoteCommand(self.testing_machine, + self.key_filename, "grep ewrqwerasdfqewr /etc/passwd").execute() + self.assertNotEqual(result,0) + + result = util.RemoteCommand(self.testing_machine, + self.key_filename, "ls -al /etc/passwd").execute() + self.assertEqual(result,0) + + def test_clone_git_repo(self): + self.assertIsNotNone(util.clone_git_repo("https://github.com/alal3177/automaton.git")) + + def test_is_executable(self): + self.assertFalse(util.is_executable_file("wrong/path")) + self.assertTrue(util.is_executable_file("/bin/echo")) + self.assertFalse(util.is_executable_file("/tmp")) + + def test_get_executable_files(self): + self.assertIsNotNone(common.get_executable_files("/bin")) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file