From 75dde836a5b030c7305f8f6d32443f5f4382323a Mon Sep 17 00:00:00 2001 From: Dmitry Duplyakin Date: Mon, 5 Nov 2012 16:37:07 -0700 Subject: [PATCH] Added config files: global.conf, benchmarking.conf, clouds.conf Added lib/config.py with classes that retrieve and store options from the config files Implemented classes Cloud, Clouds, Cluster, Clusters Update #1: changed Log.info to Log.debug Update #2: removed etc/automaton.conf Update #3: added a check before registering a key Update #4: replaced string concatenation in %s Update #5: added file default locations in the help for command line arguments Update #6: replaces old function names with log_info and get_fqdns --- .gitignore | 3 ++ automaton.py | 24 +++++++--- etc/automaton.conf | 4 -- etc/benchmarking.conf | 8 ++++ etc/clouds.conf | 21 +++++++++ etc/global.conf | 3 ++ lib/config.py | 49 ++++++++++++++++++-- lib/util.py | 21 ++++++--- resources/__init__.py | 0 resources/cloud/clouds.py | 82 +++++++++++++++++++++++++++++++-- resources/cluster/clusters.py | 85 +++++++++++++++++++++++++++++++++-- 11 files changed, 274 insertions(+), 26 deletions(-) delete mode 100644 etc/automaton.conf create mode 100644 etc/benchmarking.conf create mode 100644 etc/clouds.conf create mode 100644 etc/global.conf create mode 100644 resources/__init__.py diff --git a/.gitignore b/.gitignore index f24cd99..50ff77d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ pip-log.txt #Mr Developer .mr.developer.cfg + +#PyCharm +.idea \ No newline at end of file diff --git a/automaton.py b/automaton.py index c89eca5..023b7a9 100755 --- a/automaton.py +++ b/automaton.py @@ -2,26 +2,36 @@ import logging import signal +from threading import Thread from lib.logger import configure_logging from lib.util import parse_options from lib.util import read_config -from threading import Thread - +from lib.config import Config +from resources.cloud.clouds import Cloud, Clouds +from resources.cluster.clusters import Clusters SIGEXIT = False LOG = logging.getLogger(__name__) class Automaton(Thread): - def __init__(self, config): + def __init__(self, config, clusters): Thread.__init__(self) self.config = config + self.clusters = clusters def run(self): LOG.info("Starting Automaton") #TODO(pdmars): do something + # Code below demonstrates functionality of Cluster class: + # for cluster in self.clusters.list: + # cluster.connect() + # cluster.launch() + # cluster.log_info() + # fqdns = cluster.get_fqdns() + # cluster.terminate_all() def clean_exit(signum, frame): global SIGEXIT @@ -29,13 +39,15 @@ def clean_exit(signum, frame): LOG.critical("Exit signal received. Exiting at the next sane time. " "Please stand by.") - def main(): (options, args) = parse_options() configure_logging(options.debug) - config = read_config(options.config_file) + + config = Config(options) + clusters = Clusters(config) + signal.signal(signal.SIGINT, clean_exit) - automaton = Automaton(config) + automaton = Automaton(config, clusters) automaton.start() # wake every seconed to make sure signals are handled by the main thread # need this due to a quirk in the way Python threading handles signals diff --git a/etc/automaton.conf b/etc/automaton.conf deleted file mode 100644 index 889a9b9..0000000 --- a/etc/automaton.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Want to be able to specify: -# - applications to run and their location -# - output directory -# - resources to benchmark (clouds or clusters) \ No newline at end of file diff --git a/etc/benchmarking.conf b/etc/benchmarking.conf new file mode 100644 index 0000000..54e51e0 --- /dev/null +++ b/etc/benchmarking.conf @@ -0,0 +1,8 @@ +[Benchmark-01] +sierra = 0 +hotel = 1 + +[Benchmark-02] + +[Benchmark-03] + diff --git a/etc/clouds.conf b/etc/clouds.conf new file mode 100644 index 0000000..9355190 --- /dev/null +++ b/etc/clouds.conf @@ -0,0 +1,21 @@ +[hotel] +cloud_uri = svc.uc.futuregrid.org +cloud_port = 8444 +image_id = debian-6.0.5.gz +cloud_type = nimbus +availability_zone = us-east-1 +instance_type = m1.paul +instance_cores = 1 +access_id = $NIMBUS_IAAS_ACCESS_KEY +secret_key = $NIMBUS_IAAS_SECRET_KEY + +[sierra] +cloud_uri = s83r.idp.sdsc.futuregrid.org +cloud_port = 8444 +image_id = debian-lenny.gz +cloud_type = nimbus +availability_zone = us-east-1 +instance_type = m1.paul +instance_cores = 1 +access_id = $NIMBUS_IAAS_ACCESS_KEY +secret_key = $NIMBUS_IAAS_SECRET_KEY \ No newline at end of file diff --git a/etc/global.conf b/etc/global.conf new file mode 100644 index 0000000..ee7dc1f --- /dev/null +++ b/etc/global.conf @@ -0,0 +1,3 @@ +[DEFAULT] +key_name = automaton +key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub \ No newline at end of file diff --git a/lib/config.py b/lib/config.py index 1e9040e..37650fd 100644 --- a/lib/config.py +++ b/lib/config.py @@ -1,3 +1,46 @@ -class AutomatonConfig(object): - def __init__(self, name, config): - pass +from lib.util import read_config + +class GlobalConfig(object): + """ GlobalConfig class retrieves information from the file that specifies global parameters """ + + def __init__(self, config): + self.config = config + default_dict = self.config.defaults() + self.key_name = default_dict['key_name'] + self.key_path = default_dict['key_path'] + +class CloudsConfig(object): + """ CloudsConfig class retrieves information from the file that specifies global parameters """ + + def __init__(self, config): + self.config = config + self.list = config.sections() + +class Benchmark(object): + """ Benchmark class retrieves information from one of the section of the benchmarking file """ + + def __init__(self, benchmark_name, config): + self.config = config + self.name = benchmark_name + dict = self.config.items(self.name) + self.dict = {} + # Form a dictionary out of items in the specified section + for pair in dict: + self.dict[pair[0]] = pair[1] + +class BenchmarkingConfig(object): + """ BenchmarkingConfig class retrieves benchmarking information and populates benchmark list """ + + def __init__(self, config): + self.config = config + self.list = list() + for sec in self.config.sections(): + self.list.append(Benchmark(sec, self.config)) + +class Config(object): + """ Config class retrieves all configuration information """ + + def __init__(self, options): + self.globals = GlobalConfig(read_config(options.global_file)) + self.clouds = CloudsConfig(read_config(options.clouds_file)) + self.benchmarking = BenchmarkingConfig(read_config(options.benchmarking_file)) \ No newline at end of file diff --git a/lib/util.py b/lib/util.py index e4d8675..2382637 100644 --- a/lib/util.py +++ b/lib/util.py @@ -23,19 +23,30 @@ def execute(self): return process.returncode -def read_config(config_file): +def read_config(file): config = SafeConfigParser() - config.read(config_file) + config.read(file) return config def parse_options(): parser = OptionParser() - parser.add_option("-c", "--config_file", action="store", - dest="config_file", help="Location of the config file.") + parser.add_option("-d", "--debug", action="store_true", dest="debug", help="Enable debugging log level.") - parser.set_defaults(config_file="etc/automaton.conf") parser.set_defaults(debug=False) + + parser.add_option("-g", "--global_file", action="store", dest="global_file", + help="Location of the file with global parameters (default: etc/global.conf).") + parser.set_defaults(global_file="etc/global.conf") + + parser.add_option("-c", "--clouds_file", action="store", dest="clouds_file", + help="Location of the file with cloud parameters (default: etc/clouds.conf).") + parser.set_defaults(clouds_file="etc/clouds.conf") + + parser.add_option("-b", "--benchmarking_file", action="store", dest="benchmarking_file", + help="Location of the file with benchmarking parameters (default: etc/benchmarking.conf).") + parser.set_defaults(benchmarking_file="etc/benchmarking.conf") + (options, args) = parser.parse_args() return (options, args) diff --git a/resources/__init__.py b/resources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/resources/cloud/clouds.py b/resources/cloud/clouds.py index 8be8068..b38e33b 100644 --- a/resources/cloud/clouds.py +++ b/resources/cloud/clouds.py @@ -1,8 +1,82 @@ +import logging +import os + +from boto.ec2.connection import EC2Connection +from boto.ec2.regioninfo import RegionInfo + +LOG = logging.getLogger(__name__) + class Cloud(object): - def __init__(self): - pass + """ Cloud class provides functionality for connecting to a specified cloud and launching an instance there + + cloud_name should match one of the section names in the file that specifies cloud information + + """ + def __init__(self, cloud_name, config): + self.config = config + self.name = cloud_name + self.cloud_config = self.config.clouds.config + self.cloud_uri = self.cloud_config.get(self.name, "cloud_uri") + self.cloud_port = int(self.cloud_config.get(self.name, "cloud_port")) + self.cloud_type = self.cloud_config.get(self.name, "cloud_type") + self.image_id = self.cloud_config.get(self.name, "image_id") + self.access_var = self.cloud_config.get(self.name, "access_id").strip('$') + self.secret_var = self.cloud_config.get(self.name, "secret_key").strip('$') + self.access_id = os.environ[self.access_var] + self.secret_key = os.environ[self.secret_var] + self.conn = None + + def connect(self): + """ Connects to the cloud using boto interface """ + + self.region = RegionInfo(name=self.cloud_type, endpoint=self.cloud_uri) + self.conn = EC2Connection( + self.access_id, self.secret_key, + port=self.cloud_port, region=self.region) + self.conn.host = self.cloud_uri + LOG.debug("Connected to cloud: %s" % (self.name)) + def register_key(self): + """ Registers the public key that will be used in the launched instance """ + + with open(self.config.globals.key_path,'r') as key_file_object: + key_content = key_file_object.read().strip() + import_result = self.conn.import_key_pair(self.config.globals.key_name, key_content) + LOG.debug("Registered key \"%s\"" % (self.config.globals.key_name)) + return import_result + + def boot_image(self): + """ Registers the public key and launches an instance of specified image """ + + # Check if a key with specified name is already registered. If not, register the key + registered = False + for key in self.conn.get_all_key_pairs(): + if key.name == self.config.globals.key_name: + registered = True + break + if not registered: + self.register_key() + else: + LOG.debug("Key \"%s\" is already registered" % (self.config.globals.key_name)) + + image_object = self.conn.get_image(self.image_id) + boot_result = image_object.run(key_name=self.config.globals.key_name) + LOG.debug("Attempted to boot an instance. Result: %s" % (boot_result)) + return boot_result class Clouds(object): - def __init__(self): - pass + """ Clusters class represents a collection of clouds specified in the clouds file """ + + def __init__(self, config): + self.config = config + self.list = list() + for cloud_name in self.config.clouds.list: + self.list.append(Cloud(cloud_name, self.config)) + + def lookup_by_name(self, name): + """ Finds a cloud in the collection with a given name; if does not exist, returns None """ + + for cloud in self.list: + if cloud.name == name: + return cloud + return None \ No newline at end of file diff --git a/resources/cluster/clusters.py b/resources/cluster/clusters.py index 00199b7..06bf708 100644 --- a/resources/cluster/clusters.py +++ b/resources/cluster/clusters.py @@ -1,8 +1,85 @@ +# coding=utf-8 +import logging + +from resources.cloud.clouds import Cloud, Clouds + +LOG = logging.getLogger(__name__) + class Cluster(object): - def __init__(self): - pass + """ Cluster class represents resources used for a set of benchmarks. + + Each section of the file that specifies benchmarks + might have references to sections of the file that specifies available clouds, e.g.: + sierra = 0 + hotel = 1 + In this case "sierra" is a reference to the "sierra" cloud, "hotel" is a reference to + the "hotel" cloud. References should exactly match section names in the cloud file + (both references and section names are case-sensitive). + + """ + def __init__(self, config, avail_clouds, benchmark): + self.config = config + self.benchmark = benchmark + self.clouds = list() # clouds from which instances are requested + self.requests = list() # number of instances requested + for option in self.benchmark.dict: + cloud = avail_clouds.lookup_by_name(option) + request = int(self.benchmark.dict[option]) + if cloud != None and request > 0: + self.clouds.append(cloud) + self.requests.append(request) + if len(self.clouds) == 0: + LOG.debug("Benchmark \"%s\" does not have references to available clouds" % (self.benchmark.name)) + self.reservations = list() # list of reservations that is populated in the launch() method + + def connect(self): + """ Establishes connections to the clouds from which instances are requested """ + + for cloud in self.clouds: + cloud.connect() + + def launch(self): + """ Launches requested instances and populates reservation list """ + for i in range(len(self.clouds)): # for every cloud + for j in range(self.requests[i]): # spawn as many instances as requested + reservation = self.clouds[i].boot_image() + self.reservations.append(reservation) + + def log_info(self): + """ Loops through reservations and logs status information for every instance """ + + for reservation in self.reservations: + for instance in reservation.instances: + status = "Cluster: %s, Reservation: %s, Instance: %s, Status: %s, FQDN: %s, Key: %s" %\ + (self.benchmark.name, reservation.id, instance.id, instance.state, + instance.public_dns_name, instance.key_name) + LOG.debug(status) + + def get_fqdns(self): + """ Loops through reservations and returns Fully Qualified Domain Name (FQDN) for every instance """ + + fqdns = list() + for reservation in self.reservations: + for instance in reservation.instances: + fqdns.append(instance.public_dns_name) + return fqdns + + def terminate_all(self): + """ Loops through reservations and terminates every instance """ + for reservation in self.reservations: + for instance in reservation.instances: + instance.terminate() + LOG.debug("Terminated instance: " + instance.id) class Clusters(object): - def __init__(self): - pass + """ Clusters class represents a collection of clusters specified in the benchmarking file """ + + def __init__(self, config): + self.config = config + avail_clouds = Clouds(self.config) + + self.list = list() + for benchmark in self.config.benchmarking.list: + LOG.debug("Creating cluster for benchmark: " + benchmark.name) + self.list.append(Cluster(self.config, avail_clouds, benchmark)) \ No newline at end of file