Added config files: global.conf, benchmarking.conf, clouds.conf

Added lib/config.py with classes that retrieve and store options from the config files
Implemented classes Cloud, Clouds, Cluster, Clusters

Update cu-csc#1: changed Log.info to Log.debug
Update cu-csc#2: removed etc/automaton.conf
Update cu-csc#3: added a check before registering a key
Update cu-csc#4: replaced string concatenation in %s
Update cu-csc#5: added file default locations in the help for command line arguments
Update cu-csc#6: replaces old function names with log_info and get_fqdns

.gitignore

@@ -25,3 +25,6 @@ pip-log.txt
 
 #Mr Developer
 .mr.developer.cfg
+
+#PyCharm
+.idea

automaton.py

@@ -2,40 +2,52 @@
 
 import logging
 import signal
+from threading import Thread
 
 from lib.logger import configure_logging
 from lib.util import parse_options
 from lib.util import read_config
-from threading import Thread
-
+from lib.config import Config
+from resources.cloud.clouds import Cloud, Clouds
+from resources.cluster.clusters import Clusters
 
 SIGEXIT = False
 LOG = logging.getLogger(__name__)
 
 
 class Automaton(Thread):
-    def __init__(self, config):
+    def __init__(self, config, clusters):
         Thread.__init__(self)
         self.config = config
+        self.clusters = clusters
 
     def run(self):
         LOG.info("Starting Automaton")
         #TODO(pdmars): do something
 
+        # Code below demonstrates functionality of Cluster class:
+        # for cluster in self.clusters.list:
+        #    cluster.connect()
+        #    cluster.launch()
+        #    cluster.log_info()
+        #    fqdns = cluster.get_fqdns()
+        #    cluster.terminate_all()
 
 def clean_exit(signum, frame):
     global SIGEXIT
     SIGEXIT = True
     LOG.critical("Exit signal received. Exiting at the next sane time. "
                  "Please stand by.")
 
-
 def main():
     (options, args) = parse_options()
     configure_logging(options.debug)
-    config = read_config(options.config_file)
+
+    config = Config(options)
+    clusters = Clusters(config)
+
     signal.signal(signal.SIGINT, clean_exit)
-    automaton = Automaton(config)
+    automaton = Automaton(config, clusters)
     automaton.start()
     # wake every seconed to make sure signals are handled by the main thread
     # need this due to a quirk in the way Python threading handles signals

etc/benchmarking.conf

@@ -0,0 +1,8 @@
+[Benchmark-01]
+sierra = 0
+hotel = 1
+
+[Benchmark-02]
+
+[Benchmark-03]
+

etc/clouds.conf

@@ -0,0 +1,21 @@
+[hotel]
+cloud_uri = svc.uc.futuregrid.org
+cloud_port = 8444
+image_id = debian-6.0.5.gz
+cloud_type = nimbus
+availability_zone = us-east-1
+instance_type = m1.paul
+instance_cores = 1
+access_id = $NIMBUS_IAAS_ACCESS_KEY
+secret_key = $NIMBUS_IAAS_SECRET_KEY
+
+[sierra]
+cloud_uri = s83r.idp.sdsc.futuregrid.org
+cloud_port = 8444
+image_id = debian-lenny.gz
+cloud_type = nimbus
+availability_zone = us-east-1
+instance_type = m1.paul
+instance_cores = 1
+access_id = $NIMBUS_IAAS_ACCESS_KEY
+secret_key = $NIMBUS_IAAS_SECRET_KEY

etc/global.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+key_name = automaton
+key_path = /Users/dmdu/.ssh/id_rsa_futuregrid.pub

lib/config.py

@@ -1,3 +1,46 @@
-class AutomatonConfig(object):
-    def __init__(self, name, config):
-        pass
+from lib.util import read_config
+
+class GlobalConfig(object):
+    """ GlobalConfig class retrieves information from the file that specifies global parameters """
+
+    def __init__(self, config):
+        self.config = config
+        default_dict = self.config.defaults()
+        self.key_name = default_dict['key_name']
+        self.key_path = default_dict['key_path']
+
+class CloudsConfig(object):
+    """ CloudsConfig class retrieves information from the file that specifies global parameters """
+
+    def __init__(self, config):
+        self.config = config
+        self.list = config.sections()
+
+class Benchmark(object):
+    """ Benchmark class retrieves information from one of the section of the benchmarking file """
+
+    def __init__(self, benchmark_name, config):
+        self.config = config
+        self.name = benchmark_name
+        dict = self.config.items(self.name)
+        self.dict = {}
+        # Form a dictionary out of items in the specified section
+        for pair in dict:
+            self.dict[pair[0]] = pair[1]
+
+class BenchmarkingConfig(object):
+    """ BenchmarkingConfig class retrieves benchmarking information and populates benchmark list """
+
+    def __init__(self, config):
+        self.config = config
+        self.list = list()
+        for sec in self.config.sections():
+            self.list.append(Benchmark(sec, self.config))
+
+class Config(object):
+    """ Config class retrieves all configuration information """
+
+    def __init__(self, options):
+        self.globals = GlobalConfig(read_config(options.global_file))
+        self.clouds = CloudsConfig(read_config(options.clouds_file))
+        self.benchmarking = BenchmarkingConfig(read_config(options.benchmarking_file))

lib/util.py

@@ -23,19 +23,30 @@ def execute(self):
         return process.returncode
 
 
-def read_config(config_file):
+def read_config(file):
     config = SafeConfigParser()
-    config.read(config_file)
+    config.read(file)
     return config
 
 
 def parse_options():
     parser = OptionParser()
-    parser.add_option("-c", "--config_file", action="store",
-                      dest="config_file", help="Location of the config file.")
+
     parser.add_option("-d", "--debug", action="store_true", dest="debug",
                       help="Enable debugging log level.")
-    parser.set_defaults(config_file="etc/automaton.conf")
     parser.set_defaults(debug=False)
+
+    parser.add_option("-g", "--global_file", action="store", dest="global_file",
+                      help="Location of the file with global parameters (default: etc/global.conf).")
+    parser.set_defaults(global_file="etc/global.conf")
+
+    parser.add_option("-c", "--clouds_file", action="store", dest="clouds_file",
+                      help="Location of the file with cloud parameters (default: etc/clouds.conf).")
+    parser.set_defaults(clouds_file="etc/clouds.conf")
+
+    parser.add_option("-b", "--benchmarking_file", action="store", dest="benchmarking_file",
+                      help="Location of the file with benchmarking parameters (default: etc/benchmarking.conf).")
+    parser.set_defaults(benchmarking_file="etc/benchmarking.conf")
+
     (options, args) = parser.parse_args()
     return (options, args)

resources/cloud/clouds.py

@@ -1,8 +1,82 @@
+import logging
+import os
+
+from boto.ec2.connection import EC2Connection
+from boto.ec2.regioninfo import RegionInfo
+
+LOG = logging.getLogger(__name__)
+
 class Cloud(object):
-    def __init__(self):
-        pass
+    """ Cloud class provides functionality for connecting to a specified cloud and launching an instance there
+
+    cloud_name should match one of the section names in the file that specifies cloud information
+
+    """
+    def __init__(self, cloud_name, config):
+        self.config = config
+        self.name = cloud_name
+        self.cloud_config = self.config.clouds.config
+        self.cloud_uri = self.cloud_config.get(self.name, "cloud_uri")
+        self.cloud_port = int(self.cloud_config.get(self.name, "cloud_port"))
+        self.cloud_type = self.cloud_config.get(self.name, "cloud_type")
+        self.image_id = self.cloud_config.get(self.name, "image_id")
+        self.access_var = self.cloud_config.get(self.name, "access_id").strip('$')
+        self.secret_var = self.cloud_config.get(self.name, "secret_key").strip('$')
+        self.access_id = os.environ[self.access_var]
+        self.secret_key = os.environ[self.secret_var]
+        self.conn = None
+
+    def connect(self):
+        """ Connects to the cloud using boto interface """
+
+        self.region = RegionInfo(name=self.cloud_type, endpoint=self.cloud_uri)
+        self.conn = EC2Connection(
+            self.access_id, self.secret_key,
+            port=self.cloud_port, region=self.region)
+        self.conn.host = self.cloud_uri
+        LOG.debug("Connected to cloud: %s" % (self.name))
 
+    def register_key(self):
+        """ Registers the public key that will be used in the launched instance """
+
+        with open(self.config.globals.key_path,'r') as key_file_object:
+            key_content = key_file_object.read().strip()
+        import_result = self.conn.import_key_pair(self.config.globals.key_name, key_content)
+        LOG.debug("Registered key \"%s\"" % (self.config.globals.key_name))
+        return import_result
+
+    def boot_image(self):
+        """ Registers the public key and launches an instance of specified image  """
+
+        # Check if a key with specified name is already registered. If not, register the key
+        registered = False
+        for key in self.conn.get_all_key_pairs():
+            if key.name == self.config.globals.key_name:
+                registered = True
+                break
+        if not registered:
+            self.register_key()
+        else:
+            LOG.debug("Key \"%s\" is already registered" % (self.config.globals.key_name))
+
+        image_object = self.conn.get_image(self.image_id)
+        boot_result = image_object.run(key_name=self.config.globals.key_name)
+        LOG.debug("Attempted to boot an instance. Result: %s" % (boot_result))
+        return boot_result
 
 class Clouds(object):
-    def __init__(self):
-        pass
+    """ Clusters class represents a collection of clouds specified in the clouds file """
+
+    def __init__(self, config):
+        self.config = config
+        self.list = list()
+        for cloud_name in self.config.clouds.list:
+            self.list.append(Cloud(cloud_name, self.config))
+
+    def lookup_by_name(self, name):
+        """ Finds a cloud in the collection with a given name; if does not exist, returns None """
+
+        for cloud in self.list:
+            if cloud.name == name:
+                return cloud
+        return None

resources/cluster/clusters.py

@@ -1,8 +1,85 @@
+# coding=utf-8
+import logging
+
+from resources.cloud.clouds import Cloud, Clouds
+
+LOG = logging.getLogger(__name__)
+
 class Cluster(object):
-    def __init__(self):
-        pass
+    """ Cluster class represents resources used for a set of benchmarks.
+
+    Each section of the file that specifies benchmarks
+    might have references to sections of the file that specifies available clouds, e.g.:
+      sierra = 0
+      hotel = 1
+    In this case "sierra" is a reference to the "sierra" cloud, "hotel" is a reference to
+    the "hotel" cloud. References should exactly match section names in the cloud file
+    (both references and section names are case-sensitive).
+
+    """
+    def __init__(self, config, avail_clouds, benchmark):
+        self.config = config
+        self.benchmark = benchmark
+        self.clouds = list()        # clouds from which instances are requested
+        self.requests = list()      # number of instances requested
+        for option in self.benchmark.dict:
+            cloud = avail_clouds.lookup_by_name(option)
+            request = int(self.benchmark.dict[option])
+            if cloud != None and request > 0:
+                self.clouds.append(cloud)
+                self.requests.append(request)
+        if len(self.clouds) == 0:
+            LOG.debug("Benchmark \"%s\" does not have references to available clouds" % (self.benchmark.name))
+        self.reservations = list()  # list of reservations that is populated in the launch() method
+
+    def connect(self):
+        """ Establishes connections to the clouds from which instances are requested """
+
+        for cloud in self.clouds:
+            cloud.connect()
+
+    def launch(self):
+        """ Launches requested instances and populates reservation list """
 
+        for i in range(len(self.clouds)):           # for every cloud
+            for j in range(self.requests[i]):       # spawn as many instances as requested
+                reservation = self.clouds[i].boot_image()
+                self.reservations.append(reservation)
+
+    def log_info(self):
+        """ Loops through reservations and logs status information for every instance """
+
+        for reservation in self.reservations:
+            for instance in reservation.instances:
+                status = "Cluster: %s, Reservation: %s, Instance: %s, Status: %s, FQDN: %s, Key: %s" %\
+                          (self.benchmark.name, reservation.id, instance.id, instance.state,
+                          instance.public_dns_name, instance.key_name)
+                LOG.debug(status)
+
+    def get_fqdns(self):
+        """ Loops through reservations and returns Fully Qualified Domain Name (FQDN) for every instance """
+
+        fqdns = list()
+        for reservation in self.reservations:
+            for instance in reservation.instances:
+                fqdns.append(instance.public_dns_name)
+        return fqdns
+
+    def terminate_all(self):
+        """ Loops through reservations and terminates every instance """
+        for reservation in self.reservations:
+            for instance in reservation.instances:
+                instance.terminate()
+                LOG.debug("Terminated instance: " + instance.id)
 
 class Clusters(object):
-    def __init__(self):
-        pass
+    """ Clusters class represents a collection of clusters specified in the benchmarking file """
+
+    def __init__(self, config):
+        self.config = config
+        avail_clouds = Clouds(self.config)
+
+        self.list = list()
+        for benchmark in self.config.benchmarking.list:
+            LOG.debug("Creating cluster for benchmark: " + benchmark.name)
+            self.list.append(Cluster(self.config, avail_clouds, benchmark))