Merge pull request #2 from franTarkenton/master

Configure for use in Pipeline

loggingConfig.yaml

@@ -0,0 +1,35 @@
+version: 1
+formatters:
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(lineno)d - %(message)s"
+    datefmt: "%Y-%m-%d %H:%M:%S"
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: simple
+    stream: ext://sys.stdout
+loggers:
+  Reporting:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+  ScheduleEvaluation:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+  Emailer:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+  DBEvaluation:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+  DataUtil:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+root:
+  level: INFO
+  handlers: [console]

requirements.txt

@@ -1,9 +1,10 @@
-git+https://github.com/bcgov/dbc-pylib@v3.0.7#egg=DBCSecrets&subdirectory=DBCSecrets
-git+https://github.com/bcgov/dbc-pylib@v3.0.7#egg=PMP&subdirectory=PMP
-git+https://github.com/bcgov/dbc-pylib@v3.0.7#egg=FMEUtil&subdirectory=FMEUtil
-git+https://github.com/bcgov/dbc-pylib@v3.0.7#egg=KirkUtil&subdirectory=KirkUtil
-git+https://github.com/bcgov/dbc-pylib@v3.0.7#egg=DB&subdirectory=DB
-requests==2.20.1
+git+https://github.com/bcgov/dbc-pylib@v4.0.0#egg=DBCSecrets&subdirectory=DBCSecrets
+git+https://github.com/bcgov/dbc-pylib@v4.0.0#egg=PMP&subdirectory=PMP
+git+https://github.com/bcgov/dbc-pylib@v4.0.0#egg=FMEUtil&subdirectory=FMEUtil
+git+https://github.com/bcgov/dbc-pylib@v4.0.0#egg=KirkUtil&subdirectory=KirkUtil
+git+https://github.com/bcgov/dbc-pylib@v4.0.0#egg=DB&subdirectory=DB
+requests==2.21.1
 enum34==1.1.6
 python-dateutil==2.7.5
 cx_Oracle==7.0.0
+PyYAML>=4

src/DataUtil.py

@@ -16,6 +16,7 @@
 import Constants
 import pprint
 
+
 class GetData(object):
 
     def __init__(self, env):
@@ -24,8 +25,8 @@ def __init__(self, env):
 
         self.fme = None
         self.kirk = None
-        
-        # schedules can be used by more than one query here 
+
+        # schedules can be used by more than one query here
         # so the results are cached.
         self.schedules = None
 
@@ -35,7 +36,7 @@ def getKIRK(self):
         returns the wrapper object
         :return: a reference to the kirk api python wrapper
         :rtype: KirkUtil.PyKirk.Kirk
-        
+
         '''
         if self.kirk is None:
             secrets = DBCSecrets.GetSecrets.CredentialRetriever()
@@ -69,7 +70,7 @@ def getFME(self):
             self.fme = PyFMEServer.FMEServer(fmeBaseUrl, fmeToken)
 
         return self.fme
-    
+
     def getMiscParam(self, label):
         secrets = DBCSecrets.GetSecrets.CredentialRetriever()
         miscParams = secrets.getMiscParams()
@@ -78,9 +79,9 @@ def getMiscParam(self, label):
 
     def getFMESchedules(self):
         '''
-        :return: a schedules data object (wraps the schedules data, and 
-                 provides builtin iterator) 
-        
+        :return: a schedules data object (wraps the schedules data, and
+                 provides builtin iterator)
+
         :rtype: FMEUtil.FMEServerApiData.Schedules
         '''
         if self.schedules is None:
@@ -89,13 +90,13 @@ def getFMESchedules(self):
             scheduleList = scheds.getSchedules()
             self.schedules = FMEUtil.FMEServerApiData.Schedules(scheduleList)
         return self.schedules
-    
+
     def getFMWs(self, repoName):
         '''
         gets the summary information about each FMW in the provided
         respository name.
-        
-        :param repoName: name of the repository who's workspaces are 
+
+        :param repoName: name of the repository who's workspaces are
                          to be retrieved
         :type repoName: str
         '''
@@ -105,13 +106,13 @@ def getFMWs(self, repoName):
         wrkspcList = wrkspcs.getWorkspaces()
         wrkspcsDataObj = FMEUtil.FMEServerApiData.Workspaces(wrkspcList)
         return wrkspcsDataObj
-    
+
     def getScheduledFMWDetailInfo(self):
         '''
-        identifies the source fmw / repo for each schedule item then 
-        issues a query to retrieve detailed information about that 
+        identifies the source fmw / repo for each schedule item then
+        issues a query to retrieve detailed information about that
         repository, caches all this info in a list and then returns
-        
+
         Can take a few minutes to run
         '''
         scheds = self.getFMESchedules()
@@ -126,8 +127,8 @@ def getScheduledFMWDetailInfo(self):
             wrkspcInfo = wrkspc.getWorkspaceInfo(fmwName)
             wrkspcData = FMEUtil.FMEServerApiData.WorkspaceInfo(wrkspcInfo)
             detailedFMWInfo.append(wrkspcData)
-            print '*-*-'*20
+            self.logger.debug('*-*-' * 20)
+            # print '*-*-'*20
             pp.pprint(wrkspcInfo)
         return detailedFMWInfo
 
-

src/Emailer.py

@@ -167,12 +167,12 @@ def __init__(self, emailTo, emailFrom, emailSubject, emailBody=None):
             msg = 'The emailTo parameter should be a list.  It is currently a' + \
                   ' {0}.  Values: {1}'
             msg = msg.format(type(self.emailTo), self.emailTo)
-            raise ValueError, msg
+            raise ValueError(msg)
         if not isinstance(self.emailFrom, list):
             msg = 'The emailFrom parameter should be a list.  It is currently a' + \
                   ' {0}.  Values: {1}'
             msg = msg.format(type(self.emailFrom), self.emailFrom)
-            raise ValueError, msg
+            raise ValueError(msg)
 
     def getFMWNameLogFile(self):
         '''
@@ -219,11 +219,11 @@ def __init__(self, emailSrvr, emailObj):
         if not isinstance(emailSrvr, EmailServer):
             msg = 'The property emailSrvr in the class constructor received an ' + \
                   'object of type {0}.  This property must of type: EmailServer'
-            raise ValueError, msg.format(type(emailSrvr))
+            raise ValueError(msg.format(type(emailSrvr)))
         if not isinstance(emailObj, Email):
             msg = 'The property emailObj in the class constructor received an ' + \
                   'object of type {0}.  This property must of type: Email'
-            raise ValueError, msg.format(type(emailObj))
+            raise ValueError(msg.format(type(emailObj)))
 
     def setup(self):
         '''
@@ -239,7 +239,9 @@ def setup(self):
                           ','.join(self.emailObj.emailTo))
 
         # Create the body of the message (a plain-text and an HTML version).
-        body = MIMEText(self.emailObj.emailBody, 'html')
+        #body = MIMEText(self.emailObj.emailBody, 'html')
+        body = MIMEText(self.emailObj.emailBody, 'plain')
+
 
         # Record the MIME types of both parts - text/plain and text/html.
         # body = MIMEText(html, 'html')

src/Reporting.py

@@ -77,6 +77,7 @@ def getDisableEmailStr(self, disabledList):
         # append each line in the msg list into a carriage return delimited
         # string
         msgStr = '\n'.join(msgList)
+        #msgStr = '<p>{0}</p>'.format(msgStr)
         return msgStr
 
     def getUnsheduledRepoFMWsStr(self, unscheduledList, repositoryName):
@@ -135,6 +136,10 @@ def getNonProdSchedulesEmailStr(self, embedData):
         formatStr = self.getFormatString(embedData)
         self.logger.info("format string is: {0}".format(formatStr))
         for lineData in embedData:
+            for cnt in range(0, len(lineData)):
+                if lineData[cnt] is None:
+                    lineData[cnt] = 'None'
+            self.logger.debug("lineData: {0}".format(lineData))
             msgList.append(formatStr.format(*lineData))
         msgStr = '\n'.join(msgList)
         return msgStr

src/main.py

@@ -9,30 +9,44 @@
 '''
 import ScheduleEvaluation
 import DataUtil
-import logging
+import logging.config
 import Constants
 import Reporting
 import DBEvaluation
 import warnings
 import Emailer
+import yaml
+import os.path
 
 msg = 'Unverified HTTPS request is being made. Adding certificate verif' + \
       'ication is strongly advised. See: https://urllib3.readthedocs.i' + \
       'o/en/latest/advanced-usage.html#ssl-warnings'
 warnings.filterwarnings("ignore", message=msg)
 
+
+
 if __name__ == '__main__':
-    # Settup logger
-    formatStr = '%(asctime)s - %(lineno)s - %(name)s - %(levelname)s - %(lineno)d - %(message)s'
-    datefmt = '%Y-%m-%d %H:%M:%S'
-    # logging.basicConfig(level=logging.DEBUG)
-    # formatStr = '[%(asctime)s] - %(name)s - {%(pathname)s:%(lineno)d} ' + \
-    #            '%(levelname)s - %(message)s', datefmt
-    logging.basicConfig(format=formatStr,
-                        datefmt=datefmt,
-                        level=logging.DEBUG)
+    logConfigYaml = os.path.join(os.path.dirname(__file__), '..',
+                                 'loggingConfig.yaml')
+    with open(logConfigYaml, 'rt') as f:
+        config = yaml.safe_load(f.read())
+    logging.config.dictConfig(config)
+
+
+    # Settup logger initially
+#     formatStr = '%(asctime)s - %(name)s - %(levelname)s - %(lineno)d - %(message)s'
+#     datefmt = '%Y-%m-%d %H:%M:%S'
+#     # logging.basicConfig(level=logging.DEBUG)
+#     # formatStr = '[%(asctime)s] - %(name)s - {%(pathname)s:%(lineno)d} ' + \
+#     #            '%(levelname)s - %(message)s', datefmt
+#     logging.basicConfig(format=formatStr,
+#                         datefmt=datefmt,
+#                         level=logging.DEBUG)
+
+
+
     logger = logging.getLogger(__name__)
-
+    logger.info("fiurst log message!")
     # define what environment to work with
     env = 'PRD'
 
@@ -48,8 +62,8 @@
     disabledSchedules = schedsEval.getDisabled()
     schedEvalStr = emailReporter.getDisableEmailStr(disabledSchedules)
     dataCache.setString(schedEvalStr)
-    print 'schedEvalStr'
-    print schedEvalStr
+    #print 'schedEvalStr'
+    #print schedEvalStr
 
     # getting fmw's in scheduled repo that don't have schedules
     scheduledRepoName = dataUtil.getMiscParam(Constants.SCHEDULE_REPO_LABEL)
@@ -58,34 +72,33 @@
     unschedFMWsStr = emailReporter.getUnsheduledRepoFMWsStr(notScheduled,
                                                             scheduledRepoName)
     dataCache.setString(unschedFMWsStr)
-    print 'unschedFMWsStr'
-    print unschedFMWsStr
+    #print 'unschedFMWsStr'
+    #print unschedFMWsStr
 
     # schedules that reference data on the E: drive
     embedData = schedsEval.getEmbeddedData()
     embedDataEmailStr = emailReporter.getEmbeddedDataEmailStr(embedData)
     dataCache.setString(embedDataEmailStr)
-    print 'embedDataEmailStr'
-    print embedDataEmailStr
+    #print 'embedDataEmailStr'
+    #print embedDataEmailStr
 
-    ''' comment out at these take a while to run
     # now non prod or non OTHR replications
     nonProd = schedsEval.getNonProdSchedules()
     nonProdEmailStr = emailReporter.getNonProdSchedulesEmailStr(nonProd)
     dataCache.setString(nonProdEmailStr)
     logger.info('nonProd: %s', nonProd)
-    print 'nonProdEmailStr:'
-    print nonProdEmailStr
+    #print 'nonProdEmailStr:'
+    #print nonProdEmailStr
 
     # get destinations with 0 records
-    nonProd = schedsEval.getAllBCGWDestinations()
-    db = DBEvaluation.DBScheduleQueries(nonProd)
-    schedsZeroRecords = db.getZeroRecordDestinations()
-    zeroRecords = emailReporter.getZeroRecordsSchedule(schedsZeroRecords)
-    dataCache.setString(zeroRecords)
-    print 'zeroRecords'
-    print zeroRecords
-    '''
+#     nonProd = schedsEval.getAllBCGWDestinations()
+#     db = DBEvaluation.DBScheduleQueries(nonProd)
+#     schedsZeroRecords = db.getZeroRecordDestinations()
+#     zeroRecords = emailReporter.getZeroRecordsSchedule(schedsZeroRecords)
+#     dataCache.setString(zeroRecords)
+    #print 'zeroRecords'
+    #print zeroRecords
+
     # now send the email
     emailer = Emailer.EmailCoorindator(dataCache)
     emailer.sendEmail()