Merge pull request #436 from HubSpot/logfetch_polish

more logfetch features (WIP)

scripts/README.md

@@ -14,22 +14,28 @@ pip install singularity-logfetch
 - Any arguments specified in the log file can be overriden on the command line
 - You can store a number of configuration files for different clusters in the config directory (`~/.logfetch` by default) and choose which config to use with the -c option
 
+#Logfetch and Logcat
+Two commands exist for downloading logs. 
+- `logfetch` will download and optionally output a grep command for the logs
+- `logcat` will download logs and pipe the contents to stdout
+
 ##Options
 |Flags|Description|Default|
 |:---:|:---------|:-----:|
-|-f , --conf_folder|Folder to look for configuration files|`~/.logfetch`|
-|-c , --conf_file|configuration file to use(path relative to conf_folder)|default|
-|-t , --taskId|TaskId to fetch logs for|
-|-r , --requestId|RequestId to fetch logs for|
-|--task-count|Number of recent tasks (belonging to a request) to fetch live logs (on machine not s3)|1|
-|-d , --deployId|DeployId to fetch logs for (Must also specify requestId when using this option)|
-|--dest|Destination folder for downloaded log files|`~/.logfetch_cache`|
+|-f , --conf-folder|Folder to look for configuration files|`~/.logfetch`|
+|-c , --conf-file|configuration file to use(path relative to conf_folder)|default|
+|-t , --task-id|Task Id to fetch logs for|
+|-r , --request-id|Request Id to fetch logs for|
+|-tc, --task-count|Number of recent tasks (belonging to a request) to fetch live logs (on machine not s3)|1|
+|-d , --deploy-id|Deploy Id to fetch logs for (Must also specify requestId when using this option)|
+|-o, --dest|Destination folder for download output|`~/.logfetch_cache`|
 |-n --num-parallel-fetches|Max number of log fetches to make at once|5
-|-cs, --chunk_size|Chunk size for writing responses to file system|8192
-|-u, --singularity-uri-base|Base url for singularity (e.g. `localhost:8080/singularity/v2/api`), This MUST be set|
+|-cs, --chunk-size|Chunk size for writing responses to file system|8192
+|-u, --singularity-uri-base|Base url for singularity (e.g. `localhost:8080/singularity/v2/api`)| Must be set!|
 |-s , --start-days|Search for logs no older than this many days|7
 |-e , --end-days|Search for logs no newer than this many days| None (today)
-|-g, --grep|Grep string for searching log files|
+|-g, --grep|Grep string for searching log files(Only for `logfetch`)|
+|-l, --logtype|Glob matcher for type of log file to download| None (match all)|
 
 ##Grep and Log Files
 When the `-g` option is set, the log fetcher will grep the downloaded files for the provided regex.
@@ -64,9 +70,14 @@ When the `-g` option is set, the log fetcher will grep the downloaded files for
 
 - Don't search, just download logs
 
-`logfetch -r 'My_Jobs_Id'`
+`logfetch -r 'My_Request_Id'`
+
+- Only get logs that match a glob or logfile name with the `-l` option
 
-##Tailing Logs
+`logfetch -r ‘My_Request_Id’ -l ‘*.out’`
+`logfetch -r ‘My_Request_Id’ -l ‘access.log’`
+
+#Logtail
 You can tail live log files using `logtail`. Just provide the request, task, or request and deploy along with a log file path.
 
 For example, to tail the `service.log` file for all tasks for a request named `MyRequest`, you would use the command:
@@ -76,3 +87,17 @@ For example, to tail the `service.log` file for all tasks for a request named `M
 - The path for the log file is relative to the base path for that task's sandbox. For example, to tail a file in `(sandbox path)/logs/access.log`, the argument to -l would be `logs/access.log`
 
 You can also provide the `-g` option which will provide the grep string to the singularity API and search the results. You cannot provide a full grep command as in some of the above examples, just a string to match on.
+
+##Options
+|Flags|Description|Default|
+|:---:|:---------|:-----:|
+|-f , --conf-folder|Folder to look for configuration files|`~/.logfetch`|
+|-c , --conf-file|configuration file to use(path relative to conf_folder)|default|
+|-t , --task-id|Task Id to fetch logs for|
+|-r , --request-id|Request Id to fetch logs for|
+|-d , --deploy-id|Deploy Id to fetch logs for (Must also specify requestId when using this option)|
+|-u, --singularity-uri-base|Base url for singularity (e.g. `localhost:8080/singularity/v2/api`)|Must be set!|
+|-g, --grep|Grep string for searching log files|
+|-l, --logfile|Log file path to tail (ie logs/access.log)|Must be set!|
+|-v, --verbose|Extra output about the task id associated with logs in the output|False|
+

scripts/logfetch/callbacks.py

@@ -10,7 +10,7 @@ def callback(response, **kwargs):
     with open(path, 'wb') as f:
       for chunk in response.iter_content(chunk_size):
         f.write(chunk)
-    sys.stderr.write(colored('finished downloading {0}'.format(path), 'green') + '\n')
+    sys.stderr.write(colored('Downloaded ', 'green') + colored(path, 'white') + '\n')
 
   return callback
 

scripts/logfetch/cat.py

@@ -0,0 +1,23 @@
+import os
+import sys
+
+CAT_COMMAND_FORMAT = 'xargs -n {0} cat < {1}'
+def cat_files(args, all_logs):
+    if all_logs:
+        catlist_filename = '{0}/.catlist'.format(args.dest)
+        create_catlist(args, all_logs, catlist_filename)
+        command = CAT_COMMAND_FORMAT.format(len(all_logs), catlist_filename)
+        sys.stdout.write(os.popen(command).read() + '\n')
+    else:
+        sys.stderr.write(colored('No log files found\n', 'magenta'))
+
+def create_catlist(args, all_logs, catlist_filename):
+  catlist_file = open(catlist_filename, 'wb')
+  for log in all_logs:
+    catlist_file.write('{0}\n'.format(log))
+  catlist_file.close()
+
+def remove_catlist(catlist_filename):
+  if os.path.isfile(catlist_filename):
+    os.remove(catlist_filename)
+

scripts/logfetch/entrypoint.py

@@ -3,44 +3,61 @@
 import sys
 import os
 from termcolor import colored
-
 from fake_section_head import FakeSectionHead
 from live_logs import download_live_logs
 from s3_logs import download_s3_logs
 from tail import start_tail
 from grep import grep_files
+from cat import cat_files
 
 CONF_READ_ERR_FORMAT = 'Could not load config from {0} due to {1}'
 DEFAULT_CONF_DIR = os.path.expanduser('~/.logfetch')
 DEFAULT_CONF_FILE = 'default'
-DEFAULT_PARALLEL_FETCHES = 5
+DEFAULT_PARALLEL_FETCHES = 10
 DEFAULT_CHUNK_SIZE = 8192
 DEFAULT_DEST = os.path.expanduser('~/.logfetch_cache')
 DEFAULT_TASK_COUNT = 10
 DEFAULT_DAYS = 7
 
-def exit(reason):
-  sys.stderr.write(colored(reason, 'red') + '\n')
+def exit(reason, color='red'):
+  sys.stderr.write(colored(reason, color) + '\n')
   sys.exit(1)
 
 def tail_logs(args):
-  start_tail(args)
+  try:
+    start_tail(args)
+  except KeyboardInterrupt:
+    exit('Stopping logtail...', 'magenta')
 
 def fetch_logs(args):
-  check_dest(args)
-  all_logs = []
-  all_logs += download_s3_logs(args)
-  all_logs += download_live_logs(args)
-  grep_files(args, all_logs)
+  try:
+    check_dest(args)
+    all_logs = []
+    all_logs += download_s3_logs(args)
+    all_logs += download_live_logs(args)
+    grep_files(args, all_logs)
+  except KeyboardInterrupt:
+    exit('Stopping logfetch...', 'magenta')
+
+def cat_logs(args):
+  try:
+    check_dest(args)
+    all_logs = []
+    all_logs += download_s3_logs(args)
+    all_logs += download_live_logs(args)
+    cat_files(args, all_logs)
+  except KeyboardInterrupt:
+    exit('Stopping logcat...', 'magenta')
+
 
 def check_dest(args):
   if not os.path.exists(args.dest):
     os.makedirs(args.dest)
 
 def fetch():
   conf_parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
-  conf_parser.add_argument("-f", "--conf_folder", help="specify a folder for config files to live")
-  conf_parser.add_argument("-c", "--conf_file", help="Specify config file within the conf folder", metavar="FILE")
+  conf_parser.add_argument("-f", "--conf-folder", dest='conf_folder', help="specify a folder for config files to live")
+  conf_parser.add_argument("-c", "--conf-file", dest='conf_file', help="Specify config file within the conf folder", metavar="FILE")
   args, remaining_argv = conf_parser.parse_known_args()
   conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
   conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
@@ -61,36 +78,87 @@ def fetch():
     sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')
 
   parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
-          prog="log_fetcher")
+          prog="logfetch")
 
   parser.set_defaults(**defaults)
-  parser.add_argument("-t", "--taskId", help="TaskId of task to fetch logs for", metavar="taskId")
-  parser.add_argument("-r", "--requestId", help="RequestId of request to fetch logs for", metavar="requestId")
-  parser.add_argument("--task-count", help="Number of recent tasks per request to fetch logs from", metavar="taskCount")
-  parser.add_argument("-d", "--deployId", help="DeployId of task to fetch logs for", metavar="deployId")
-  parser.add_argument("--dest", help="Destination directory", metavar="DIR")
-  parser.add_argument("-n", "--num-parallel-fetches", help="Number of fetches to make at once", type=int, metavar="INT")
-  parser.add_argument("-cs", "--chunk-size", help="Chunk size for writing from response to filesystem", type=int, metavar="INT")
-  parser.add_argument("-u", "--singularity-uri-base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)", metavar="URI")
-  parser.add_argument("-s", "--start-days", help="Search for logs no older than this many days", type=int, metavar="start_days")
-  parser.add_argument("-e", "--end-days", help="Search for logs no new than this many days (defaults to None/today)", type=int, metavar="end_days")
-  parser.add_argument("-g", "--grep", help="Regex to grep for (normal grep syntax) or a full grep command", metavar='grep')
+  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
+  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
+  parser.add_argument("-tc","--task-count", dest="task_count", help="Number of recent tasks per request to fetch logs from")
+  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of task to fetch logs for (can be a glob)")
+  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
+  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
+  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
+  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
+  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this many days", type=int)
+  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no new than this many days (defaults to None/today)", type=int)
+  parser.add_argument("-l", "--log-type", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
+  parser.add_argument("-g", "--grep", dest="grep", help="Regex to grep for (normal grep syntax) or a full grep command")
 
   args = parser.parse_args(remaining_argv)
 
   if args.deployId and not args.requestId:
-    exit("Must specify requestId (-r) when specifying deployId")
+    exit("Must specify request-id (-r) when specifying deploy-id")
   elif not args.requestId and not args.deployId and not args.taskId:
-    exit('Must specify one of\n -t taskId\n -r requestId and -d deployId\n -r requestId')
+    exit('Must specify one of\n -t task-id\n -r request-id and -d deploy-id\n -r request-id')
 
   args.dest = os.path.expanduser(args.dest)
 
   fetch_logs(args)
 
+def cat():
+  conf_parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
+  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
+  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
+  args, remaining_argv = conf_parser.parse_known_args()
+  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
+  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
+  config = ConfigParser.SafeConfigParser()
+
+  defaults = {
+    "num_parallel_fetches" : DEFAULT_PARALLEL_FETCHES,
+    "chunk_size" : DEFAULT_CHUNK_SIZE,
+    "dest" : DEFAULT_DEST,
+    "task_count" : DEFAULT_TASK_COUNT,
+    "start_days" : DEFAULT_DAYS
+  }
+
+  try:
+    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
+    defaults.update(dict(config.items("Defaults")))
+  except Exception, err:
+    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')
+
+  parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity and cat to stdout. One can specify either a TaskId, RequestId and DeployId, or RequestId",
+          prog="logcat")
+
+  parser.set_defaults(**defaults)
+  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
+  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
+  parser.add_argument("-tc","--task-count", dest="taskCount", help="Number of recent tasks per request to fetch logs from")
+  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
+  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
+  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
+  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
+  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
+  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this many days", type=int)
+  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no new than this many days (defaults to None/today)", type=int)
+  parser.add_argument("-l", "--logtype", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
+
+  args = parser.parse_args(remaining_argv)
+
+  if args.deployId and not args.requestId:
+    exit("Must specify requestId (-r) when specifying deploy-id")
+  elif not args.requestId and not args.deployId and not args.taskId:
+    exit('Must specify one of\n -t task-id\n -r request-id and -d deploy-id\n -r request-id')
+
+  args.dest = os.path.expanduser(args.dest)
+
+  cat_logs(args)
+
 def tail():
   conf_parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
-  conf_parser.add_argument("-f", "--conf_folder", help="specify a folder for config files to live")
-  conf_parser.add_argument("-c", "--conf_file", help="Specify config file within the conf folder", metavar="FILE")
+  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
+  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
   args, remaining_argv = conf_parser.parse_known_args()
   conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
   conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
@@ -105,23 +173,23 @@ def tail():
     sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')
 
   parser = argparse.ArgumentParser(parents=[conf_parser], description="Tail log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
-          prog="log_fetcher")
+          prog="logtail")
 
   parser.set_defaults(**defaults)
-  parser.add_argument("-t", "--taskId", help="TaskId of task to fetch logs for", metavar="taskId")
-  parser.add_argument("-r", "--requestId", help="RequestId of request to fetch logs for", metavar="requestId")
-  parser.add_argument("-d", "--deployId", help="DeployId of task to fetch logs for", metavar="deployId")
-  parser.add_argument("-u", "--singularity-uri-base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)", metavar="URI")
-  parser.add_argument("-g", "--grep", help="String to grep for", metavar='grep')
-  parser.add_argument("-l", "--logfile", help="Logfile path/name to tail (ie 'logs/access.log')", metavar="logfile")
-  parser.add_argument("-v", "--verbose", help="more verbose output", action='store_true')
+  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
+  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
+  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
+  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
+  parser.add_argument("-g", "--grep", dest="grep", help="String to grep for")
+  parser.add_argument("-l", "--logfile", dest="logfile", help="Logfile path/name to tail (ie 'logs/access.log')")
+  parser.add_argument("-v", "--verbose", dest="verbose", help="more verbose output", action='store_true')
 
   args = parser.parse_args(remaining_argv)
 
   if args.deployId and not args.requestId:
-    exit("Must specify requestId (-r) when specifying deployId")
+    exit("Must specify request-id (-r) when specifying deploy-id")
   elif not args.requestId and not args.deployId and not args.taskId:
-    exit('Must specify one of\n -t taskId\n -r requestId and -d deployId\n -r requestId')
+    exit('Must specify one of\n -t task-id\n -r request-id and -d deploy-id\n -r request-id')
   elif not args.logfile:
     exit("Must specify logfile to tail (-l)")