diff --git a/explore.py b/dxt-explorer old mode 100644 new mode 100755 similarity index 51% rename from explore.py rename to dxt-explorer index 56ada2b..6b439e8 --- a/explore.py +++ b/dxt-explorer @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """ DXT Explorer. @@ -26,12 +26,12 @@ import shlex import argparse import subprocess +import webbrowser import logging import logging.handlers import pandas as pd -from plotnine import * from distutils.spawn import find_executable @@ -39,16 +39,20 @@ class Explorer: def __init__(self, args): """Initialize the explorer.""" + self.args = args + self.configure_log() self.has_dxt_parser() self.has_r_support() - self.args = args - def configure_log(self): """Configure the logging system.""" self.logger = logging.getLogger('DXT Explorer') - self.logger.setLevel(logging.DEBUG) + + if self.args.debug: + self.logger.setLevel(logging.DEBUG) + else: + self.logger.setLevel(logging.INFO) # Defines the format of the logger formatter = logging.Formatter('%(asctime)s %(module)s - %(levelname)s - %(message)s') @@ -61,8 +65,13 @@ def configure_log(self): def run(self): self.is_darshan_file(self.args.darshan) - self.parse(self.args.darshan) + + if self.args.list_files: + self.list_files(self.args.darshan) + + exit() + self.generate_plot(self.args.darshan) if self.args.transfer: @@ -87,7 +96,7 @@ def is_darshan_file(self, file): def has_dxt_parser(self): """Check if `darshan-dxt-parser` is on PATH.""" if find_executable('darshan-dxt-parser') is not None: - self.logger.info('darshan-dxt-parser: FOUND') + self.logger.debug('darshan-dxt-parser: FOUND') else: self.logger.error('darshan-dxt-parser: NOT FOUND') @@ -96,7 +105,7 @@ def has_dxt_parser(self): def has_r_support(self): """Check if `Rscript` is on PATH.""" if find_executable('Rscript') is not None: - self.logger.info('Rscript: FOUND') + self.logger.debug('Rscript: FOUND') else: self.logger.error('Rscript: NOT FOUND') @@ -104,11 +113,16 @@ def has_r_support(self): def dxt(self, file): """Parse the Darshan file to generate the .dxt trace file.""" + if os.path.exists(file + '.dxt'): + self.logger.debug('using existing parsed Darshan file') + + return + command = 'darshan-dxt-parser {0}'.format(file) args = shlex.split(command) - self.logger.info('parsing {} file'.format(file)) + self.logger.debug('parsing {} file'.format(file)) with open('{}.dxt'.format(file), 'w') as output: s = subprocess.run(args, stderr=subprocess.PIPE, stdout=output) @@ -120,7 +134,12 @@ def parse(self, file): """Parse the .darshan.dxt file to generate a CSV file.""" self.dxt(file) - self.logger.info('generating an intermediate CSV file') + if os.path.exists(file + '.dxt.csv'): + self.logger.debug('using existing intermediate CSV file') + + return + + self.logger.debug('generating an intermediate CSV file') with open(file + '.dxt') as f: lines = f.readlines() @@ -139,7 +158,8 @@ def parse(self, file): 'offset', 'size', 'start', - 'end' + 'end', + 'ost' ]) for line in lines: @@ -159,6 +179,11 @@ def parse(self, file): start = info[6] end = info[7] + if len(info) == 9: + ost = info[8] + else: + ost = None + w.writerow([ file_id, api.replace('X_', ''), @@ -168,7 +193,8 @@ def parse(self, file): offset, size, start, - end + end, + ost ]) if 'X_MPIIO' in line: @@ -192,6 +218,11 @@ def parse(self, file): start = info[5] end = info[6] + if len(info) == 9: + ost = info[8] + else: + ost = None + w.writerow([ file_id, api.replace('X_', ''), @@ -201,61 +232,161 @@ def parse(self, file): offset, size, start, - end + end, + ost ]) + def list_files(self, file): + files = {} + + with open(file + '.dxt') as f: + lines = f.readlines() + + for line in lines: + if 'file_id' in line: + file_id = line.split(',')[1].split(':')[1].strip() + file_name = line.split(',')[2].split(':')[1].strip() + + if file_id not in files.keys(): + files[file_id] = file_name + + for file_id, file_name in files.items(): + self.logger.info('FILE: {} (ID {})'.format(file_name, file_id)) + + self.logger.info('{} I/O trace observation records from {} files'.format(len(lines), len(files))) + + return files + + def subset_dataset(self, file, file_ids): + for file_id in file_ids: + subset_dataset_file = '{}.{}'.format(file, file_id) + + with open(file + '.dxt.csv') as f: + rows = csv.DictReader(f) + + with open(subset_dataset_file + '.dxt.csv', 'w', newline='') as csvfile: + w = csv.writer(csvfile) + + w.writerow([ + 'file_id', + 'api', + 'rank', + 'operation', + 'segment', + 'offset', + 'size', + 'start', + 'end', + 'ost' + ]) + + for row in rows: + if file_id == row['file_id']: + w.writerow(row.values()) def generate_plot(self, file): """Generate an interactive operation plot.""" - command = 'plots/operation.R -f {0}.dxt.csv'.format(file) + limits = '' - args = shlex.split(command) + if self.args.start: + limits += ' -s {} '.format(self.args.start) - self.logger.info('generating interactive operation plot') + if self.args.end: + limits += ' -e {} '.format(self.args.end) - s = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if self.args.start_rank: + limits += ' -n {} '.format(self.args.start_rank) - if s.returncode == 0: - self.logger.info('SUCCESS') - else: - self.logger.error('Failed to generate the interactive plots (error %s)', s.returncode) + if self.args.end_rank: + limits += ' -m {} '.format(self.args.end_rank) - if s.stdout is not None: - for item in s.stdout.decode().split('\n'): - if item.strip() != '': - self.logger.debug(item) + file_ids = self.list_files(file) - if s.stderr is not None: - for item in s.stderr.decode().split('\n'): - if item.strip() != '': - self.logger.error(item) + # Generated the CSV files for each plot + observations = self.subset_dataset(file, file_ids) + + for file_id, file_name in file_ids.items(): + output_file = '{}.{}.operation.html'.format(file, file_id) + + command = 'plots/operation.R -f {}.{}.dxt.csv {} -o {} -x {}'.format(file, file_id, limits, output_file, file_name) + + args = shlex.split(command) + + self.logger.info('generating interactive operation plot') + self.logger.debug(command) + + s = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + if s.returncode == 0: + if os.path.exists(output_file): + self.logger.info('SUCCESS') + else: + self.logger.warning('no data to generate interactive plots') + + if self.args.browser: + webbrowser.open('file://{}'.format(output_file), new=2) + else: + self.logger.error('failed to generate the interactive plots (error %s)', s.returncode) + + if s.stdout is not None: + for item in s.stdout.decode().split('\n'): + if item.strip() != '': + self.logger.debug(item) + + if s.stderr is not None: + for item in s.stderr.decode().split('\n'): + if item.strip() != '': + self.logger.error(item) def generate_transfer_plot(self, file): """Generate an interactive transfer plot.""" - command = 'plots/transfer.R -f {0}.dxt.csv'.format(file) + limits = '' - args = shlex.split(command) + if self.args.start: + limits += ' -s {} '.format(self.args.start) - self.logger.info('generating interactive transfer plot') + if self.args.end: + limits += ' -e {} '.format(self.args.end) - s = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if self.args.start_rank: + limits += ' -n {} '.format(self.args.start_rank) - if s.returncode == 0: - self.logger.info('SUCCESS') - else: - self.logger.error('Failed to generate the interactive plots (error %s)', s.returncode) + if self.args.end_rank: + limits += ' -m {} '.format(self.args.end_rank) - if s.stdout is not None: - for item in s.stdout.decode().split('\n'): - if item.strip() != '': - self.logger.debug(item) + file_ids = self.list_files(file) - if s.stderr is not None: - for item in s.stderr.decode().split('\n'): - if item.strip() != '': - self.logger.error(item) + # Generated the CSV files for each plot + self.subset_dataset(file, file_ids) + + for file_id in file_ids: + command = 'plots/transfer.R -f {}.{}.dxt.csv {} -o {}.{}.transfer.html'.format(file, file_id, limits, file, file_id) + + args = shlex.split(command) + + self.logger.info('generating interactive transfer plot') + self.logger.debug(command) + + s = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + if s.returncode == 0: + self.logger.info('SUCCESS') + + if self.args.browser: + webbrowser.open('file://{}.{}.transfer.html'.format(file, file_id), new=2) + else: + self.logger.error('failed to generate the interactive plots (error %s)', s.returncode) + + if s.stdout is not None: + for item in s.stdout.decode().split('\n'): + if item.strip() != '': + self.logger.debug(item) + if s.stderr is not None: + for item in s.stderr.decode().split('\n'): + if item.strip() != '': + self.logger.error(item) def generate_spatiality_plot(self, file): """Generate an interactive spatiality plot.""" @@ -270,7 +401,7 @@ def generate_spatiality_plot(self, file): if s.returncode == 0: self.logger.info('SUCCESS') else: - self.logger.error('Failed to generate the interactive plots (error %s)', s.returncode) + self.logger.error('failed to generate the interactive plots (error %s)', s.returncode) if s.stdout is not None: for item in s.stdout.decode().split('\n'): @@ -317,6 +448,57 @@ def generate_spatiality_plot(self, file): help='Generate an interactive spatiality explorer' ) +PARSER.add_argument( + '-d', + '--debug', + action='store_true', + dest='debug', + help='Enable debug mode' +) + +PARSER.add_argument( + '-l', + '--list', + action='store_true', + dest='list_files', + help='List all the files with trace' +) + +PARSER.add_argument( + '--start', + action='store', + dest='start', + help='Report starts from X seconds (e.g., 3.7) from beginning of the job' +) + +PARSER.add_argument( + '--end', + action='store', + dest='end', + help='Report ends at X seconds (e.g., 3.9) from beginning of the job' +) + +PARSER.add_argument( + '--from', + action='store', + dest='start_rank', + help='Report start from rank N' +) + +PARSER.add_argument( + '--to', + action='store', + dest='end_rank', + help='Report up to rank M' +) + +PARSER.add_argument( + '--browser', + default=False, + action='store_true', + dest='browser', + help='Open the browser with the generated plot' +) ARGS = PARSER.parse_args() diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..f0079fe Binary files /dev/null and b/logo.png differ diff --git a/logo.svg b/logo.svg new file mode 100644 index 0000000..b6ba940 --- /dev/null +++ b/logo.svg @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plots/operation.R b/plots/operation.R index 6b338b5..639871e 100755 --- a/plots/operation.R +++ b/plots/operation.R @@ -17,12 +17,13 @@ # works, and perform publicly and display publicly, and to permit others to do so. packages <- c( - 'ggplot2', - 'optparse', - 'plyr', - 'plotly', - 'rmarkdown', - 'htmlwidgets' + 'png', + 'ggplot2', + 'optparse', + 'plyr', + 'plotly', + 'rmarkdown', + 'htmlwidgets' ) # Install packages not yet installed @@ -31,28 +32,77 @@ installed_packages <- packages %in% rownames(installed.packages()) dir.create(path = Sys.getenv("R_LIBS_USER"), showWarnings = FALSE, recursive = TRUE) if (any(installed_packages == FALSE)) { - install.packages(packages[!installed_packages], repos='http://cran.us.r-project.org', lib=Sys.getenv("R_LIBS_USER")) + install.packages(packages[!installed_packages], repos='http://cran.us.r-project.org', lib=Sys.getenv("R_LIBS_USER")) } # Packages loading -invisible(lapply(packages, library, character.only = TRUE)) +invisible(lapply(packages, library, warn.conflicts = FALSE, quietly = TRUE, character.only = TRUE)) + +if (pandoc_available()) { + self_contained = TRUE +} else { + self_contained = FALSE +} option_list = list( make_option( - c('-f', '--file'), - type = 'character', - default = NULL, + c('-f', '--file'), + type = 'character', + default = NULL, help = 'DXT CSV file name', - metavar = 'character' + metavar = 'file' + ), + make_option( + c('-s', '--start'), + type = 'numeric', + default = NULL, + help = 'Mark trace start time', + metavar = 'start' + ), + make_option( + c('-e', '--end'), + type = 'numeric', + default = NULL, + help = 'Mark trace end time', + metavar = 'end' + ), + make_option( + c('-n', '--from'), + type = 'numeric', + default = NULL, + help = 'Display trace from rank N', + metavar = 'from' + ), + make_option( + c('-m', '--to'), + type = 'numeric', + default = NULL, + help = 'Display trace up to rank M', + metavar = 'to' + ), + make_option( + c('-o', '--output'), + type = 'character', + default = NULL, + help = 'Name of the output file', + metavar = 'output' + ), + make_option( + c('-u', '--html'), + type = 'logical', + default = TRUE, + help = 'Generate a self-contained HTML file (requires pandoc)', + metavar = 'html' + ), + make_option( + c('-x', '--identifier'), + type = 'character', + default = TRUE, + help = 'Set the identifier of the original file captured by Darshan DXT', + metavar = 'identifier' ) ) -if (pandoc_available()) { - self_contained = TRUE -} else { - self_contained = FALSE -} - opt_parser = OptionParser(option_list=option_list) opt = parse_args(opt_parser) @@ -60,147 +110,510 @@ df <- read.csv(file=opt$file, sep = ',') df$duration = df$end - df$start +duration = max(df$end) - min(df$start) + +minimum = 0 +maximum = max(df$end) + +maximum_rank = max(df$rank) + +minimum_limit = -0.05 +maximum_limit = max(df$end) + (duration * 0.05) + +if (!is.null(opt$start)) { + df <- df[df$start >= opt$start, ] +} + +if (!is.null(opt$end)) { + df <- df[df$end <= opt$end, ] +} + +if (!is.null(opt$from)) { + df <- df[df$rank >= opt$from, ] +} + +if (!is.null(opt$to)) { + df <- df[df$rank <= opt$to, ] +} + +if (nrow(df) == 0) { + quit() +} + df$label = paste0( - 'Rank: ', df$rank, '\n', - 'Operation: ', df$operation, '\n', - 'Duration: ', round(df$duration, digits = 3), ' seconds\n', - 'Size: ', (df$size / 1024), ' KB\n', - 'Offset: ', df$offset + 'Rank: ', df$rank, '\n', + 'Operation: ', df$operation, '\n', + 'Duration: ', round(df$duration, digits = 3), ' seconds\n', + 'Size: ', (df$size / 1024), ' KB\n', + 'Offset: ', df$offset, '\n', + 'Lustre OST: ', ifelse(is.na(df$ost), '-', df$ost) ) # Include a zero record to ensure we can facet the plot df <- rbind(df, - data.frame( - file_id = c(0, 0, 0, 0), - api = c('POSIX', 'POSIX', 'MPIIO', 'MPIIO'), - rank = c(0, 0, 0, 0), - operation = c('write', 'read', 'write', 'read'), - segment = c(0, 0, 0, 0), - offset = c(0, 0, 0, 0), - size = c(0, 0, 0, 0), - start = c(0, 0, 0, 0), - end = c(0, 0, 0, 0), - duration = c(0, 0), - label = c('', '') - ) + data.frame( + file_id = c(0, 0, 0, 0), + api = c('POSIX', 'POSIX', 'MPIIO', 'MPIIO'), + rank = c(0, 0, 0, 0), + operation = c('write', 'read', 'write', 'read'), + segment = c(0, 0, 0, 0), + offset = c(0, 0, 0, 0), + size = c(0, 0, 0, 0), + start = c(0, 0, 0, 0), + end = c(0, 0, 0, 0), + duration = c(0, 0, 0, 0), + label = c('', '', '', ''), + ost = c(NA, NA, NA, NA) + ) ) df$operation <- as.factor(df$operation) -maximum = max(df$end) + (max(df$end) * 0.01) - plot_posix <- ggplot( - df[df$api == 'POSIX', ], - aes( - x = start, - xend = end, - y = rank, - yend = rank, - color = operation, - text = label - )) + - geom_segment() + - scale_color_manual( - "", - values = c( - "#f0746e", - "#3c93c2" - ), - drop = FALSE - ) + - scale_x_continuous(breaks = seq(0, maximum, length.out = 10)) + - facet_grid(api ~ .) + - expand_limits(x = 0) + - ylim(0, max(df$rank)) + - xlab('Time') + - ylab('Rank #') + - theme_bw() + - theme( - legend.position = "top", - plot.title = element_text(size = 10), - strip.background = element_rect(colour = NA, fill = NA) - ) + df[df$api == 'POSIX', ], + aes( + xmin = start, + xmax = end, + y = rank, + color = operation, + text = label + )) + + geom_errorbarh(height=0) + +# geom_segment() + + scale_color_manual( + "", + values = c( + "#f0746e", + "#3c93c2" + ), + drop = FALSE + ) + + scale_x_continuous(limits = c(minimum_limit, maximum_limit), breaks = seq(minimum_limit, maximum_limit, length.out = 10)) + + facet_grid(api ~ ., scales="free_x") + + ylim(0, maximum_rank) + +if (!is.null(opt$start)) { + plot_posix <- plot_posix + + geom_vline( + xintercept = opt$start, + linetype = "longdash" + ) + + geom_vline( + xintercept = minimum_limit - (duration * 0.05), + alpha = 0 + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = opt$start, + ymin = 0, + ymax = maximum_rank + ) +} + + plot_posix <- plot_posix + + geom_vline( + xintercept = minimum + ) + + geom_vline( + xintercept = minimum_limit, + alpha = 0 + ) + +if (!is.null(opt$end)) { + plot_posix <- plot_posix + + geom_vline( + xintercept = opt$end, + linetype = "longdash" + ) + + geom_vline( + xintercept = maximum_limit + (duration * 0.05), + alpha = 0 + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = opt$end, + xmax = maximum, + ymin = 0, + ymax = maximum_rank + ) +} + + plot_posix <- plot_posix + + geom_vline( + xintercept = maximum + ) + + geom_vline( + xintercept = maximum_limit, + alpha = 0 + ) + +if (!is.null(opt$from)) { + plot_posix <- plot_posix + + geom_hline( + yintercept = opt$from, + linetype = "longdash" + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = maximum, + ymin = 0, + ymax = opt$from + ) +} + +if (!is.null(opt$to)) { + plot_posix <- plot_posix + + geom_hline( + yintercept = opt$to, + linetype = "longdash" + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = maximum, + ymin = opt$to, + ymax = maximum_rank + ) +} + +plot_posix <- plot_posix + + xlab('Time') + + ylab('Rank #') + + theme_bw() + + theme( + legend.position = "top", + plot.title = element_text(size = 10), + strip.background = element_rect(colour = NA, fill = NA) + ) plot_mpiio <- ggplot( - df[df$api == 'MPIIO', ], - aes( - x = start, - xend = end, - y = rank, - yend = rank, - color = operation, - text = label - )) + - geom_segment() + - scale_color_manual( - "", - values = c( - "#f0746e", - "#3c93c2" - ), - drop = FALSE - ) + - scale_x_continuous(breaks = seq(0, maximum, length.out = 10)) + - facet_grid(api ~ .) + - expand_limits(x = 0) + - ylim(0, max(df$rank)) + - xlab('Time') + - ylab('Rank #') + - theme_bw() + - theme( - legend.position = "top", - plot.title = element_text(size = 10), - strip.background = element_rect(colour = NA, fill = NA) - ) + df[df$api == 'MPIIO', ], + aes( + xmin = start, + xmax = end, + y = rank, + color = operation, + text = label + )) + + geom_errorbarh(height=0) + +# geom_segment() + + scale_color_manual( + "", + values = c( + "#f0746e", + "#3c93c2" + ), + drop = FALSE + ) + + scale_x_continuous(limits = c(minimum_limit, maximum_limit), breaks = seq(minimum_limit, maximum_limit, length.out = 10)) + + facet_grid(api ~ ., scales="free_x") + + ylim(0, maximum_rank) + +if (!is.null(opt$start)) { + plot_mpiio <- plot_mpiio + + geom_vline( + xintercept = opt$start, + linetype = "longdash" + ) + + geom_vline( + xintercept = minimum_limit - (duration * 0.05), + alpha = 0 + )+ + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = opt$start, + ymin = 0, + ymax = maximum_rank + ) +} + plot_mpiio <- plot_mpiio + + geom_vline( + xintercept = minimum + ) + + geom_vline( + xintercept = minimum_limit, + alpha = 0 + ) + +if (!is.null(opt$end)) { + plot_mpiio <- plot_mpiio + + geom_vline( + xintercept = opt$end, + linetype = "longdash" + ) + + geom_vline( + xintercept = maximum_limit + (duration * 0.05), + alpha = 0 + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = opt$end, + xmax = maximum, + ymin = 0, + ymax = maximum_rank + ) +} + plot_mpiio <- plot_mpiio + + geom_vline( + xintercept = maximum + ) + + geom_vline( + xintercept = maximum_limit, + alpha = 0 + ) + +if (!is.null(opt$from)) { + plot_mpiio <- plot_mpiio + + geom_hline( + yintercept = opt$from, + linetype = "longdash" + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = maximum, + ymin = 0, + ymax = opt$from + ) +} + +if (!is.null(opt$to)) { + plot_mpiio <- plot_mpiio + + geom_hline( + yintercept = opt$to, + linetype = "longdash" + ) + + annotate( + "rect", + fill = "gray", + alpha = 0.5, + xmin = minimum, + xmax = maximum, + ymin = opt$to, + ymax = maximum_rank + ) +} + +plot_mpiio <- plot_mpiio + + xlab('Time') + + ylab('Rank #') + + theme_bw() + + theme( + legend.position = "top", + plot.title = element_text(size = 10), + strip.background = element_rect(colour = NA, fill = NA) + ) p_posix <- ggplotly( - plot_posix, - width = 1800, - height = 1000, - tooltip = "text", - legendgroup = operation, - dynamicTicks = TRUE - ) %>% - rangeslider(min(df$start), max(df$end), thickness = 0.03) %>% - layout( - margin = list(pad = 0), - legend = list(orientation = "h", x = 0, y = length(df$ranks) + 6), - autosize = TRUE, - xaxis = list(title = 'Runtime (seconds)', matches = 'x'), - yaxis = list(title = 'Rank', matches = 'y', fixedrange = FALSE), - hoverlabel = list(font = list(color = 'white')), - title = 'DXT Explorer Operation' - ) %>% - style( - showlegend = FALSE - ) %>% - toWebGL() + plot_posix, + width = 1800, + height = 1000, + tooltip = "text", + dynamicTicks = TRUE + ) + +if (!is.null(opt$start)) { + p_posix <- p_posix %>% add_annotations( + x = opt$start, + y = maximum_rank / 2, + ax = -20, + ay = 0, + text = "TIMELINE IS TRUNCATED", + textangle = -90, + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$end)) { + p_posix <- p_posix %>% add_annotations( + x = opt$end, + y = maximum_rank / 2, + ax = 20, + ay = 0, + text = "TIMELINE IS TRUNCATED", + textangle = 90, + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$from)) { + p_posix <- p_posix %>% add_annotations( + x = maximum / 2, + y = opt$from, + ax = 0, + ay = 20, + text = "RANK BEHAVIOR IS TRUNCATED", + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$to)) { + p_posix <- p_posix %>% add_annotations( + x = maximum / 2, + y = opt$to, + ax = 0, + ay = -20, + text = "RANK BEHAVIOR IS TRUNCATED", + font = list( + color = '#454545', + size = 10 + ) + ) +} + +p_posix <- p_posix %>% + rangeslider(minimum, maximum, thickness = 0.03) %>% + layout( + margin = list(pad = 0), + legend = list(orientation = "h", x = 1, y = length(df$ranks) + 6), + autosize = TRUE, + xaxis = list(title = 'Runtime (seconds)', matches = 'x'), + yaxis = list(title = 'Rank', matches = 'y', fixedrange = FALSE), + hoverlabel = list(font = list(color = 'white')), + title = paste0( + 'Explore Operation', + '
', + '', + opt$identifier, + '' + ) + ) %>% + style( + showlegend = FALSE + ) %>% + toWebGL() p_mpiio <- ggplotly( - plot_mpiio, - width = 1800, - height = 1000, - tooltip = "text", - legendgroup = operation, - dynamicTicks = TRUE - ) %>% - layout( - margin = list(pad = 0), - legend = list(orientation = "h", x = 0, y = length(df$ranks) + 6), - autosize = TRUE, - xaxis = list(matches = 'x'), - yaxis = list(title = 'Rank', matches = 'y', fixedrange = FALSE), - hoverlabel = list(font = list(color = 'white')) - ) %>% - toWebGL() + plot_mpiio, + width = 1800, + height = 1000, + tooltip = "text", + legendgroup = operation, + dynamicTicks = TRUE + ) + +if (!is.null(opt$start)) { + p_mpiio <- p_mpiio %>% add_annotations( + x = opt$start, + y = maximum_rank / 2, + ax = -20, + ay = 0, + text = "TIMELINE IS TRUNCATED", + textangle = -90, + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$end)) { + p_mpiio <- p_mpiio %>% add_annotations( + x = opt$end, + y = maximum_rank / 2, + ax = 20, + ay = 0, + text = "TIMELINE IS TRUNCATED", + textangle = 90, + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$from)) { + p_mpiio <- p_mpiio %>% add_annotations( + x = maximum / 2, + y = opt$from, + ax = 0, + ay = 20, + text = "RANK BEHAVIOR IS TRUNCATED", + font = list( + color = '#454545', + size = 10 + ) + ) +} + +if (!is.null(opt$to)) { + p_mpiio <- p_mpiio %>% add_annotations( + x = maximum / 2, + y = opt$to, + ax = 0, + ay = -20, + text = "RANK BEHAVIOR IS TRUNCATED", + font = list( + color = '#454545', + size = 10 + ) + ) +} + +p_mpiio <- p_mpiio %>% + layout( + margin = list(pad = 0), + # legend = list(orientation = "h", x = 0, y = length(df$ranks) + 6), + autosize = TRUE, + xaxis = list(matches = 'x'), + yaxis = list(title = 'Rank', matches = 'y', fixedrange = FALSE), + hoverlabel = list(font = list(color = 'white')) + ) %>% + style( + showlegend = FALSE + ) %>% + toWebGL() p <- subplot( - p_mpiio, p_posix, - nrows = 2, - titleY = TRUE, - titleX = TRUE, - shareX = TRUE, - shareY = TRUE + p_mpiio, p_posix, + nrows = 2, + titleY = TRUE, + titleX = TRUE, + shareX = TRUE, + shareY = TRUE +) %>% +layout ( + margin = list(t = 130), + images = list( + source = raster2uri(as.raster(readPNG('dxt-explorer.png'))), + x = 0, + y = 1.02, + sizex = 0.2, + sizey = 0.2, + xref = 'paper', + yref = 'paper', + xanchor = 'middle', + yanchor = 'bottom' + ) +) %>% +config( + displaylogo = FALSE ) -saveWidget(p, selfcontained = self_contained, 'explore.html') +saveWidget(p, selfcontained = self_contained, opt$output) diff --git a/plots/spatiality.R b/plots/spatiality.R index c776d4f..b4dcec5 100755 --- a/plots/spatiality.R +++ b/plots/spatiality.R @@ -21,6 +21,7 @@ packages <- c( 'optparse', 'plyr', 'plotly', + 'rmarkdown', 'htmlwidgets', 'rmarkdown', 'wesanderson' diff --git a/requirements.txt b/requirements.txt index d78ae50..67f6f40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ argparse -pandas -plotnine +pandas \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6f08915 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +import setuptools + +with open("README.md", "r") as f: + long_description = f.read() + +with open("requirements.txt") as f: + requirements = f.readlines() + +setuptools.setup( + name="dxt-explorer", + keywords="dxt-explorer", + version="1.2", + author="Jean Luca Bez, Suren Byna", + author_email="jlbez@lbl.gov, sbyna@lbl.gov", + description="DXT Explorer is an interactive web-based log analysis tool to visualize Darshan DXT logs and help understand the I/O behavior of applications.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/hpc-io/dxt-explorer", + install_requires=requirements, + packages=setuptools.find_packages(), + scripts=[ + "dxt-explorer" + ], + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: Other/Proprietary License", + "Programming Language :: Python :: 3 :: Only" + ], + python_requires='>=3.6', +) \ No newline at end of file