From 38f842ea7218fa753b2796a55b6d965c1186dcf0 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Fri, 26 May 2023 18:12:26 -0600 Subject: [PATCH 1/4] Added format key to results_parse yc. On the same level as 'action'. This avoids printing results with an absurd number of digits to the terminal. And facilitates displaying results as the user desires. --- lib/pavilion/result/parse.py | 30 ++++++++++++++++++++- lib/pavilion/result_parsers/base_classes.py | 7 ++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/result/parse.py b/lib/pavilion/result/parse.py index ead75322b..cb6f3746c 100644 --- a/lib/pavilion/result/parse.py +++ b/lib/pavilion/result/parse.py @@ -70,6 +70,31 @@ def __init__(self, parser_name: str, key: str, config: dict): ProcessFileArgs = NewType('ProcessFileArgs', Tuple[Path, List[KeySet]]) +def format_results(result_val, format_spec): + """Format the result value according to the format spec. + :param result_val: The value to format. + :param format_spec: The format spec. + :return: The formatted value. + """ + + #Check if result_val is a string or bool. + if isinstance(result_val, (str, bool)): + return result_val + + #Check if result_val is numeric. + if isinstance(result_val, (int, float)): + return format_spec.format(result_val) + + if isinstance(result_val, (list, set)): + formatted_result=[] + for res_v in result_val: + if isinstance(res_v, (int, float)): + formatted_result.append(format_spec.format(res_v)) + else: + formatted_result.append(res_v) + + return formatted_result + def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: """Parse the results of the given test using all the result parsers @@ -107,6 +132,8 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: per_file = {} # Action values by key actions = {} + # Format values by key + formats = {} # A list of encountered error messages. errors = [] @@ -120,6 +147,7 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: per_file[key] = rconf['per_file'] actions[key] = rconf['action'] + formats[key] = rconf['format'] for file_glob in rconf['files']: base_glob = file_glob @@ -190,7 +218,7 @@ def parse_results(pav_cfg, test, results: Dict, base_log: IndentedLog) -> None: for key, per_file_name in per_file.items(): per_file_func = PER_FILES[per_file_name] # type: per_first action_name = actions[key] - presults = ordered_filed_results[key] + presults = format_results(ordered_filed_results[key], formats[key]) try: log("Applying per-file option '{}' and action '{}' to key '{}'." diff --git a/lib/pavilion/result_parsers/base_classes.py b/lib/pavilion/result_parsers/base_classes.py index c125fa6b6..050ce90c5 100644 --- a/lib/pavilion/result_parsers/base_classes.py +++ b/lib/pavilion/result_parsers/base_classes.py @@ -211,7 +211,7 @@ def check_args(self, **kwargs) -> dict: args[key] = kwargs[key] kwargs = args - base_keys = ('action', 'per_file', 'files', 'match_select', + base_keys = ('action', 'per_file', 'files', 'format', 'match_select', 'for_lines_matching', 'preceded_by') for key in base_keys: @@ -313,6 +313,10 @@ def check_args(self, **kwargs) -> dict: help_text="Path to the file/s that this result parser " "will examine. Each may be a file glob," "such as '*.log'"), + yc.StrElem( + "format", + help_text="Python string format 'eg. {:.3f}' to store the result." + ), yc.StrElem( "per_file", help_text=( @@ -470,6 +474,7 @@ def add_arg_doc(arg): _DEFAULTS = { 'per_file': PER_FIRST, 'action': ACTION_STORE, + 'format': '{.3f}', 'files': ['../run.log'], 'match_select': MATCH_FIRST, 'for_lines_matching': '', From 37d1763ee897a0500bc916902d9ddb1534457981 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Fri, 26 May 2023 18:30:50 -0600 Subject: [PATCH 2/4] Formatting --- lib/pavilion/result/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/result/parse.py b/lib/pavilion/result/parse.py index cb6f3746c..684fd69fc 100644 --- a/lib/pavilion/result/parse.py +++ b/lib/pavilion/result/parse.py @@ -92,7 +92,7 @@ def format_results(result_val, format_spec): formatted_result.append(format_spec.format(res_v)) else: formatted_result.append(res_v) - + return formatted_result From e74482908f8b63e98f2c08d85ed11e673d2f401a Mon Sep 17 00:00:00 2001 From: Daniel J Magee <43071310+dmageeLANL@users.noreply.github.com> Date: Thu, 1 Jun 2023 13:12:30 -0600 Subject: [PATCH 3/4] Fix default format. --- lib/pavilion/result_parsers/base_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/result_parsers/base_classes.py b/lib/pavilion/result_parsers/base_classes.py index 050ce90c5..1bbd61143 100644 --- a/lib/pavilion/result_parsers/base_classes.py +++ b/lib/pavilion/result_parsers/base_classes.py @@ -474,7 +474,7 @@ def add_arg_doc(arg): _DEFAULTS = { 'per_file': PER_FIRST, 'action': ACTION_STORE, - 'format': '{.3f}', + 'format': '{:.3f}', 'files': ['../run.log'], 'match_select': MATCH_FIRST, 'for_lines_matching': '', From 1ee0f0bd4388a0c6bfbb37335be04c0753f95fcc Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Thu, 1 Jun 2023 13:58:00 -0600 Subject: [PATCH 4/4] Got rid of all trailing whitespace in lib/pavilion. I've included the script which does this in test/utils. It's a standalone executable which only acts on .py files and takes a space separated list of directories to recursively cleanse of trailing whitespace. If no argument is given it acts on the directory that contains the script file. Put it in your PATH and enjoy. --- lib/pavilion/parsers/expressions.py | 22 ++++++++--------- lib/pavilion/parsers/strings.py | 14 +++++------ lib/pavilion/resolver.py | 2 +- test/utils/trimwstrail | 38 +++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 19 deletions(-) create mode 100755 test/utils/trimwstrail diff --git a/lib/pavilion/parsers/expressions.py b/lib/pavilion/parsers/expressions.py index 37e69774a..12066e4bb 100644 --- a/lib/pavilion/parsers/expressions.py +++ b/lib/pavilion/parsers/expressions.py @@ -19,31 +19,31 @@ // All expressions will resolve to the start expression. start: expr _WS? - | // An empty string is valid - + | // An empty string is valid + // Trailing whitespace is ignored. Whitespace between tokens is // ignored below. _WS: /\s+/ expr: or_expr -// These set order of operations. +// These set order of operations. // See https://en.wikipedia.org/wiki/Operator-precedence_parser -or_expr: and_expr ( OR and_expr )* +or_expr: and_expr ( OR and_expr )* and_expr: not_expr ( AND not_expr )* -not_expr: NOT? compare_expr +not_expr: NOT? compare_expr compare_expr: add_expr ((EQ | NOT_EQ | LT | GT | LT_EQ | GT_EQ ) add_expr)* add_expr: mult_expr ((PLUS | MINUS) mult_expr)* mult_expr: pow_expr ((TIMES | DIVIDE | INT_DIV | MODULUS) pow_expr)* pow_expr: primary ("^" primary)? -primary: literal - | var_ref +primary: literal + | var_ref | negative | "(" expr ")" | function_call | list_ -// A function call can contain zero or more arguments. +// A function call can contain zero or more arguments. function_call: NAME "(" (expr ("," expr)*)? ")" negative: (MINUS|PLUS) primary @@ -53,7 +53,7 @@ | FLOAT | BOOL | ESCAPED_STRING - + // Allows for trailing commas list_: L_BRACKET (expr ("," expr)* ","?)? R_BRACKET @@ -92,11 +92,11 @@ // This will be prioritized over 'NAME' matches BOOL.2: "True" | "False" -// Names can be lower-case or capitalized, but must start with a letter or +// Names can be lower-case or capitalized, but must start with a letter or // underscore NAME.1: /[a-zA-Z_][a-zA-Z0-9_]*/ -// Ignore all whitespace between tokens. +// Ignore all whitespace between tokens. %ignore / +(?=[^.])/ ''' diff --git a/lib/pavilion/parsers/strings.py b/lib/pavilion/parsers/strings.py index 6325bda5f..3ba772857 100644 --- a/lib/pavilion/parsers/strings.py +++ b/lib/pavilion/parsers/strings.py @@ -15,13 +15,13 @@ from .expressions import get_expr_parser, ExprTransformer, VarRefVisitor STRING_GRAMMAR = r''' -// All strings resolve to this token. +// All strings resolve to this token. start: string TRAILING_NEWLINE? TRAILING_NEWLINE: /\n/ -// It's important that each of these start with a terminal, rather than -// a reference back to the 'string' rule. A 'STRING' terminal (or nothing) +// It's important that each of these start with a terminal, rather than +// a reference back to the 'string' rule. A 'STRING' terminal (or nothing) // is definite, but a 'string' would be non-deterministic. string: STRING? | STRING? iter string @@ -35,15 +35,15 @@ iter_inner: STRING? | STRING? expr iter_inner - + expr: _START_EXPR EXPR? (ESCAPED_STRING EXPR?)* FORMAT? _END_EXPR _START_EXPR: "{{" _END_EXPR: "}}" EXPR: /[^}~{":]+/ -// Match anything enclosed in quotes as long as the last +// Match anything enclosed in quotes as long as the last // escape doesn't escape the close quote. -// A minimal match, but the required close quote will force this to +// A minimal match, but the required close quote will force this to // consume most of the string. _STRING_ESC_INNER: /.*?/ // If the string ends in a backslash, it must end with an even number @@ -62,7 +62,7 @@ // we can't match the start of the string in the look-behind. // - Strings can contain anything, but they can't start with an open // expression '{{' or open iteration '[~'. -// - Strings cannot end in an odd number of backslashes (that would +// - Strings cannot end in an odd number of backslashes (that would // escape the closing characters). // - Strings must end with the end of string, an open expression '{{', // an open iteration '[~', or a tilde. diff --git a/lib/pavilion/resolver.py b/lib/pavilion/resolver.py index c82c2b951..b0670ad87 100644 --- a/lib/pavilion/resolver.py +++ b/lib/pavilion/resolver.py @@ -665,7 +665,7 @@ def _load_raw_config(self, name: str, config_type: str, optional=False) \ "Pavilion config directories.\n" "Run `pav show {2}` to get a list of available {0} files." .format(config_type, name, show_type)) - + try: with path.open() as cfg_file: # Load the host test config defaults. diff --git a/test/utils/trimwstrail b/test/utils/trimwstrail new file mode 100755 index 000000000..e0272a5c1 --- /dev/null +++ b/test/utils/trimwstrail @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import os +import sys + +def trim_py_files(directories): + """Remove trailing whitespace on all .py files in the given directories. + """ + nchanged = 0 + print(directories) + for directory in directories: + for root, dirs, files in os.walk(directory): + if ".git" in root: + continue + for fname in files: + filename = os.path.join(root, fname) + if fname.endswith('.py'): + with open(filename, 'rb') as f: + code1 = f.read().decode() + lines = [line.rstrip() for line in code1.splitlines()] + while lines and not lines[-1]: + lines.pop(-1) + lines.append('') # always end with a newline + code2 = '\n'.join(lines) + if code1 != code2: + nchanged += 1 + print(' Removing trailing whitespace on', filename) + with open(filename, 'wb') as f: + f.write(code2.encode()) + print('Removed trailing whitespace on {} files.'.format(nchanged)) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + trim_py_files(list(os.path.dirname(__file__))) + else: + trim_py_files(sys.argv[1:]) +