diff --git a/lib/yaml_config/elements.py b/lib/yaml_config/elements.py index e928cd9d0..23b31e00a 100644 --- a/lib/yaml_config/elements.py +++ b/lib/yaml_config/elements.py @@ -94,7 +94,7 @@ class ConfigElement: _type_name = None # The regular expression that all element names are matched against. - _NAME_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]+$') + _NAME_RE = re.compile(r'^[a-zA-Z*?\[\]-][a-zA-Z0-9_*?\[\]-]+$') # We use the representer functions in this to consistently represent # certain types diff --git a/lib/yaml_config/structures.py b/lib/yaml_config/structures.py index 732128896..b3746764c 100644 --- a/lib/yaml_config/structures.py +++ b/lib/yaml_config/structures.py @@ -1,4 +1,5 @@ from collections import defaultdict, OrderedDict +import fnmatch import io import similarity @@ -304,13 +305,65 @@ def _merge_extend(self, key, sub_elem, old_value, value): class KeyedElem(_DictElem): """A dictionary configuration item with predefined keys that may have non-uniform types. The valid keys are are given as ConfigItem objects, - with their names being used as the key name.""" + with their names being used as the key name. Keys can contain wildcards, + as per globbing syntax (see the fnmatch library). Keys are checked against + exact matches first, then are checked against wildcard keys in the order + those elements were originally given.""" + + class _FuzzyOrderedDict(OrderedDict): + + def __getitem__(self, key): + """Get an item, but do fnmatch'ing on the given key vs the existing keys.""" + + if key in self.keys(): + return super().__getitem__(key) + + for pattern in self.keys(): + if fnmatch.fnmatchcase(key, pattern): + return super().__getitem__(pattern) + + raise KeyError(key) + + def get(self, key, default=None): + """Use fuzzy mathing with the get() method too.""" + + if key in self.keys(): + return super().__getitem__(key) + + for pattern in self.keys(): + if fnmatch.fnmatchcase(key, pattern): + return super().__getitem__(pattern) + + return default + + def __contains__(self, key): + """Return true if the key is in the dict (or matches a fuzzy key).""" + + if key in self.keys(): + return True + + for pattern in self.keys(): + if fnmatch.fnmatchcase(key, pattern): + return True + + return False + + def get_key(self, key): + """Return the key pattern that would match the given key.""" + + if key in self.keys(): + return key + + for pattern in self.keys(): + if fnmatch.fnmatchcase(key, pattern): + return pattern + + return None type = ConfigDict _type_name = '' - def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED, - **kwargs): + def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED, **kwargs): """ :param key_case: Must be one of the .KC_* values. Determines whether keys are automatically converted to lower or upper case, @@ -319,7 +372,7 @@ def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED, of accepted keys, with the element.name being the key. """ - self.config_elems = OrderedDict() + self.config_elems = self._FuzzyOrderedDict() if elements is None: elements = [] @@ -513,6 +566,8 @@ def normalize(self, value, root_name=None): elem = self.config_elems.get(final_key, None) if elem is None: + print(list(self.config_elems.keys())) + print(elem, final_key) msg = self._make_missing_key_message(root_name, key, value) raise KeyError(msg, key) @@ -632,6 +687,11 @@ def yaml_events(self, value, if value is None: value = dict() + resolved_vals = defaultdict(list) + for vkey, val in value.items(): + pattern = self.config_elems.get_key(vkey) + resolved_vals[pattern].append((vkey, val)) + events = list() events.append(yaml.MappingStartEvent(anchor=None, tag=None, implicit=True)) @@ -643,18 +703,19 @@ def yaml_events(self, value, if isinstance(elem, DerivedElem): continue - val = value.get(key, None) - if show_comments: - comment = elem.make_comment(show_choices=show_choices) - events.append(yaml.CommentEvent(value=comment)) - - # Add the mapping key - events.append(yaml.ScalarEvent(value=key, anchor=None, - tag=None, implicit=(True, True))) - # Add the mapping value - events.extend(elem.yaml_events(val, - show_comments, - show_choices)) + items = resolved_vals.get(key, [(key, None)]) + for vkey, val in items: + if show_comments: + comment = elem.make_comment(show_choices=show_choices) + events.append(yaml.CommentEvent(value=comment)) + + # Add the mapping key + events.append(yaml.ScalarEvent(value=vkey, anchor=None, + tag=None, implicit=(True, True))) + # Add the mapping value + events.extend(elem.yaml_events(val, + show_comments, + show_choices)) events.append(yaml.MappingEndEvent()) return events @@ -1007,3 +1068,97 @@ def yaml_events(self, value, show_comments, show_choices): def set_default(self, dotted_key, value): raise RuntimeError("You can't set defaults on derived elements.") + + +class AnyElem(ConfigElement): + """A generic, unchecked config element that can contain arbitrary YAML.""" + + class NoType: + """Never match this type in the normalize method.""" + + def str_converter(self, value): + """Convert an arbitrary structure of lists/dicts into the same structure with + all leaf elements converted into strings.""" + + if isinstance(value, dict): + norm_dict = {} + for key, subval in value.items(): + norm_dict[str(key)] = self.str_converter(subval) + return norm_dict + elif isinstance(value, (list, tuple)): + return [self.str_converter(item) for item in value] + else: + return str(value) + + type = NoType + type_converter = str_converter + _type_name = 'Arbitrary-YAML' + + def yaml_events(self, value, show_comments, show_choices): + """Convert value to a list of yaml events.""" + + events = list() + if isinstance(value, (list, tuple)): + events.append(yaml.SequenceStartEvent( + anchor=None, + tag=None, + implicit=True, + flow_style=False, + )) + + if show_comments: + comment = self._sub_elem.make_comment( + show_choices=show_choices, + recursive=True, + ) + events.append(yaml.CommentEvent(value=comment)) + + if not value: + value = [] + + # Value is expected to be a list of items at this point. + for val in value: + events.extend( + self.yaml_events( + value=val, + show_comments=False, + show_choices=False)) + + events.append(yaml.SequenceEndEvent()) + + elif isinstance(value, dict): + if value is None: + value = dict() + + events.append(yaml.MappingStartEvent(anchor=None, tag=None, + implicit=True)) + for key, elem in self.config_elems.items(): + if elem.hidden: + continue + + # Don't output anything for Derived Elements + if isinstance(elem, DerivedElem): + continue + + val = value.get(key, None) + if show_comments: + comment = elem.make_comment(show_choices=show_choices) + events.append(yaml.CommentEvent(value=comment)) + + # Add the mapping key + events.append(yaml.ScalarEvent(value=key, anchor=None, + tag=None, implicit=(True, True))) + # Add the mapping value + events.extend(self.yaml_events(val, + show_comments=False, + show_choices=False)) + events.append(yaml.MappingEndEvent()) + else: + tag = None + if value is not None: + value, tag = self._represent(value) + + events.append(yaml.ScalarEvent(value=value, anchor=None, tag=tag, + implicit=(True, True))) + + return events diff --git a/lib/yaml_config/testlib.py b/lib/yaml_config/testlib.py index ae8eae474..18e1900b5 100644 --- a/lib/yaml_config/testlib.py +++ b/lib/yaml_config/testlib.py @@ -37,6 +37,8 @@ class Config(yaml_config.YamlConfigLoader): "properties", help_text="Pet properties", elements=[ yaml_config.scalars.StrElem( "description", help_text="General pet description."), + yaml_config.scalars.StrElem( + 'fuzzy_*', help_text="I accept all sorts of keys."), yaml_config.scalars.RegexElem( "greeting", regex=r'hello \w+$', help_text="A regex of some sort."), diff --git a/test/tests/yaml_config_tests/data/test1.yaml b/test/tests/yaml_config_tests/data/test1.yaml index 6d15b2ae8..278ebfea2 100644 --- a/test/tests/yaml_config_tests/data/test1.yaml +++ b/test/tests/yaml_config_tests/data/test1.yaml @@ -21,6 +21,8 @@ properties: # LEGS(required int) # Valid Range: > 0 legs: 4 + # Accept wildcard keys + fuzzy_wuzzy: "is here" traits: int: 'low' agility: 'terrible' diff --git a/test/tests/yaml_config_tests/format_tests.py b/test/tests/yaml_config_tests/format_tests.py index a32718732..eb3954da6 100644 --- a/test/tests/yaml_config_tests/format_tests.py +++ b/test/tests/yaml_config_tests/format_tests.py @@ -51,6 +51,7 @@ class TestConfig(yc.YamlConfigLoader): elements=[ yaml_config.scalars.StrElem("description", help_text="General pet description."), + yaml_config.scalars.StrElem("fuzzy_*", help_text="a wildcard key"), yaml_config.scalars.RegexElem("greeting", regex=r'hello \w+$', help_text="A regex of some sort."), yaml_config.scalars.IntRangeElem("legs", vmin=0)