Adding AnyElem. It isn't used yet, but will be for future Spack addit…

lib/yaml_config/elements.py

@@ -94,7 +94,7 @@ class ConfigElement:
     _type_name = None
 
     # The regular expression that all element names are matched against.
-    _NAME_RE = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]+$')
+    _NAME_RE = re.compile(r'^[a-zA-Z*?\[\]-][a-zA-Z0-9_*?\[\]-]+$')
 
     # We use the representer functions in this to consistently represent
     # certain types

lib/yaml_config/structures.py

@@ -1,4 +1,5 @@
 from collections import defaultdict, OrderedDict
+import fnmatch
 import io
 
 import similarity
@@ -304,13 +305,65 @@ def _merge_extend(self, key, sub_elem, old_value, value):
 class KeyedElem(_DictElem):
     """A dictionary configuration item with predefined keys that may have
     non-uniform types. The valid keys are are given as ConfigItem objects,
-    with their names being used as the key name."""
+    with their names being used as the key name. Keys can contain wildcards,
+    as per globbing syntax (see the fnmatch library). Keys are checked against
+    exact matches first, then are checked against wildcard keys in the order 
+    those elements were originally given."""
+
+    class _FuzzyOrderedDict(OrderedDict):
+
+        def __getitem__(self, key):
+            """Get an item, but do fnmatch'ing on the given key vs the existing keys."""
+
+            if key in self.keys():
+                return super().__getitem__(key)
+
+            for pattern in self.keys():
+                if fnmatch.fnmatchcase(key, pattern):
+                    return super().__getitem__(pattern)
+
+            raise KeyError(key)
+
+        def get(self, key, default=None):
+            """Use fuzzy mathing with the get() method too."""
+
+            if key in self.keys():
+                return super().__getitem__(key)
+
+            for pattern in self.keys():
+                if fnmatch.fnmatchcase(key, pattern):
+                    return super().__getitem__(pattern)
+
+            return default
+
+        def __contains__(self, key):
+            """Return true if the key is in the dict (or matches a fuzzy key)."""
+
+            if key in self.keys():
+                return True
+
+            for pattern in self.keys():
+                if fnmatch.fnmatchcase(key, pattern):
+                    return True
+
+            return False
+
+        def get_key(self, key):
+            """Return the key pattern that would match the given key."""
+
+            if key in self.keys():
+                return key
+
+            for pattern in self.keys():
+                if fnmatch.fnmatchcase(key, pattern):
+                    return pattern
+
+            return None
 
     type = ConfigDict
     _type_name = ''
 
-    def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED,
-                 **kwargs):
+    def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED, **kwargs):
         """
         :param key_case: Must be one of the <cls>.KC_* values. Determines
             whether keys are automatically converted to lower or upper case,
@@ -319,7 +372,7 @@ def __init__(self, name=None, elements=None, key_case=_DictElem.KC_MIXED,
             of accepted keys, with the element.name being the key.
         """
 
-        self.config_elems = OrderedDict()
+        self.config_elems = self._FuzzyOrderedDict()
 
         if elements is None:
             elements = []
@@ -513,6 +566,8 @@ def normalize(self, value, root_name=None):
 
             elem = self.config_elems.get(final_key, None)
             if elem is None:
+                print(list(self.config_elems.keys()))
+                print(elem, final_key)
                 msg = self._make_missing_key_message(root_name, key, value)
                 raise KeyError(msg, key)
 
@@ -632,6 +687,11 @@ def yaml_events(self, value,
         if value is None:
             value = dict()
 
+        resolved_vals = defaultdict(list)
+        for vkey, val in value.items():
+            pattern = self.config_elems.get_key(vkey)
+            resolved_vals[pattern].append((vkey, val))
+
         events = list()
         events.append(yaml.MappingStartEvent(anchor=None, tag=None,
                                              implicit=True))
@@ -643,18 +703,19 @@ def yaml_events(self, value,
             if isinstance(elem, DerivedElem):
                 continue
 
-            val = value.get(key, None)
-            if show_comments:
-                comment = elem.make_comment(show_choices=show_choices)
-                events.append(yaml.CommentEvent(value=comment))
-
-            # Add the mapping key
-            events.append(yaml.ScalarEvent(value=key, anchor=None,
-                                           tag=None, implicit=(True, True)))
-            # Add the mapping value
-            events.extend(elem.yaml_events(val,
-                                           show_comments,
-                                           show_choices))
+            items = resolved_vals.get(key, [(key, None)])
+            for vkey, val in items:
+                if show_comments:
+                    comment = elem.make_comment(show_choices=show_choices)
+                    events.append(yaml.CommentEvent(value=comment))
+
+                # Add the mapping key
+                events.append(yaml.ScalarEvent(value=vkey, anchor=None,
+                                               tag=None, implicit=(True, True)))
+                # Add the mapping value
+                events.extend(elem.yaml_events(val,
+                                               show_comments,
+                                               show_choices))
         events.append(yaml.MappingEndEvent())
         return events
 
@@ -1007,3 +1068,97 @@ def yaml_events(self, value, show_comments, show_choices):
 
     def set_default(self, dotted_key, value):
         raise RuntimeError("You can't set defaults on derived elements.")
+
+
+class AnyElem(ConfigElement):
+    """A generic, unchecked config element that can contain arbitrary YAML."""
+
+    class NoType:
+        """Never match this type in the normalize method."""
+
+    def str_converter(self, value):
+        """Convert an arbitrary structure of lists/dicts into the same structure with
+        all leaf elements converted into strings."""
+
+        if isinstance(value, dict):
+            norm_dict = {}
+            for key, subval in value.items():
+                norm_dict[str(key)] = self.str_converter(subval)
+            return norm_dict
+        elif isinstance(value, (list, tuple)):
+            return [self.str_converter(item) for item in value]
+        else:
+            return str(value)
+
+    type = NoType
+    type_converter = str_converter
+    _type_name = 'Arbitrary-YAML'
+
+    def yaml_events(self, value, show_comments, show_choices):
+        """Convert value to a list of yaml events."""
+
+        events = list()
+        if isinstance(value, (list, tuple)):
+            events.append(yaml.SequenceStartEvent(
+                anchor=None,
+                tag=None,
+                implicit=True,
+                flow_style=False,
+            ))
+
+            if show_comments:
+                comment = self._sub_elem.make_comment(
+                    show_choices=show_choices,
+                    recursive=True,
+                )
+                events.append(yaml.CommentEvent(value=comment))
+
+            if not value:
+                value = []
+
+            # Value is expected to be a list of items at this point.
+            for val in value:
+                events.extend(
+                    self.yaml_events(
+                        value=val,
+                        show_comments=False,
+                        show_choices=False))
+
+            events.append(yaml.SequenceEndEvent())
+
+        elif isinstance(value, dict):
+            if value is None:
+                value = dict()
+
+            events.append(yaml.MappingStartEvent(anchor=None, tag=None,
+                                                 implicit=True))
+            for key, elem in self.config_elems.items():
+                if elem.hidden:
+                    continue
+
+                # Don't output anything for Derived Elements
+                if isinstance(elem, DerivedElem):
+                    continue
+
+                val = value.get(key, None)
+                if show_comments:
+                    comment = elem.make_comment(show_choices=show_choices)
+                    events.append(yaml.CommentEvent(value=comment))
+
+                # Add the mapping key
+                events.append(yaml.ScalarEvent(value=key, anchor=None,
+                                               tag=None, implicit=(True, True)))
+                # Add the mapping value
+                events.extend(self.yaml_events(val,
+                                               show_comments=False,
+                                               show_choices=False))
+            events.append(yaml.MappingEndEvent())
+        else:
+            tag = None
+            if value is not None:
+                value, tag = self._represent(value)
+
+            events.append(yaml.ScalarEvent(value=value, anchor=None, tag=tag,
+                                           implicit=(True, True)))
+
+        return events

lib/yaml_config/testlib.py

@@ -37,6 +37,8 @@ class Config(yaml_config.YamlConfigLoader):
                 "properties", help_text="Pet properties", elements=[
                 yaml_config.scalars.StrElem(
                     "description", help_text="General pet description."),
+                yaml_config.scalars.StrElem(
+                    'fuzzy_*', help_text="I accept all sorts of keys."),
                 yaml_config.scalars.RegexElem(
                     "greeting", regex=r'hello \w+$',
                     help_text="A regex of some sort."),

test/tests/yaml_config_tests/data/test1.yaml

@@ -21,6 +21,8 @@ properties:
   # LEGS(required int)
   # Valid Range: > 0
   legs: 4
+  # Accept wildcard keys
+  fuzzy_wuzzy: "is here"
 traits:
     int: 'low'
     agility: 'terrible'

test/tests/yaml_config_tests/format_tests.py

@@ -51,6 +51,7 @@ class TestConfig(yc.YamlConfigLoader):
                                                  elements=[
                                  yaml_config.scalars.StrElem("description",
                                                              help_text="General pet description."),
+                                 yaml_config.scalars.StrElem("fuzzy_*", help_text="a wildcard key"),
                                  yaml_config.scalars.RegexElem("greeting", regex=r'hello \w+$',
                                                                help_text="A regex of some sort."),
                                  yaml_config.scalars.IntRangeElem("legs", vmin=0)