Putting utils.py out of its misery.

bitformat/_bits.py

@@ -5,15 +5,12 @@
 import sys
 import struct
 import io
-import re
 import functools
 from ast import literal_eval
 from collections import abc
 from typing import Union, Iterable, Any, TextIO, overload, Iterator, Type, Sequence
-from bitformat import _utils
 from bitformat._dtypes import Dtype, Register, DtypeTuple
-from bitformat._common import Colour, Endianness
-from typing import Pattern
+from bitformat._common import Colour
 from bitformat._options import Options
 from bitformat.bit_rust import BitRust
 
@@ -22,38 +19,31 @@
 # Things that can be converted to Bits when a Bits type is needed
 BitsType = Union["Bits", str, bytearray, bytes, memoryview]
 
-# name[length]=value
-NAME_INT_VALUE_RE: Pattern[str] = re.compile(
-    r"^([a-zA-Z][a-zA-Z0-9_]*?)(\d*)(?:=(.*))$"
-)
-
 # The size of various caches used to improve performance
 CACHE_SIZE = 256
 
 
 @functools.lru_cache(CACHE_SIZE)
 def token_to_bitstore(token: str) -> BitRust:
     if token[0] != "0":
-        match = NAME_INT_VALUE_RE.match(token)
-        if not match:
+        dtype_str, value_str = token.split("=", 1)
+        try:
+            dtype = Dtype.from_string(dtype_str)
+        except ValueError:
             raise ValueError(
                 f"Can't parse token '{token}'. It should be in the form 'name[length]=value' (e.g. "
                 "'u8 = 44') or a literal starting with '0b', '0o' or '0x'."
             )
-        name, length_str, value = match.groups()
-        name, modifier = _utils.parse_name_to_name_and_modifier(name)
-
-        length = int(length_str) if length_str else 0
-        dtype = Dtype.from_params(name, length, endianness=Endianness(modifier))
-        value_str = value
         if dtype.return_type in (bool, bytes):  # TODO: Is this right? Needs more tests.
             try:
-                value = literal_eval(value)
+                value = literal_eval(value_str)
             except ValueError:
                 raise ValueError(
                     f"Can't parse token '{token}'. The value '{value_str}' can't be converted to the appropriate type."
                 )
-        return dtype.pack(value)._bitstore
+            return dtype.pack(value)._bitstore
+        else:
+            return dtype.pack(value_str)._bitstore
     if token.startswith("0x"):
         return BitRust.from_hex_checked(token)
     if token.startswith("0b"):

bitformat/_dtypes.py

@@ -5,7 +5,6 @@
 import inspect
 import bitformat
 import re
-from bitformat import _utils
 from ._common import Expression, Endianness, byteorder
 from typing import Pattern
 
@@ -35,6 +34,14 @@ def parse_name_expression_token(fmt: str) -> tuple[str, str]:
     name, expression = match.groups()
     return name, expression
 
+def parse_name_to_name_and_modifier(name: str) -> tuple[str, str]:
+    modifiers = name.split("_")
+    if len(modifiers) == 1:
+        return name, ""
+    if len(modifiers) == 2:
+        return modifiers[0], modifiers[1]
+    raise ValueError(f"Can't parse Dtype name '{name}' as more than one '_' is present.")
+
 
 class Dtype:
     """A data type class, representing a concrete interpretation of binary data.
@@ -107,7 +114,7 @@ def from_string(cls, token: str, /) -> Dtype:
             t = token[p + 1 : -1]
             items = int(t) if t else 0
             name, size = parse_name_size_token(token[1:p])
-            name, modifier = _utils.parse_name_to_name_and_modifier(name)
+            name, modifier = parse_name_to_name_and_modifier(name)
             endianness = Endianness(modifier)
             return Register().get_array_dtype(name, size, items, endianness)
         else:
@@ -120,7 +127,7 @@ def from_string(cls, token: str, /) -> Dtype:
                     )
                 else:
                     raise e
-            name, modifier = _utils.parse_name_to_name_and_modifier(name)
+            name, modifier = parse_name_to_name_and_modifier(name)
             endianness = Endianness(modifier)
             return Register().get_single_dtype(name, size, endianness)
 
@@ -686,7 +693,7 @@ def from_string(cls, token: str, /) -> DtypeWithExpression:
             except ValueError:
                 x.size_expression = Expression(size_str)
                 size = 0
-            name, modifier = _utils.parse_name_to_name_and_modifier(name)
+            name, modifier = parse_name_to_name_and_modifier(name)
             endianness = Endianness(modifier)
             x.base_dtype = Register().get_array_dtype(name, size, items, endianness)
             return x
@@ -698,7 +705,7 @@ def from_string(cls, token: str, /) -> DtypeWithExpression:
             except ValueError:
                 x.size_expression = Expression(size_str)
                 size = 0
-            name, modifier = _utils.parse_name_to_name_and_modifier(name)
+            name, modifier = parse_name_to_name_and_modifier(name)
             endianness = Endianness(modifier)
             x.base_dtype = Register().get_single_dtype(name, size, endianness)
             return x