Allow enabling individual experimental features (#13790)

Ref #13685

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>

Author: ilevkivskyi

Diff for: mypy/config_parser.py

@@ -154,6 +154,7 @@ def check_follow_imports(choice: str) -> str:
     "plugins": lambda s: [p.strip() for p in s.split(",")],
     "always_true": lambda s: [p.strip() for p in s.split(",")],
     "always_false": lambda s: [p.strip() for p in s.split(",")],
+    "enable_incomplete_feature": lambda s: [p.strip() for p in s.split(",")],
     "disable_error_code": lambda s: validate_codes([p.strip() for p in s.split(",")]),
     "enable_error_code": lambda s: validate_codes([p.strip() for p in s.split(",")]),
     "package_root": lambda s: [p.strip() for p in s.split(",")],
@@ -176,6 +177,7 @@ def check_follow_imports(choice: str) -> str:
         "plugins": try_split,
         "always_true": try_split,
         "always_false": try_split,
+        "enable_incomplete_feature": try_split,
         "disable_error_code": lambda s: validate_codes(try_split(s)),
         "enable_error_code": lambda s: validate_codes(try_split(s)),
         "package_root": try_split,

Diff for: mypy/main.py

@@ -18,7 +18,7 @@
 from mypy.find_sources import InvalidSourceList, create_source_list
 from mypy.fscache import FileSystemCache
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path
-from mypy.options import BuildType, Options
+from mypy.options import INCOMPLETE_FEATURES, BuildType, Options
 from mypy.split_namespace import SplitNamespace
 from mypy.version import __version__
 
@@ -979,6 +979,12 @@ def add_invertible_flag(
         action="store_true",
         help="Disable experimental support for recursive type aliases",
     )
+    parser.add_argument(
+        "--enable-incomplete-feature",
+        action="append",
+        metavar="FEATURE",
+        help="Enable support of incomplete/experimental features for early preview",
+    )
     internals_group.add_argument(
         "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
     )
@@ -1107,6 +1113,7 @@ def add_invertible_flag(
     parser.add_argument(
         "--cache-map", nargs="+", dest="special-opts:cache_map", help=argparse.SUPPRESS
     )
+    # This one is deprecated, but we will keep it for few releases.
     parser.add_argument(
         "--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS
     )
@@ -1274,6 +1281,17 @@ def set_strict_flags() -> None:
     # Enabling an error code always overrides disabling
     options.disabled_error_codes -= options.enabled_error_codes
 
+    # Validate incomplete features.
+    for feature in options.enable_incomplete_feature:
+        if feature not in INCOMPLETE_FEATURES:
+            parser.error(f"Unknown incomplete feature: {feature}")
+    if options.enable_incomplete_features:
+        print(
+            "Warning: --enable-incomplete-features is deprecated, use"
+            " --enable-incomplete-feature=FEATURE instead"
+        )
+        options.enable_incomplete_feature = list(INCOMPLETE_FEATURES)
+
     # Compute absolute path for custom typeshed (if present).
     if options.custom_typeshed_dir is not None:
         options.abs_custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir)

Diff for: mypy/options.py

@@ -60,6 +60,11 @@ class BuildType:
     "debug_cache"
 }
 
+# Features that are currently incomplete/experimental
+TYPE_VAR_TUPLE: Final = "TypeVarTuple"
+UNPACK: Final = "Unpack"
+INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))
+
 
 class Options:
     """Options collected from flags."""
@@ -268,7 +273,8 @@ def __init__(self) -> None:
         self.dump_type_stats = False
         self.dump_inference_stats = False
         self.dump_build_stats = False
-        self.enable_incomplete_features = False
+        self.enable_incomplete_features = False  # deprecated
+        self.enable_incomplete_feature: list[str] = []
         self.timing_stats: str | None = None
 
         # -- test options --

Diff for: mypy/semanal.py

@@ -178,7 +178,7 @@
     type_aliases_source_versions,
     typing_extensions_aliases,
 )
-from mypy.options import Options
+from mypy.options import TYPE_VAR_TUPLE, Options
 from mypy.patterns import (
     AsPattern,
     ClassPattern,
@@ -3911,8 +3911,7 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool:
         if len(call.args) > 1:
             self.fail("Only the first argument to TypeVarTuple has defined semantics", s)
 
-        if not self.options.enable_incomplete_features:
-            self.fail('"TypeVarTuple" is not supported by mypy yet', s)
+        if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s):
             return False
 
         name = self.extract_typevarlike_name(s, call)
@@ -5973,6 +5972,16 @@ def note(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None:
             return
         self.errors.report(ctx.get_line(), ctx.get_column(), msg, severity="note", code=code)
 
+    def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool:
+        if feature not in self.options.enable_incomplete_feature:
+            self.fail(
+                f'"{feature}" support is experimental,'
+                f" use --enable-incomplete-feature={feature} to enable",
+                ctx,
+            )
+            return False
+        return True
+
     def accept(self, node: Node) -> None:
         try:
             node.accept(self)

Diff for: mypy/semanal_shared.py

@@ -82,6 +82,10 @@ def fail(
     def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None:
         raise NotImplementedError
 
+    @abstractmethod
+    def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool:
+        raise NotImplementedError
+
     @abstractmethod
     def record_incomplete_ref(self) -> None:
         raise NotImplementedError

Diff for: mypy/test/testcheck.py

@@ -10,6 +10,7 @@
 from mypy.build import Graph
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths
+from mypy.options import TYPE_VAR_TUPLE, UNPACK
 from mypy.semanal_main import core_modules
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path
@@ -110,7 +111,8 @@ def run_case_once(
         # Parse options after moving files (in case mypy.ini is being moved).
         options = parse_options(original_program_text, testcase, incremental_step)
         options.use_builtins_fixtures = True
-        options.enable_incomplete_features = True
+        if not testcase.name.endswith("_no_incomplete"):
+            options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         options.show_traceback = True
 
         # Enable some options automatically based on test file name.

Diff for: mypy/test/testfinegrained.py

@@ -29,7 +29,7 @@
 from mypy.errors import CompileError
 from mypy.find_sources import create_source_list
 from mypy.modulefinder import BuildSource
-from mypy.options import Options
+from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
 from mypy.server.mergecheck import check_consistency
 from mypy.server.update import sort_messages_preserving_file_order
 from mypy.test.config import test_temp_dir
@@ -153,7 +153,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         options.use_fine_grained_cache = self.use_cache and not build_cache
         options.cache_fine_grained = self.use_cache
         options.local_partial_types = True
-        options.enable_incomplete_features = True
+        options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         # Treat empty bodies safely for these test cases.
         options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
         if re.search("flags:.*--follow-imports", source) is None:

Diff for: mypy/test/testsemanal.py

@@ -11,7 +11,7 @@
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource
 from mypy.nodes import TypeInfo
-from mypy.options import Options
+from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
@@ -46,7 +46,7 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti
     options.semantic_analysis_only = True
     options.show_traceback = True
     options.python_version = PYTHON3_VERSION
-    options.enable_incomplete_features = True
+    options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
     return options
 
 

Diff for: mypy/test/testtransform.py

@@ -7,6 +7,7 @@
 from mypy import build
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource
+from mypy.options import TYPE_VAR_TUPLE, UNPACK
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options
@@ -39,7 +40,7 @@ def test_transform(testcase: DataDrivenTestCase) -> None:
         options = parse_options(src, testcase, 1)
         options.use_builtins_fixtures = True
         options.semantic_analysis_only = True
-        options.enable_incomplete_features = True
+        options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         options.show_traceback = True
         result = build.build(
             sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir

Diff for: mypy/typeanal.py

@@ -38,7 +38,7 @@
     check_arg_names,
     get_nongen_builtins,
 )
-from mypy.options import Options
+from mypy.options import UNPACK, Options
 from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
 from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs
 from mypy.tvar_scope import TypeVarLikeScope
@@ -569,9 +569,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ
             # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args)
             return self.named_type("builtins.bool")
         elif fullname in ("typing.Unpack", "typing_extensions.Unpack"):
-            # We don't want people to try to use this yet.
-            if not self.options.enable_incomplete_features:
-                self.fail('"Unpack" is not supported yet, use --enable-incomplete-features', t)
+            if not self.api.incomplete_feature_enabled(UNPACK, t):
                 return AnyType(TypeOfAny.from_error)
             return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column)
         return None

Diff for: mypyc/test-data/run-functions.test

@@ -1242,7 +1242,7 @@ def g() -> None:
 
 g()
 
-[case testIncompleteFeatureUnpackKwargsCompiled]
+[case testUnpackKwargsCompiled]
 from typing_extensions import Unpack, TypedDict
 
 class Person(TypedDict):

Diff for: mypyc/test/test_run.py

@@ -15,7 +15,7 @@
 
 from mypy import build
 from mypy.errors import CompileError
-from mypy.options import Options
+from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase
 from mypy.test.helpers import assert_module_equivalence, perform_file_operations
@@ -194,8 +194,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) ->
         options.preserve_asts = True
         options.allow_empty_bodies = True
         options.incremental = self.separate
-        if "IncompleteFeature" in testcase.name:
-            options.enable_incomplete_features = True
+        options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
 
         # Avoid checking modules/packages named 'unchecked', to provide a way
         # to test interacting with code we don't have types for.

Diff for: test-data/unit/check-flags.test

@@ -2117,3 +2117,14 @@ x: int = ""  # E: Incompatible types in assignment (expression has type "str", v
 [case testHideErrorCodes]
 # flags: --hide-error-codes
 x: int = ""  # E: Incompatible types in assignment (expression has type "str", variable has type "int")
+
+[case testTypeVarTupleDisabled_no_incomplete]
+from typing_extensions import TypeVarTuple
+Ts = TypeVarTuple("Ts")  # E: "TypeVarTuple" support is experimental, use --enable-incomplete-feature=TypeVarTuple to enable
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarTupleEnabled_no_incomplete]
+# flags: --enable-incomplete-feature=TypeVarTuple
+from typing_extensions import TypeVarTuple
+Ts = TypeVarTuple("Ts")  # OK
+[builtins fixtures/tuple.pyi]

Diff for: test-data/unit/cmdline.test

@@ -1419,7 +1419,6 @@ exclude = (?x)(
 [out]
 c/cpkg.py:1: error: "int" not callable
 
-
 [case testCmdlineTimingStats]
 # cmd: mypy --timing-stats timing.txt .
 [file b/__init__.py]
@@ -1435,6 +1434,9 @@ b\.c \d+
 # cmd: mypy --enable-incomplete-features a.py
 [file a.py]
 pass
+[out]
+Warning: --enable-incomplete-features is deprecated, use --enable-incomplete-feature=FEATURE instead
+== Return code: 0
 
 [case testShadowTypingModuleEarlyLoad]
 # cmd: mypy dir

Diff for: test-data/unit/fine-grained.test

@@ -9901,7 +9901,6 @@ x = 0  # Arbitrary change to trigger reprocessing
 a.py:3: note: Revealed type is "Tuple[Literal[1]?, Literal['x']?]"
 
 [case testUnpackKwargsUpdateFine]
-# flags: --enable-incomplete-features
 import m
 [file shared.py]
 from typing_extensions import TypedDict

Diff for: test-data/unit/lib-stub/typing_extensions.pyi

@@ -10,6 +10,9 @@ class _SpecialForm:
     def __getitem__(self, typeargs: Any) -> Any:
         pass
 
+    def __call__(self, arg: Any) -> Any:
+        pass
+
 NamedTuple = 0
 Protocol: _SpecialForm = ...
 def runtime_checkable(x: _T) -> _T: pass