Merge branch 'main' into feature/client

.github/workflows/check-test-release.yml

@@ -74,7 +74,7 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install pre-commit .[tests] --use-deprecated=legacy-resolver
-    - run: pre-commit run pylint -a --show-diff-on-failure
+    - run: pre-commit run pylint -a -v --show-diff-on-failure
       if: matrix.python != '3.7'
     - name: Run tests
       timeout-minutes: 30

.gitignore

@@ -138,3 +138,5 @@ dmypy.json
 cython_debug/
 
 .idea
+
+mlem/_mlem_version.py

mlem/api/commands.py

@@ -254,7 +254,6 @@ def link(
 def pack(
     packager: Union[str, Packager],
     model: Union[str, ModelMeta],
-    out: str,
     **packager_kwargs,
 ):
     """Pack model in docker-build-ready folder or directly build a docker image.
@@ -263,11 +262,10 @@ def pack(
         packager (Union[str, Packager]): Packager to use.
             Out-of-the-box supported string values are "docker_dir" and "docker".
         model (Union[str, ModelMeta]): The model to pack.
-        out (str): Path for "docker_dir", image name for "docker".
     """
     model = get_model_meta(model)
     return ensure_mlem_object(Packager, packager, **packager_kwargs).package(
-        model, out
+        model
     )
 
 

mlem/cli/__init__.py

@@ -6,6 +6,7 @@
 from mlem.cli.config import config
 from mlem.cli.create import create
 from mlem.cli.deploy import deploy
+from mlem.cli.dev import dev
 from mlem.cli.import_object import import_object
 from mlem.cli.info import ls, pretty_print
 from mlem.cli.init import init
@@ -30,6 +31,7 @@
     "create",
     "import_object",
     "list_types",
+    "dev",
 ]
 
 

mlem/cli/create.py

@@ -4,7 +4,13 @@
 
 from ..core.base import build_mlem_object
 from ..core.objects import MlemMeta
-from .main import mlem_command, option_external, option_link, option_repo
+from .main import (
+    mlem_command,
+    option_external,
+    option_link,
+    option_repo,
+    wrap_build_error,
+)
 
 
 @mlem_command("create", section="object")
@@ -29,5 +35,6 @@ def create(
         $ mlem create env heroku production -c api_key=<...>
     """
     cls = MlemMeta.__type_map__[object_type]
-    meta = build_mlem_object(cls, subtype, conf, [])
+    with wrap_build_error(subtype, cls):
+        meta = build_mlem_object(cls, subtype, conf, [])
     meta.dump(path, repo=repo, link=link, external=external)

mlem/cli/dev.py

@@ -0,0 +1,35 @@
+from typer import Argument, Typer
+
+from mlem.cli.main import MlemGroupSection, app, mlem_command
+from mlem.ext import MLEM_ENTRY_POINT, find_implementations, load_entrypoints
+from mlem.ui import echo
+
+dev = Typer(name="dev", cls=MlemGroupSection("common"), hidden=True)
+app.add_typer(dev)
+
+
+@dev.callback()
+def dev_callback():
+    """Developer utility tools"""
+
+
+@mlem_command(parent=dev, aliases=["fi"])
+def find_implementations_diff(
+    root: str = Argument(MLEM_ENTRY_POINT, help="root entry point")
+):
+    """Loads `root` module or package and finds implementations of MLEM base classes
+    Shows differences between what was found and what is registered in entrypoints
+
+    Examples:
+        $ mlem dev fi
+    """
+    exts = {e.entry for e in load_entrypoints().values()}
+    impls = set(find_implementations(root)[MLEM_ENTRY_POINT])
+    extra = exts.difference(impls)
+    if extra:
+        echo("Remove implementations:")
+        echo("\n".join(extra))
+    new = impls.difference(exts)
+    if new:
+        echo("Add implementations:")
+        echo("\n".join(new))

mlem/cli/main.py

@@ -1,3 +1,4 @@
+import contextlib
 import logging
 import typing as t
 from collections import defaultdict
@@ -9,7 +10,8 @@
 import typer
 from click import Abort, ClickException, Command, HelpFormatter, pass_context
 from click.exceptions import Exit
-from pydantic import ValidationError, parse_obj_as
+from pydantic import BaseModel, MissingError, ValidationError, parse_obj_as
+from pydantic.error_wrappers import ErrorWrapper
 from typer import Context, Option, Typer
 from typer.core import TyperCommand, TyperGroup
 from yaml import safe_load
@@ -19,6 +21,8 @@
 from mlem.constants import MLEM_DIR
 from mlem.core.base import MlemObject, build_mlem_object
 from mlem.core.errors import MlemError
+from mlem.core.metadata import load_meta
+from mlem.core.objects import MlemMeta
 from mlem.ui import EMOJI_FAIL, EMOJI_MLEM, bold, cli_echo, color, echo
 
 
@@ -213,7 +217,7 @@ def mlem_callback(
         $ mlem link logreg latest
         $ mlem apply latest https://github.com/iterative/example-mlem/data/test_x -o pred
         $ mlem serve latest fastapi -c port=8001
-        $ mlem pack latest build/ docker_dir -c server.type=fastapi
+        $ mlem pack latest docker_dir -c target=build/ -c server.type=fastapi
     """
     if ctx.invoked_subcommand is None and show_version:
         with cli_echo():
@@ -290,6 +294,7 @@ def inner(ctx, *iargs, **ikwargs):
                     echo(
                         "Please report it here: <https://github.com/iterative/mlem/issues>"
                     )
+                raise typer.Exit(1)
             finally:
                 send_cli_call(cmd_name, error_msg=error, **res)
 
@@ -351,25 +356,80 @@ def option_file_conf(type_: str = None):
     )
 
 
+def _iter_errors(
+    errors: t.Sequence[t.Any], model: Type, loc: Optional[Tuple] = None
+):
+    for error in errors:
+        if isinstance(error, ErrorWrapper):
+
+            if loc:
+                error_loc = loc + error.loc_tuple()
+            else:
+                error_loc = error.loc_tuple()
+
+            if isinstance(error.exc, ValidationError):
+                yield from _iter_errors(
+                    error.exc.raw_errors, error.exc.model, error_loc
+                )
+            else:
+                yield error_loc, model, error.exc
+
+
+def _format_validation_error(error: ValidationError) -> List[str]:
+    res = []
+    for loc, model, exc in _iter_errors(error.raw_errors, error.model):
+        path = ".".join(loc_part for loc_part in loc if loc_part != "__root__")
+        field_type = model.__fields__[loc[-1]].type_
+        if (
+            isinstance(exc, MissingError)
+            and isinstance(field_type, type)
+            and issubclass(field_type, BaseModel)
+        ):
+            msgs = [
+                str(EMOJI_FAIL + f"field `{path}.{f.name}`: {exc}")
+                for f in field_type.__fields__.values()
+                if f.required
+            ]
+            if msgs:
+                res.extend(msgs)
+            else:
+                res.append(str(EMOJI_FAIL + f"field `{path}`: {exc}"))
+        else:
+            res.append(str(EMOJI_FAIL + f"field `{path}`: {exc}"))
+    return res
+
+
+@contextlib.contextmanager
+def wrap_build_error(subtype, model: Type[MlemObject]):
+    try:
+        yield
+    except ValidationError as e:
+        msgs = "\n".join(_format_validation_error(e))
+        raise typer.BadParameter(
+            f"Error on constructing {subtype} {model.abs_name}:\n{msgs}"
+        ) from e
+
+
 def config_arg(
     model: Type[MlemObject],
     load: Optional[str],
     subtype: str,
     conf: Optional[List[str]],
     file_conf: Optional[List[str]],
 ):
+    obj: MlemObject
     if load is not None:
-        with open(load, "r", encoding="utf8") as of:
-            obj = parse_obj_as(model, safe_load(of))
+        if issubclass(model, MlemMeta):
+            obj = load_meta(load, force_type=model)
+        else:
+            with open(load, "r", encoding="utf8") as of:
+                obj = parse_obj_as(model, safe_load(of))
     else:
         if not subtype:
             raise typer.BadParameter(
                 f"Cannot configure {model.abs_name}: either subtype or --load should be provided"
             )
-        try:
+        with wrap_build_error(subtype, model):
             obj = build_mlem_object(model, subtype, conf, file_conf)
-        except ValidationError as e:
-            raise typer.BadParameter(
-                f"Error on constructing {subtype} {model.__name__}: {e}"
-            ) from e
+
     return obj

mlem/cli/package.py

@@ -19,7 +19,6 @@
 @mlem_command("pack", section="runtime")
 def pack(
     model: str = Argument(..., help="Path to model"),
-    out: str = Argument(..., help="Name of the output"),
     subtype: str = Argument(
         "",
         help=f"Type of packing. Choices: {list_implementations(Packager)}",
@@ -36,12 +35,15 @@ def pack(
 
     Examples:
         Build docker image from model
-        $ mlem pack mymodel mymodel-docker-image docker -c server.type=fastapi -c image.name=myimage
+        $ mlem pack mymodel docker -c server.type=fastapi -c image.name=myimage
+
+        Create pack docker_dir declaration and build it
+        $ mlem create packager docker_dir -c server=fastapi -c target=build pack_dock
+        $ mlem pack mymodel --load pack_dock
     """
     from mlem.api.commands import pack
 
     pack(
         config_arg(Packager, load, subtype, conf, file_conf),
         load_meta(model, repo, rev),
-        out,
     )

mlem/cli/types.py

@@ -4,16 +4,18 @@
 
 from mlem.cli.main import mlem_command
 from mlem.core.base import MlemObject
+from mlem.core.objects import MlemMeta
 from mlem.ext import list_implementations
-from mlem.ui import EMOJI_CASE, echo
+from mlem.ui import EMOJI_BASE, bold, echo
 
 
 @mlem_command("types", hidden=True)
 def list_types(
     subtype: Optional[str] = Argument(
         None,
         help="Subtype to list implementations. List subtypes if not provided",
-    )
+    ),
+    meta_type: Optional[str] = Argument(None, help="Type of `meta` subtype"),
 ):
     """List MLEM types implementations available in current env.
     If subtype is not provided, list ABCs
@@ -27,10 +29,18 @@ def list_types(
     """
     if subtype is None:
         for at in MlemObject.abs_types:
-            echo(EMOJI_CASE + at.abs_name + ":")
+            echo(EMOJI_BASE + bold(at.abs_name) + ":")
             echo(
                 f"\tBase class: {at.__module__}.{at.__name__}\n\t{at.__doc__.strip()}"
             )
-
+    elif subtype == MlemMeta.abs_name:
+        if meta_type is None:
+            echo(list(MlemMeta.non_abstract_subtypes().keys()))
+        else:
+            echo(
+                list_implementations(
+                    MlemMeta, MlemMeta.non_abstract_subtypes()[meta_type]
+                )
+            )
     else:
         echo(list_implementations(subtype))

mlem/contrib/docker/base.py

@@ -291,40 +291,43 @@ def image_exists(self, image: DockerImage):
             return image.exists(client)
 
 
-class DockerDirPackager(Packager):
-    type: ClassVar[str] = "docker_dir"
+class _DockerPackMixin(BaseModel):
     server: Server
     args: DockerBuildArgs = DockerBuildArgs()
 
-    def package(self, obj: ModelMeta, out: str):
+
+class DockerDirPackager(Packager, _DockerPackMixin):
+    type: ClassVar[str] = "docker_dir"
+    target: str
+
+    def package(self, obj: ModelMeta):
         docker_dir = DockerModelDirectory(
             model=obj,
             server=self.server,
-            path=out,
+            path=self.target,
             docker_args=self.args,
             debug=True,
         )
         docker_dir.write_distribution()
         return docker_dir
 
 
-class DockerImagePackager(DockerDirPackager):
+class DockerImagePackager(Packager, _DockerPackMixin):
     type: ClassVar[str] = "docker"
     image: DockerImage
     env: DockerEnv = DockerEnv()
     force_overwrite: bool = False
     push: bool = True
 
-    def package(self, obj: ModelMeta, out: str) -> DockerImage:
+    def package(self, obj: ModelMeta) -> DockerImage:
         with tempfile.TemporaryDirectory(prefix="mlem_build_") as tempdir:
             if self.args.prebuild_hook is not None:
                 self.args.prebuild_hook(  # pylint: disable=not-callable # but it is
                     self.args.python_version
                 )
-            super().package(obj, tempdir)
-            if not self.image.name:
-                # TODO: https://github.com/iterative/mlem/issues/65
-                self.image.name = out
+            DockerDirPackager(
+                server=self.server, args=self.args, target=tempdir
+            ).package(obj)
 
             return self.build(tempdir)