ls command

.github/workflows/check-test-release.yml

@@ -32,12 +32,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9]
+        python: ["3.7", "3.8", "3.9"]
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
     - uses: actions/setup-python@v2
+      with:
+         python-version: ${{ matrix.python }}
     # legacy-resolver makes install faster by order of magnitude
     - run: pip install -U pre-commit .[tests] --use-deprecated=legacy-resolver
     - run: pytest

mlem/api/commands.py

@@ -2,17 +2,26 @@
 MLEM's Python API
 """
 import os
-from typing import Any, Optional, Union
+from collections import defaultdict
+from inspect import isabstract
+from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
 import click
 from pydantic import parse_obj_as
 
 from mlem.core.errors import InvalidArgumentError
-from mlem.core.meta_io import MLEM_DIR, deserialize
+from mlem.core.meta_io import (
+    META_FILE_NAME,
+    MLEM_DIR,
+    MLEM_EXT,
+    deserialize,
+    get_fs,
+)
 from mlem.core.metadata import load, load_meta, save
 from mlem.core.objects import DatasetMeta, MlemLink, MlemMeta, ModelMeta
 from mlem.pack import Packager
 from mlem.runtime.server.base import Server
+from mlem.utils.root import find_mlem_root
 
 
 def _get_dataset(dataset: Any) -> Any:
@@ -237,3 +246,39 @@ def serve(model: ModelMeta, server: Union[Server, str], **server_kwargs):
     else:
         server_obj = server
     server_obj.serve(interface)
+
+
+def ls(
+    repo: str = ".",
+    type_filter: Union[Type[MlemMeta], Sequence[Type[MlemMeta]], None] = None,
+    include_links: bool = True,
+) -> Dict[Type[MlemMeta], List[MlemMeta]]:
+    if type_filter is None:
+        type_filter = [
+            cls
+            for cls in MlemMeta.__type_map__.values()
+            if not isabstract(cls)
+        ]
+    if isinstance(type_filter, type) and issubclass(type_filter, MlemMeta):
+        type_filter = [type_filter]
+    fs, path = get_fs(repo)
+    mlem_root = find_mlem_root(path, fs)
+    res = defaultdict(list)
+    for cls in type_filter:
+        root_path = os.path.join(mlem_root, MLEM_DIR, cls.object_type)
+        files = fs.glob(
+            os.path.join(root_path, f"**{MLEM_EXT}"), recursive=True
+        )
+        for file in files:
+            meta = load_meta(file, follow_links=False, fs=fs, load_value=False)
+            if isinstance(meta, MlemLink):
+                link_name = os.path.relpath(file, root_path)[: -len(MLEM_EXT)]
+                is_auto_link = meta.mlem_link == os.path.join(
+                    link_name, META_FILE_NAME
+                )
+                if is_auto_link:
+                    meta = meta.load_link()
+                elif not include_links:
+                    continue
+            res[cls].append(meta)
+    return res

mlem/cli/info.py

@@ -1,12 +1,12 @@
 import os
 from pprint import pprint
+from typing import List, Type
 
 import click
 from fsspec.implementations.local import LocalFileSystem
 
 from mlem.cli.main import mlem_command
 from mlem.core.meta_io import get_fs
-from mlem.core.metadata import load_meta
 from mlem.core.objects import (
     MLEM_DIR,
     MLEM_EXT,
@@ -17,27 +17,26 @@
 from mlem.utils.root import find_mlem_root
 
 
-def _print_objects_of_type(path, type_):
-    cls = MlemMeta.__type_map__[type_]
-    fs, path = get_fs(path)
-    root_path = os.path.join(
-        find_mlem_root(path, fs), MLEM_DIR, cls.object_type
-    )
-    files = fs.glob(os.path.join(root_path, f"**{MLEM_EXT}"), recursive=True)
-    if len(files) == 0:
+def _print_objects_of_type(
+    cls: Type[MlemMeta], objects: List[MlemMeta], mlem_root: str
+):
+    if len(objects) == 0:
         return
-    print(type_.capitalize() + "s:")
-    for file in files:
-        file = file[: -len(MLEM_EXT)]
-        obj_name = os.path.relpath(file, root_path)
-        meta = load_meta(obj_name, follow_links=False, fs=fs)
+
+    print(cls.object_type.capitalize() + "s:")
+    for meta in objects:
+        obj_name = os.path.relpath(meta.name or "?", mlem_root)
         if (
             isinstance(meta, MlemLink)
             and obj_name != meta.mlem_link[: -len(MLEM_EXT)]
         ):
-            link = f"-> {meta.mlem_link}"
+            link = f"-> {os.path.dirname(meta.mlem_link)}"
+            obj_name = os.path.relpath(
+                obj_name, os.path.join(MLEM_DIR, cls.object_type)
+            )[: -len(MLEM_EXT)]
         else:
             link = ""
+            obj_name = os.path.dirname(obj_name)
         print("", "-", obj_name, *[link] if link else [])
 
 
@@ -50,19 +49,28 @@ def _print_objects_of_type(path, type_):
 
 @mlem_command()
 @click.argument(
-    "type_",
+    "type_filter",
     default="all",
 )
 @click.option("-r", "--repo", default=".")
-def ls(type_: str, repo: str):
+@click.option("+l/-l", "--links/--no-links", default=True, is_flag=True)
+def ls(type_filter: str, repo: str, links: bool):
     """List MLEM objects of {type} in current mlem_root."""
-    if type_ == "all":
-        for tp in MlemMeta.subtype_mapping():
-            _print_objects_of_type(repo, tp)
+    from mlem.api.commands import ls
+
+    if type_filter == "all":
+        types = None
     else:
-        type = TYPE_ALIASES.get(type_, type_)
-        _print_objects_of_type(repo, type)
-    return {"type": type_}
+        types = MlemMeta.__type_map__[
+            TYPE_ALIASES.get(type_filter, type_filter)
+        ]
+
+    objects = ls(repo, types, include_links=links)
+    fs, path = get_fs(repo)
+    mlem_root = find_mlem_root(path, fs)
+    for cls, objs in objects.items():
+        _print_objects_of_type(cls, objs, mlem_root)
+    return {"type_filter": type_filter}
 
 
 @mlem_command("pprint")

mlem/core/objects.py

@@ -70,9 +70,6 @@ class MlemMeta(MlemObject):
     name: Optional[str] = None
     fs: ClassVar[Optional[AbstractFileSystem]] = None
 
-    # class Config:
-    #     arbitrary_types_allowed = True
-
     def bind(self, name: str, fs: AbstractFileSystem):
         self.name = name
         self.fs = fs
@@ -104,13 +101,12 @@ def read(
             payload = safe_load(f)
         res = deserialize(payload, cls).bind(path, fs)
         if follow_links and isinstance(res, MlemLink):
-            linked_obj_path, linked_obj_fs = res.parse_link()
-            return cls.__type_map__[res.link_type].read(
-                linked_obj_path, fs=linked_obj_fs
-            )
+            return res.load_link()
         return res
 
-    def write_value(self) -> Optional[Artifacts]:
+    def write_value(
+        self, mlem_root: str  # pylint: disable=unused-argument
+    ) -> Optional[Artifacts]:
         return None
 
     def load_value(self):
@@ -137,7 +133,7 @@ def dump(
         if link or mlem_root is None:
             if check_extension and not name.endswith(MLEM_EXT):
                 raise ValueError(f"name={name} should end with {MLEM_EXT}")
-            path = name
+            path = os.path.join(mlem_root or ".", name)
         else:
             path = mlem_dir_path(
                 name,
@@ -149,7 +145,7 @@ def dump(
         with fs.open(path, "w") as f:
             safe_dump(serialize(self), f)
         if link:
-            self.make_link_in_mlem_dir()
+            self.make_link_in_mlem_dir(mlem_root)
         self.name = name
         self.fs = fs
 
@@ -167,10 +163,10 @@ def make_link(
         if path is not None:
             if raise_on_exist and os.path.exists(path):
                 raise ObjectExistsError(f"Object at {path} already exists")
-            link.dump(path, fs=fs)
+            link.dump(path, fs=fs, absolute=True)
         return link
 
-    def make_link_in_mlem_dir(self) -> None:
+    def make_link_in_mlem_dir(self, mlem_root: str = None) -> None:
         if self.name is None:
             raise MlemObjectNotSavedError(
                 "Cannot create link for not saved meta object"
@@ -179,6 +175,7 @@ def make_link_in_mlem_dir(self) -> None:
             os.path.dirname(self.name),
             fs=self.fs,
             obj_type=self.object_type,
+            mlem_root=mlem_root,
         )
         self.make_link(path=path)
 
@@ -225,6 +222,12 @@ class MlemLink(MlemMeta):
     link_type: str
     object_type = "link"
 
+    def load_link(self) -> T:
+        linked_obj_path, linked_obj_fs = self.parse_link()
+        return MlemMeta.__type_map__[self.link_type].read(
+            linked_obj_path, fs=linked_obj_fs
+        )
+
     def parse_link(self) -> Tuple[str, AbstractFileSystem]:
         if self.name is None:
             raise ValueError("Link is not saved")
@@ -306,7 +309,7 @@ def dump(
         if not name.endswith(MLEM_EXT):
             name = os.path.join(name, META_FILE_NAME)
         self.name = name
-        self.artifacts = self.write_value()
+        self.artifacts = self.write_value(mlem_root or ".")
         super().dump(
             name=name,
             fs=fs,
@@ -316,7 +319,7 @@ def dump(
             absolute=absolute,
         )
 
-    def write_value(self) -> Artifacts:
+    def write_value(self, mlem_root: str) -> Artifacts:
         raise NotImplementedError()
 
     @property
@@ -389,7 +392,7 @@ def localize(self, path: str):
 
 
 class ModelMeta(_ExternalMeta):
-    object_type = "model"
+    object_type: ClassVar = "model"
     model_type: ModelType
 
     @classmethod
@@ -398,8 +401,8 @@ def from_obj(cls, model: Any, sample_data: Any = None) -> "ModelMeta":
         mt.model = model
         return ModelMeta(model_type=mt, requirements=mt.get_requirements())
 
-    def write_value(self) -> Artifacts:
-        path = self.art_dir
+    def write_value(self, mlem_root: str) -> Artifacts:
+        path = os.path.join(mlem_root, self.art_dir)
         if self.model_type.model is not None:
             artifacts = self.model_type.io.dump(
                 self.fs, path, self.model_type.model
@@ -426,7 +429,7 @@ def __getattr__(self, item):
 
 class DatasetMeta(_ExternalMeta):
     __transient_fields__ = {"dataset"}
-    object_type = "dataset"
+    object_type: ClassVar = "dataset"
     reader: Optional[DatasetReader] = None
     dataset: ClassVar[Optional[Dataset]] = None
 
@@ -445,10 +448,10 @@ def from_data(cls, data: Any) -> "DatasetMeta":
         meta.dataset = dataset
         return meta
 
-    def write_value(self) -> Artifacts:
+    def write_value(self, mlem_root: str) -> Artifacts:
         if self.dataset is not None:
             reader, artifacts = self.dataset.dataset_type.get_writer().write(
-                self.dataset, self.fs, self.art_dir
+                self.dataset, self.fs, os.path.join(mlem_root, self.art_dir)
             )
             self.reader = reader
         else:
@@ -469,7 +472,7 @@ def get_value(self):
 
 class TargetEnvMeta(MlemMeta):
     __type_root__ = True
-    object_type = "env"
+    object_type: ClassVar = "env"
     alias: ClassVar = ...
     deployment_type: ClassVar = ...
 
@@ -490,7 +493,7 @@ def update(
 
 @dataclass
 class DeployMeta(MlemMeta):
-    object_type = "deployment"
+    object_type: ClassVar = "deployment"
 
     env_path: str
     model_path: str

tests/api/test_commands.py

@@ -5,8 +5,10 @@
 from numpy import ndarray
 
 from mlem.api import apply, link, load_meta
+from mlem.api.commands import ls
 from mlem.core.meta_io import MLEM_DIR, MLEM_EXT
-from mlem.core.objects import MlemLink, ModelMeta
+from mlem.core.objects import DatasetMeta, MlemLink, ModelMeta
+from tests.conftest import long
 
 
 @pytest.mark.parametrize(
@@ -56,3 +58,36 @@ def test_link_in_mlem_dir(model_path_mlem_root):
     assert isinstance(loaded_link_object, MlemLink)
     model = load_meta(link_dumped_to)
     assert isinstance(model, ModelMeta)
+
+
+def test_ls_local(mlem_root):
+    objects = ls(mlem_root)
+    assert len(objects) == 1
+    assert ModelMeta in objects
+    models = objects[ModelMeta]
+    assert len(models) == 2
+    model, lnk = models
+    if isinstance(model, MlemLink):
+        model, lnk = lnk, model
+
+    assert isinstance(model, ModelMeta)
+    assert isinstance(lnk, MlemLink)
+    assert os.path.join(mlem_root, lnk.mlem_link) == model.name
+
+
+@long
+def test_ls_remote():
+    objects = ls("https://github.com/iterative/example-mlem/")
+    assert len(objects) == 2
+    assert ModelMeta in objects
+    models = objects[ModelMeta]
+    assert len(models) == 2
+    model, lnk = models
+    if isinstance(model, MlemLink):
+        model, lnk = lnk, model
+
+    assert isinstance(model, ModelMeta)
+    assert isinstance(lnk, MlemLink)
+
+    assert DatasetMeta in objects
+    assert len(objects[DatasetMeta]) == 3