Skip to content
This repository was archived by the owner on Sep 13, 2023. It is now read-only.

Change args for docker build #604

Merged
merged 3 commits into from
Feb 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 35 additions & 26 deletions mlem/contrib/docker/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,9 @@ def uri(self, image: str):
:param image: image name"""
return image

def image_exists(self, client: docker.DockerClient, image: "DockerImage"):
def image_exists(
self, client: docker.DockerClient, image: "DockerImageOptions"
):
"""Check if image exists in this registry

:param client: DockerClient to use
Expand All @@ -105,7 +107,7 @@ def image_exists(self, client: docker.DockerClient, image: "DockerImage"):
def delete_image(
self,
client: docker.DockerClient,
image: "DockerImage",
image: "DockerImageOptions",
force: bool = False,
**kwargs,
):
Expand Down Expand Up @@ -133,11 +135,11 @@ def push(self, client, tag):
client.images.push(tag)
echo(EMOJI_UPLOAD + f"Pushed image {tag} to docker.io")

def image_exists(self, client, image: "DockerImage"):
def image_exists(self, client, image: "DockerImageOptions"):
return image_exists_at_dockerhub(image.uri)

def delete_image(
self, client, image: "DockerImage", force=False, **kwargs
self, client, image: "DockerImageOptions", force=False, **kwargs
):
logger.warning("Skipping deleting image %s from docker.io", image.name)

Expand Down Expand Up @@ -215,7 +217,7 @@ def _get_digest(self, name, tag):
return None
return r.headers["Docker-Content-Digest"]

def image_exists(self, client, image: "DockerImage"):
def image_exists(self, client, image: "DockerImageOptions"):
name = image.fullname
digest = self._get_digest(name, image.tag)
if digest is None:
Expand All @@ -231,7 +233,7 @@ def image_exists(self, client, image: "DockerImage"):
)

def delete_image(
self, client, image: "DockerImage", force=False, **kwargs
self, client, image: "DockerImageOptions", force=False, **kwargs
):
name = image.fullname
digest = self._get_digest(name, image.tag)
Expand All @@ -246,7 +248,7 @@ class DockerDaemon(MlemABC):
host: str = (
"" # TODO: https://github.com/iterative/mlem/issues/38 credentials
)
"""adress of the docker daemon (empty string for local)"""
"""Address of the docker daemon (empty string for local)"""

@contextlib.contextmanager
def client(self) -> Iterator[docker.DockerClient]:
Expand All @@ -255,10 +257,7 @@ def client(self) -> Iterator[docker.DockerClient]:
yield c


class DockerImage(BaseModel):
""":class:`.Image.Params` implementation for docker images
full uri for image looks like registry.host/repository/name:tag"""

class DockerImageOptions(BaseModel):
name: str
"""name of the image"""
tag: str = "latest"
Expand All @@ -267,8 +266,6 @@ class DockerImage(BaseModel):
"""repository of the image"""
registry: DockerRegistry = DockerRegistry()
"""DockerRegistry instance with this image"""
image_id: Optional[str] = None
"""internal docker id of this image"""

@property
def fullname(self):
Expand All @@ -291,6 +288,14 @@ def delete(self, client: docker.DockerClient, force=False, **kwargs):
self.registry.delete_image(client, self, force, **kwargs)


class DockerImage(DockerImageOptions):
"""Docker image parameters
full uri for image looks like registry.host/repository/name:tag"""

image_id: Optional[str] = None
"""Internal docker id of this image"""


class DockerEnv(MlemEnv):
"""MlemEnv implementation for docker environment"""

Expand Down Expand Up @@ -344,8 +349,8 @@ class DockerContainer(

container_name: Optional[str] = None
"""Name to use for container"""
image_name: Optional[str] = None
"""Name to use for image"""
image: Optional[DockerImageOptions] = None
"""Image configuration"""
ports: List[str] = []
"""Publish container ports. See https://docs.docker.com/config/containers/container-networking/#published-ports"""
params: Dict[str, str] = {}
Expand Down Expand Up @@ -391,7 +396,7 @@ def get_port_mapping(

@property
def ensure_image_name(self):
return self.image_name or self.container_name
return self.image.name if self.image else self.container_name

def _get_client(self, state: DockerContainerState):
raise NotImplementedError
Expand All @@ -405,20 +410,23 @@ def deploy(self, model: MlemModel):
from .helpers import build_model_image

image_name = (
self.image_name
or self.container_name
self.image.name
if self.image
else self.container_name
or generate_docker_container_name()
)
echo(EMOJI_BUILD + f"Creating docker image {image_name}")
with set_offset(2):
env = self.get_env()
state.image = build_model_image(
model,
image_name,
self.server
or project_config(
self.loc.project if self.is_saved else None
).server,
self.get_env(),
env.daemon,
env.registry,
force_overwrite=True,
**self.args.dict(),
)
Expand Down Expand Up @@ -542,10 +550,10 @@ class DockerImageBuilder(MlemBuilder, _DockerBuildMixin):
"""Build docker image from model"""

type: ClassVar[str] = "docker"
image: DockerImage
image: DockerImageOptions
"""Image parameters"""
env: DockerEnv = DockerEnv()
"""Where to build and push image. Defaults to local docker daemon"""
daemon: DockerDaemon = DockerDaemon(host="")
"""Docker daemon to use"""
force_overwrite: bool = True
"""Ignore existing image with same name"""
push: bool = True
Expand All @@ -568,7 +576,7 @@ def build_image(self, context_dir: str) -> DockerImage:
tag = self.image.uri
logger.debug("Building docker image %s from %s...", tag, context_dir)
echo(EMOJI_BUILD + f"Building docker image {tag}...")
with self.env.daemon.client() as client:
with self.daemon.client() as client:
if self.push:
self.image.registry.login(client)
if not self.force_overwrite and self.image.exists(client):
Expand All @@ -585,13 +593,14 @@ def build_image(self, context_dir: str) -> DockerImage:
rm=True,
platform=self.args.platform,
)
self.image.image_id = image.id
docker_image = DockerImage(**self.image.dict())
docker_image.image_id = image.id
echo(EMOJI_OK + f"Built docker image {tag}")

if self.push:
self.image.registry.push(client, tag)
docker_image.registry.push(client, tag)

return self.image
return docker_image
except errors.BuildError as e:
print_docker_logs(e.build_log, logging.ERROR)
raise
Expand Down
23 changes: 15 additions & 8 deletions mlem/contrib/docker/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,38 @@
from mlem.runtime.server import Server

from . import DockerImageBuilder
from .base import DockerBuildArgs, DockerEnv, DockerImage
from .base import (
DockerBuildArgs,
DockerDaemon,
DockerImage,
DockerImageOptions,
DockerRegistry,
)


def build_model_image(
model: MlemModel,
name: str,
server: Server = None,
env: DockerEnv = None,
daemon: DockerDaemon = None,
registry: DockerRegistry = None,
tag: str = "latest",
repository: str = None,
force_overwrite: bool = True,
push: bool = True,
**build_args
) -> DockerImage:
env = env or DockerEnv()
image = DockerImage(
name=name, tag=tag, repository=repository, registry=env.registry
registry = registry or DockerRegistry()
daemon = daemon or DockerDaemon()
image = DockerImageOptions(
name=name, tag=tag, repository=repository, registry=registry
)
builder = DockerImageBuilder(
server=server,
args=DockerBuildArgs(**build_args),
image=image,
env=env,
daemon=daemon,
force_overwrite=force_overwrite,
push=push,
)
builder.build(model)
return builder.image
return builder.build(model)
5 changes: 3 additions & 2 deletions mlem/contrib/kubernetes/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from mlem.runtime.server import Server
from mlem.ui import EMOJI_BUILD, echo, set_offset

from ..docker.base import DockerDaemon, DockerEnv, DockerRegistry
from ..docker.base import DockerDaemon, DockerRegistry
from ..docker.helpers import build_model_image


Expand All @@ -23,7 +23,8 @@ def build_k8s_docker(
meta,
image_name,
server,
DockerEnv(registry=registry, daemon=daemon),
daemon=daemon,
registry=registry,
tag=meta.meta_hash(),
force_overwrite=True,
platform=platform,
Expand Down
8 changes: 5 additions & 3 deletions mlem/contrib/sagemaker/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from ...core.objects import MlemModel
from ...ui import EMOJI_BUILD, EMOJI_KEY, echo, set_offset
from ..docker.base import DockerEnv, DockerImage, RemoteRegistry
from ..docker.base import DockerEnv, DockerImageOptions, RemoteRegistry
from ..docker.helpers import build_model_image

IMAGE_NAME = "mlem-sagemaker-runner"
Expand Down Expand Up @@ -81,13 +81,15 @@ def login(self, client):
def get_host(self) -> Optional[str]:
return f"{self.account}.dkr.ecr.{self.region}.amazonaws.com"

def image_exists(self, client, image: DockerImage):
def image_exists(self, client, image: DockerImageOptions):
images = self.ecr_client.list_images(repositoryName=image.name)[
"imageIds"
]
return len(images) > 0

def delete_image(self, client, image: DockerImage, force=False, **kwargs):
def delete_image(
self, client, image: DockerImageOptions, force=False, **kwargs
):
return self.ecr_client.batch_delete_image(
repositoryName=image.name,
imageIds=[{"imageTag": image.tag}],
Expand Down