Skip to content

Commit

Permalink
Merge branch 'main' into all-flow-everything
Browse files Browse the repository at this point in the history
  • Loading branch information
jakekaplan authored Feb 7, 2025
2 parents 0e5dda0 + 2b27587 commit aa353ac
Show file tree
Hide file tree
Showing 55 changed files with 1,893 additions and 481 deletions.
133 changes: 128 additions & 5 deletions docs/v3/api-ref/rest-api/server/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -9356,8 +9356,7 @@
"content": {
"application/json": {
"schema": {
"type": "object",
"title": "Response Validate Obj Ui Schemas Validate Post"
"$ref": "#/components/schemas/SchemaValuesValidationResponse"
}
}
}
Expand Down Expand Up @@ -11875,6 +11874,7 @@
"description": "The block schema's unique checksum"
},
"fields": {
"additionalProperties": true,
"type": "object",
"title": "Fields",
"description": "The block schema's field schema"
Expand Down Expand Up @@ -14328,10 +14328,12 @@
"Body_validate_obj_ui_schemas_validate_post": {
"properties": {
"json_schema": {
"additionalProperties": true,
"type": "object",
"title": "Json Schema"
},
"values": {
"additionalProperties": true,
"type": "object",
"title": "Values"
}
Expand Down Expand Up @@ -15304,6 +15306,7 @@
"type": "null"
}
],
"additionalProperties": true,
"title": "Parameter Openapi Schema",
"description": "The parameter schema of the flow, including defaults."
},
Expand Down Expand Up @@ -16215,6 +16218,7 @@
"type": "null"
}
],
"additionalProperties": true,
"title": "Parameter Openapi Schema",
"description": "The parameter schema of the flow, including defaults."
},
Expand Down Expand Up @@ -16502,7 +16506,7 @@
}
],
"title": "Slug",
"description": "A unique slug for the schedule."
"description": "A unique identifier for the schedule."
}
},
"additionalProperties": false,
Expand Down Expand Up @@ -16572,7 +16576,7 @@
}
],
"title": "Slug",
"description": "A unique slug for the schedule."
"description": "A unique identifier for the schedule."
}
},
"additionalProperties": false,
Expand Down Expand Up @@ -16631,7 +16635,7 @@
},
"schedules": {
"items": {
"$ref": "#/components/schemas/DeploymentScheduleCreate"
"$ref": "#/components/schemas/DeploymentScheduleUpdate"
},
"type": "array",
"title": "Schedules",
Expand Down Expand Up @@ -16673,6 +16677,18 @@
"title": "Parameters",
"description": "Parameters for flow runs scheduled by the deployment."
},
"parameter_openapi_schema": {
"anyOf": [
{
"type": "object"
},
{
"type": "null"
}
],
"title": "Parameter Openapi Schema",
"description": "The parameter schema of the flow, including defaults."
},
"tags": {
"items": {
"type": "string"
Expand Down Expand Up @@ -16736,6 +16752,20 @@
"title": "Job Variables",
"description": "Overrides for the flow's infrastructure configuration."
},
"pull_steps": {
"anyOf": [
{
"items": {
"type": "object"
},
"type": "array"
},
{
"type": "null"
}
],
"title": "Pull Steps"
},
"entrypoint": {
"anyOf": [
{
Expand Down Expand Up @@ -21657,6 +21687,99 @@
"title": "SavedSearchFilter",
"description": "A filter for a saved search model. Intended for use by the Prefect UI."
},
"SchemaValueIndexError": {
"properties": {
"index": {
"type": "integer",
"title": "Index"
},
"errors": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"$ref": "#/components/schemas/SchemaValuePropertyError"
},
{
"$ref": "#/components/schemas/SchemaValueIndexError"
}
]
},
"type": "array",
"title": "Errors"
}
},
"type": "object",
"required": [
"index",
"errors"
],
"title": "SchemaValueIndexError"
},
"SchemaValuePropertyError": {
"properties": {
"property": {
"type": "string",
"title": "Property"
},
"errors": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"$ref": "#/components/schemas/SchemaValuePropertyError"
},
{
"$ref": "#/components/schemas/SchemaValueIndexError"
}
]
},
"type": "array",
"title": "Errors"
}
},
"type": "object",
"required": [
"property",
"errors"
],
"title": "SchemaValuePropertyError"
},
"SchemaValuesValidationResponse": {
"properties": {
"errors": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"$ref": "#/components/schemas/SchemaValuePropertyError"
},
{
"$ref": "#/components/schemas/SchemaValueIndexError"
}
]
},
"type": "array",
"title": "Errors"
},
"valid": {
"type": "boolean",
"title": "Valid"
}
},
"type": "object",
"required": [
"errors",
"valid"
],
"title": "SchemaValuesValidationResponse"
},
"SendNotification": {
"properties": {
"type": {
Expand Down
2 changes: 1 addition & 1 deletion src/integrations/prefect-dbt/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ classifiers = [
"Topic :: Software Development :: Libraries",
]
dependencies = [
"prefect>=3.0.0",
"prefect>=3.1.15",
"dbt-core>=1.7.0",
"prefect_shell>=0.3.0",
"sgqlc>=16.0.0",
Expand Down
54 changes: 54 additions & 0 deletions src/integrations/prefect-docker/prefect_docker/types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import re
from typing import Annotated

from pydantic import AfterValidator


def assert_volume_str(volume: str) -> str:
"""
Validate a Docker volume string and raise `ValueError` if invalid.
"""
if not isinstance(volume, str): # type: ignore[reportUnnecessaryIsInstance]
raise ValueError("Invalid volume specification: must be a string")
vol = volume.strip()
if not vol:
raise ValueError("Invalid volume specification: cannot be empty")
pattern = re.compile(
r"^(?:"
r"(?P<container_only>/[^:]+)" # anonymous volume: container path only
r"|"
r"(?P<host>(?:[A-Za-z]:\\|\\\\|/)?[^:]+):"
r"(?P<container_path>(/)?[^:]+)"
r"(?::(?P<options>.+))?"
r")$"
)
match = pattern.match(vol)
if not match:
raise ValueError(f"Invalid volume specification: {volume}")
# Anonymous volume: just a container path (must start with '/')
if match.group("container_only"):
return vol
host_part = match.group("host")
container_path = match.group("container_path")
options = match.group("options")
# Determine if host is a bind mount (absolute host path) or a named volume.
is_unix_host = host_part.startswith("/")
is_windows_drive = re.match(r"^[A-Za-z]:\\", host_part) is not None
is_unc = host_part.startswith("\\\\")
if is_unix_host or is_windows_drive or is_unc:
# For bind mounts, container path must be absolute.
if not container_path.startswith("/"):
raise ValueError("For bind mounts, container path must be absolute")
else:
# For named volumes, host must not contain path separators.
if "/" in host_part or "\\" in host_part:
raise ValueError(f"Invalid volume name: {host_part}")
if options is not None:
if options not in ("ro", "rw"):
raise ValueError(
f"Invalid volume option: {options!r}. Must be 'ro' or 'rw'"
)
return vol


VolumeStr = Annotated[str, AfterValidator(assert_volume_str)]
33 changes: 3 additions & 30 deletions src/integrations/prefect-docker/prefect_docker/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import packaging.version
from docker import DockerClient
from docker.models.containers import Container
from pydantic import AfterValidator, Field
from pydantic import Field
from slugify import slugify
from typing_extensions import Annotated, Literal
from typing_extensions import Literal

import prefect
from prefect.client.orchestration import ServerType, get_client
Expand All @@ -47,6 +47,7 @@
)
from prefect.workers.base import BaseJobConfiguration, BaseWorker, BaseWorkerResult
from prefect_docker.credentials import DockerRegistryCredentials
from prefect_docker.types import VolumeStr

CONTAINER_LABELS = {
"io.prefect.version": prefect.__version__,
Expand All @@ -61,34 +62,6 @@ class ImagePullPolicy(enum.Enum):
NEVER = "Never"


def assert_volume_str(v: str) -> str:
"""Assert that a string is a valid Docker volume string."""
if not isinstance(v, str):
raise ValueError("Volume must be a string")

# Regex pattern for valid Docker volume strings, including Windows paths
pattern = r"^([a-zA-Z]:\\|/)?[^:]+:(/)?[^:]+(:ro|:rw)?$"

match = re.match(pattern, v)
if not match:
raise ValueError(f"Invalid volume string: {v!r}")

_, _, mode = match.groups()

# Check for empty parts
if ":" not in v or v.startswith(":") or v.endswith(":"):
raise ValueError(f"Volume string contains empty part: {v!r}")

# If there's a mode, it must be 'ro' or 'rw'
if mode and mode not in (":ro", ":rw"):
raise ValueError(f"Invalid volume mode: {mode!r}. Must be ':ro' or ':rw'")

return v


VolumeStr = Annotated[str, AfterValidator(assert_volume_str)]


class DockerWorkerJobConfiguration(BaseJobConfiguration):
"""
Configuration class used by the Docker worker.
Expand Down
12 changes: 7 additions & 5 deletions src/integrations/prefect-docker/tests/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@
from docker.models.containers import Container
from exceptiongroup import ExceptionGroup
from prefect_docker.credentials import DockerRegistryCredentials
from prefect_docker.types import VolumeStr
from prefect_docker.worker import (
CONTAINER_LABELS,
DockerWorker,
DockerWorkerJobConfiguration,
VolumeStr,
)
from pydantic import TypeAdapter, ValidationError

import prefect.main # noqa
from prefect.client.schemas import FlowRun
from prefect.events import RelatedResource
from prefect.settings import (
Expand All @@ -35,12 +36,12 @@


@pytest.fixture(autouse=True)
def bypass_api_check(monkeypatch):
def bypass_api_check(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("PREFECT_DOCKER_TEST_MODE", True)


@pytest.fixture
def mock_docker_client(monkeypatch):
def mock_docker_client(monkeypatch: pytest.MonkeyPatch):
mock = MagicMock(name="DockerClient", spec=docker.DockerClient)
mock.version.return_value = {"Version": "20.10"}

Expand Down Expand Up @@ -276,9 +277,10 @@ async def test_uses_volumes_setting(
"/home/user:/home/docker:rw",
"C:\\path\\on\\windows:/path/in/container",
"\\\\host\\share:/path/in/container",
"/data", # anonymous volume
],
)
def test_valid_volume_strings(volume_str):
def test_valid_volume_strings(volume_str: str):
assert TypeAdapter(VolumeStr).validate_python(volume_str) == volume_str


Expand All @@ -296,7 +298,7 @@ def test_valid_volume_strings(volume_str):
"", # empty string
],
)
def test_invalid_volume_strings(volume_str):
def test_invalid_volume_strings(volume_str: str):
with pytest.raises(ValidationError, match="Invalid volume"):
TypeAdapter(VolumeStr).validate_python(volume_str)

Expand Down
Loading

0 comments on commit aa353ac

Please # to comment.