Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Add SOURCE_DATE_EPOCH support when building packages #766

Merged
merged 1 commit into from
Oct 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 27 additions & 14 deletions src/poetry/core/masonry/builders/sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def build(

name = distribution_name(self._package.name)
target = target_dir / f"{name}-{self._meta.version}.tar.gz"
gz = GzipFile(target.as_posix(), mode="wb", mtime=0)
gz = GzipFile(target.as_posix(), mode="wb", mtime=self._get_archive_mtime())
tar = tarfile.TarFile(
target.as_posix(), mode="w", fileobj=gz, format=tarfile.PAX_FORMAT
)
Expand All @@ -93,26 +93,25 @@ def build(

if self._poetry.package.build_should_generate_setup():
setup = self.build_setup()
tar_info = tarfile.TarInfo(pjoin(tar_dir, "setup.py"))
tar_info.size = len(setup)
tar_info.mtime = 0
tar_info = self.clean_tarinfo(tar_info)
tar.addfile(tar_info, BytesIO(setup))
self.add_file_to_tar(tar, pjoin(tar_dir, "setup.py"), setup)

pkg_info = self.build_pkg_info()

tar_info = tarfile.TarInfo(pjoin(tar_dir, "PKG-INFO"))
tar_info.size = len(pkg_info)
tar_info.mtime = 0
tar_info = self.clean_tarinfo(tar_info)
tar.addfile(tar_info, BytesIO(pkg_info))
self.add_file_to_tar(tar, pjoin(tar_dir, "PKG-INFO"), pkg_info)
finally:
tar.close()
gz.close()

logger.info(f"Built <comment>{target.name}</comment>")
return target

def add_file_to_tar(
Secrus marked this conversation as resolved.
Show resolved Hide resolved
self, tar: tarfile.TarFile, file_name: str, content: bytes
) -> None:
tar_info = tarfile.TarInfo(file_name)
tar_info.size = len(content)
tar_info = self.clean_tarinfo(tar_info)
tar.addfile(tar_info, BytesIO(content))

def build_setup(self) -> bytes:
from poetry.core.masonry.utils.package_include import PackageInclude

Expand Down Expand Up @@ -399,8 +398,7 @@ def convert_dependencies(

return main, dict(extras)

@classmethod
def clean_tarinfo(cls, tar_info: TarInfo) -> TarInfo:
def clean_tarinfo(self, tar_info: TarInfo) -> TarInfo:
"""
Clean metadata from a TarInfo object to make it more reproducible.
Expand All @@ -416,6 +414,21 @@ def clean_tarinfo(cls, tar_info: TarInfo) -> TarInfo:
ti.gid = 0
ti.uname = ""
ti.gname = ""
ti.mtime = self._get_archive_mtime()
ti.mode = normalize_file_permissions(ti.mode)

return ti

@staticmethod
def _get_archive_mtime() -> int:
if source_date_epoch := os.getenv("SOURCE_DATE_EPOCH"):
try:
return int(source_date_epoch)
except ValueError:
logger.warning(
"SOURCE_DATE_EPOCH environment variable is not an int,"
" using mtime=0"
)
return 0
logger.info("SOURCE_DATE_EPOCH environment variable is not set, using mtime=0")
return 0
44 changes: 39 additions & 5 deletions src/poetry/core/masonry/builders/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,14 @@

if TYPE_CHECKING:
from collections.abc import Iterator
from typing import Tuple

from packaging.utils import NormalizedName

from poetry.core.poetry import Poetry

ZipInfoTimestamp = Tuple[int, int, int, int, int, int]

wheel_file_template = """\
Wheel-Version: 1.0
Generator: poetry-core {version}
Expand Down Expand Up @@ -446,7 +449,7 @@ def _add_file(
) -> None:
# We always want to have /-separated paths in the zip file and in RECORD
rel_path_name = rel_path.as_posix()
zinfo = zipfile.ZipInfo(rel_path_name)
zinfo = zipfile.ZipInfo(rel_path_name, self._get_zipfile_date_time())

# Normalize permission bits to either 755 (executable) or 644
st_mode = full_path.stat().st_mode
Expand Down Expand Up @@ -479,10 +482,7 @@ def _write_to_zip(
sio = StringIO()
yield sio

# The default is a fixed timestamp rather than the current time, so
# that building a wheel twice on the same computer can automatically
# give you the exact same result.
date_time = (2016, 1, 1, 0, 0, 0)
date_time = self._get_zipfile_date_time()
zi = zipfile.ZipInfo(rel_path, date_time)
zi.external_attr = (0o644 & 0xFFFF) << 16 # Unix attributes
b = sio.getvalue().encode("utf-8")
Expand All @@ -492,6 +492,40 @@ def _write_to_zip(
wheel.writestr(zi, b, compress_type=zipfile.ZIP_DEFLATED)
self._records.append((rel_path, hash_digest, len(b)))

@staticmethod
def _get_zipfile_date_time() -> ZipInfoTimestamp:
import time

# The default is a fixed timestamp rather than the current time, so
# that building a wheel twice on the same computer can automatically
# give you the exact same result.
default = (2016, 1, 1, 0, 0, 0)
try:
_env_date = time.gmtime(int(os.environ["SOURCE_DATE_EPOCH"]))[:6]
except ValueError:
logger.warning(
"SOURCE_DATE_EPOCH environment variable value"
" is not an int, setting zipinfo date to default=%s",
default,
)
return default
except KeyError:
logger.info(
"SOURCE_DATE_EPOCH environment variable not set,"
" setting zipinfo date to default=%s",
default,
)
return default
else:
if _env_date[0] < 1980:
Secrus marked this conversation as resolved.
Show resolved Hide resolved
logger.warning(
"zipinfo date can't be earlier than 1980,"
" setting zipinfo date to default=%s",
default,
)
return default
return _env_date

def _write_entry_points(self, fp: TextIO) -> None:
"""
Write entry_points.txt.
Expand Down
69 changes: 58 additions & 11 deletions tests/masonry/builders/test_sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@


if TYPE_CHECKING:
from pytest import LogCaptureFixture
from pytest import MonkeyPatch
from pytest_mock import MockerFixture

fixtures_dir = Path(__file__).parent / "fixtures"
Expand Down Expand Up @@ -496,7 +498,6 @@ def test_default_with_excluded_data(mocker: MockerFixture) -> None:
)

assert sdist.exists()

with tarfile.open(str(sdist), "r") as tar:
names = tar.getnames()
assert len(names) == len(set(names))
Expand All @@ -506,16 +507,6 @@ def test_default_with_excluded_data(mocker: MockerFixture) -> None:
assert "my_package-1.2.3/my_package/data/data1.txt" in names
assert "my_package-1.2.3/pyproject.toml" in names
assert "my_package-1.2.3/PKG-INFO" in names
# all last modified times should be set to a valid timestamp
for tarinfo in tar.getmembers():
if tarinfo.name in [
"my_package-1.2.3/setup.py",
"my_package-1.2.3/PKG-INFO",
]:
# generated files have timestamp set to 0
assert tarinfo.mtime == 0
continue
assert tarinfo.mtime > 0


def test_src_excluded_nested_data() -> None:
Expand Down Expand Up @@ -693,3 +684,59 @@ def test_split_source() -> None:
ns: dict[str, Any] = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
assert "" in ns["package_dir"] and "module_b" in ns["package_dir"]


def test_sdist_members_mtime_default(caplog: LogCaptureFixture) -> None:
import logging

caplog.set_level(logging.INFO)
poetry = Factory().create_poetry(project("module1"))

builder = SdistBuilder(poetry)
builder.build()

sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz"

assert sdist.exists()

with tarfile.open(str(sdist), "r") as tar:
for tarinfo in tar.getmembers():
assert tarinfo.mtime == 0

assert (
"SOURCE_DATE_EPOCH environment variable is not set," " using mtime=0"
) in caplog.messages


def test_sdist_mtime_set_from_envvar(monkeypatch: MonkeyPatch) -> None:
monkeypatch.setenv("SOURCE_DATE_EPOCH", "1727883000")
poetry = Factory().create_poetry(project("module1"))

builder = SdistBuilder(poetry)
builder.build()

sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz"

assert sdist.exists()

with tarfile.open(str(sdist), "r") as tar:
for tarinfo in tar.getmembers():
assert tarinfo.mtime == 1727883000


def test_sdist_mtime_set_from_envvar_not_int(
monkeypatch: MonkeyPatch, caplog: LogCaptureFixture
) -> None:
monkeypatch.setenv("SOURCE_DATE_EPOCH", "october")
poetry = Factory().create_poetry(project("module1"))

builder = SdistBuilder(poetry)
builder.build()

sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz"

assert sdist.exists()

assert (
"SOURCE_DATE_EPOCH environment variable is not an " "int, using mtime=0"
) in caplog.messages
82 changes: 82 additions & 0 deletions tests/masonry/builders/test_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@


if TYPE_CHECKING:
from pytest import LogCaptureFixture
from pytest import MonkeyPatch
from pytest_mock import MockerFixture

Expand Down Expand Up @@ -469,3 +470,84 @@ def test_generated_script_file(tmp_path: Path) -> None:

with zipfile.ZipFile(str(whl)) as z:
assert "generated_script_file-0.1.data/scripts/script.sh" in z.namelist()


def test_dist_info_date_time_default_value(caplog: LogCaptureFixture) -> None:
import logging

caplog.set_level(logging.INFO)
module_path = fixtures_dir / "complete"
WheelBuilder.make(Factory().create_poetry(module_path))

whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl"

default_date_time = (2016, 1, 1, 0, 0, 0)

with zipfile.ZipFile(str(whl)) as z:
assert (
z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time
)

assert (
"SOURCE_DATE_EPOCH environment variable not set,"
f" setting zipinfo date to default={default_date_time}"
) in caplog.messages


def test_dist_info_date_time_value_from_envvar(monkeypatch: MonkeyPatch) -> None:
monkeypatch.setenv("SOURCE_DATE_EPOCH", "1727883000")
expected_date_time = (2024, 10, 2, 15, 30, 0)
module_path = fixtures_dir / "complete"
WheelBuilder.make(Factory().create_poetry(module_path))

whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl"

with zipfile.ZipFile(str(whl)) as z:
assert (
z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time
== expected_date_time
)


def test_dist_info_date_time_value_from_envvar_not_int(
monkeypatch: MonkeyPatch, caplog: LogCaptureFixture
) -> None:
monkeypatch.setenv("SOURCE_DATE_EPOCH", "october")
module_path = fixtures_dir / "complete"
WheelBuilder.make(Factory().create_poetry(module_path))

whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl"

default_date_time = (2016, 1, 1, 0, 0, 0)

with zipfile.ZipFile(str(whl)) as z:
assert (
z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time
)

assert (
"SOURCE_DATE_EPOCH environment variable value"
f" is not an int, setting zipinfo date to default={default_date_time}"
) in caplog.messages


def test_dist_info_date_time_value_from_envvar_older_than_1980(
monkeypatch: MonkeyPatch, caplog: LogCaptureFixture
) -> None:
monkeypatch.setenv("SOURCE_DATE_EPOCH", "1000")
module_path = fixtures_dir / "complete"
WheelBuilder.make(Factory().create_poetry(module_path))

whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl"

default_date_time = (2016, 1, 1, 0, 0, 0)

with zipfile.ZipFile(str(whl)) as z:
assert (
z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time
)

assert (
"zipinfo date can't be earlier than 1980,"
f" setting zipinfo date to default={default_date_time}"
) in caplog.messages
Loading