From e74d3ef273970a3b70d9b8640396be4deecc4634 Mon Sep 17 00:00:00 2001 From: Bartosz Sokorski Date: Wed, 2 Oct 2024 23:25:40 +0200 Subject: [PATCH] Add SOURCE_DATE_EPOCH support when building packages --- src/poetry/core/masonry/builders/sdist.py | 41 ++++++++---- src/poetry/core/masonry/builders/wheel.py | 44 ++++++++++-- tests/masonry/builders/test_sdist.py | 69 ++++++++++++++++--- tests/masonry/builders/test_wheel.py | 82 +++++++++++++++++++++++ 4 files changed, 206 insertions(+), 30 deletions(-) diff --git a/src/poetry/core/masonry/builders/sdist.py b/src/poetry/core/masonry/builders/sdist.py index a22ce9574..8b186fbaf 100644 --- a/src/poetry/core/masonry/builders/sdist.py +++ b/src/poetry/core/masonry/builders/sdist.py @@ -68,7 +68,7 @@ def build( name = distribution_name(self._package.name) target = target_dir / f"{name}-{self._meta.version}.tar.gz" - gz = GzipFile(target.as_posix(), mode="wb", mtime=0) + gz = GzipFile(target.as_posix(), mode="wb", mtime=self._get_archive_mtime()) tar = tarfile.TarFile( target.as_posix(), mode="w", fileobj=gz, format=tarfile.PAX_FORMAT ) @@ -93,19 +93,10 @@ def build( if self._poetry.package.build_should_generate_setup(): setup = self.build_setup() - tar_info = tarfile.TarInfo(pjoin(tar_dir, "setup.py")) - tar_info.size = len(setup) - tar_info.mtime = 0 - tar_info = self.clean_tarinfo(tar_info) - tar.addfile(tar_info, BytesIO(setup)) + self.add_file_to_tar(tar, pjoin(tar_dir, "setup.py"), setup) pkg_info = self.build_pkg_info() - - tar_info = tarfile.TarInfo(pjoin(tar_dir, "PKG-INFO")) - tar_info.size = len(pkg_info) - tar_info.mtime = 0 - tar_info = self.clean_tarinfo(tar_info) - tar.addfile(tar_info, BytesIO(pkg_info)) + self.add_file_to_tar(tar, pjoin(tar_dir, "PKG-INFO"), pkg_info) finally: tar.close() gz.close() @@ -113,6 +104,14 @@ def build( logger.info(f"Built {target.name}") return target + def add_file_to_tar( + self, tar: tarfile.TarFile, file_name: str, content: bytes + ) -> None: + tar_info = tarfile.TarInfo(file_name) + tar_info.size = len(content) + tar_info = self.clean_tarinfo(tar_info) + tar.addfile(tar_info, BytesIO(content)) + def build_setup(self) -> bytes: from poetry.core.masonry.utils.package_include import PackageInclude @@ -399,8 +398,7 @@ def convert_dependencies( return main, dict(extras) - @classmethod - def clean_tarinfo(cls, tar_info: TarInfo) -> TarInfo: + def clean_tarinfo(self, tar_info: TarInfo) -> TarInfo: """ Clean metadata from a TarInfo object to make it more reproducible. @@ -416,6 +414,21 @@ def clean_tarinfo(cls, tar_info: TarInfo) -> TarInfo: ti.gid = 0 ti.uname = "" ti.gname = "" + ti.mtime = self._get_archive_mtime() ti.mode = normalize_file_permissions(ti.mode) return ti + + @staticmethod + def _get_archive_mtime() -> int: + if source_date_epoch := os.getenv("SOURCE_DATE_EPOCH"): + try: + return int(source_date_epoch) + except ValueError: + logger.warning( + "SOURCE_DATE_EPOCH environment variable is not an int," + " using mtime=0" + ) + return 0 + logger.info("SOURCE_DATE_EPOCH environment variable is not set, using mtime=0") + return 0 diff --git a/src/poetry/core/masonry/builders/wheel.py b/src/poetry/core/masonry/builders/wheel.py index bc74d50ff..ae58c8f7d 100644 --- a/src/poetry/core/masonry/builders/wheel.py +++ b/src/poetry/core/masonry/builders/wheel.py @@ -33,11 +33,14 @@ if TYPE_CHECKING: from collections.abc import Iterator + from typing import Tuple from packaging.utils import NormalizedName from poetry.core.poetry import Poetry + ZipInfoTimestamp = Tuple[int, int, int, int, int, int] + wheel_file_template = """\ Wheel-Version: 1.0 Generator: poetry-core {version} @@ -446,7 +449,7 @@ def _add_file( ) -> None: # We always want to have /-separated paths in the zip file and in RECORD rel_path_name = rel_path.as_posix() - zinfo = zipfile.ZipInfo(rel_path_name) + zinfo = zipfile.ZipInfo(rel_path_name, self._get_zipfile_date_time()) # Normalize permission bits to either 755 (executable) or 644 st_mode = full_path.stat().st_mode @@ -479,10 +482,7 @@ def _write_to_zip( sio = StringIO() yield sio - # The default is a fixed timestamp rather than the current time, so - # that building a wheel twice on the same computer can automatically - # give you the exact same result. - date_time = (2016, 1, 1, 0, 0, 0) + date_time = self._get_zipfile_date_time() zi = zipfile.ZipInfo(rel_path, date_time) zi.external_attr = (0o644 & 0xFFFF) << 16 # Unix attributes b = sio.getvalue().encode("utf-8") @@ -492,6 +492,40 @@ def _write_to_zip( wheel.writestr(zi, b, compress_type=zipfile.ZIP_DEFLATED) self._records.append((rel_path, hash_digest, len(b))) + @staticmethod + def _get_zipfile_date_time() -> ZipInfoTimestamp: + import time + + # The default is a fixed timestamp rather than the current time, so + # that building a wheel twice on the same computer can automatically + # give you the exact same result. + default = (2016, 1, 1, 0, 0, 0) + try: + _env_date = time.gmtime(int(os.environ["SOURCE_DATE_EPOCH"]))[:6] + except ValueError: + logger.warning( + "SOURCE_DATE_EPOCH environment variable value" + " is not an int, setting zipinfo date to default=%s", + default, + ) + return default + except KeyError: + logger.info( + "SOURCE_DATE_EPOCH environment variable not set," + " setting zipinfo date to default=%s", + default, + ) + return default + else: + if _env_date[0] < 1980: + logger.warning( + "zipinfo date can't be earlier than 1980," + " setting zipinfo date to default=%s", + default, + ) + return default + return _env_date + def _write_entry_points(self, fp: TextIO) -> None: """ Write entry_points.txt. diff --git a/tests/masonry/builders/test_sdist.py b/tests/masonry/builders/test_sdist.py index 73ff4b722..df2df2434 100644 --- a/tests/masonry/builders/test_sdist.py +++ b/tests/masonry/builders/test_sdist.py @@ -25,6 +25,8 @@ if TYPE_CHECKING: + from pytest import LogCaptureFixture + from pytest import MonkeyPatch from pytest_mock import MockerFixture fixtures_dir = Path(__file__).parent / "fixtures" @@ -496,7 +498,6 @@ def test_default_with_excluded_data(mocker: MockerFixture) -> None: ) assert sdist.exists() - with tarfile.open(str(sdist), "r") as tar: names = tar.getnames() assert len(names) == len(set(names)) @@ -506,16 +507,6 @@ def test_default_with_excluded_data(mocker: MockerFixture) -> None: assert "my_package-1.2.3/my_package/data/data1.txt" in names assert "my_package-1.2.3/pyproject.toml" in names assert "my_package-1.2.3/PKG-INFO" in names - # all last modified times should be set to a valid timestamp - for tarinfo in tar.getmembers(): - if tarinfo.name in [ - "my_package-1.2.3/setup.py", - "my_package-1.2.3/PKG-INFO", - ]: - # generated files have timestamp set to 0 - assert tarinfo.mtime == 0 - continue - assert tarinfo.mtime > 0 def test_src_excluded_nested_data() -> None: @@ -693,3 +684,59 @@ def test_split_source() -> None: ns: dict[str, Any] = {} exec(compile(setup_ast, filename="setup.py", mode="exec"), ns) assert "" in ns["package_dir"] and "module_b" in ns["package_dir"] + + +def test_sdist_members_mtime_default(caplog: LogCaptureFixture) -> None: + import logging + + caplog.set_level(logging.INFO) + poetry = Factory().create_poetry(project("module1")) + + builder = SdistBuilder(poetry) + builder.build() + + sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz" + + assert sdist.exists() + + with tarfile.open(str(sdist), "r") as tar: + for tarinfo in tar.getmembers(): + assert tarinfo.mtime == 0 + + assert ( + "SOURCE_DATE_EPOCH environment variable is not set," " using mtime=0" + ) in caplog.messages + + +def test_sdist_mtime_set_from_envvar(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1727883000") + poetry = Factory().create_poetry(project("module1")) + + builder = SdistBuilder(poetry) + builder.build() + + sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz" + + assert sdist.exists() + + with tarfile.open(str(sdist), "r") as tar: + for tarinfo in tar.getmembers(): + assert tarinfo.mtime == 1727883000 + + +def test_sdist_mtime_set_from_envvar_not_int( + monkeypatch: MonkeyPatch, caplog: LogCaptureFixture +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "october") + poetry = Factory().create_poetry(project("module1")) + + builder = SdistBuilder(poetry) + builder.build() + + sdist = fixtures_dir / "module1" / "dist" / "module1-0.1.tar.gz" + + assert sdist.exists() + + assert ( + "SOURCE_DATE_EPOCH environment variable is not an " "int, using mtime=0" + ) in caplog.messages diff --git a/tests/masonry/builders/test_wheel.py b/tests/masonry/builders/test_wheel.py index 6ba0a5934..b837eacf4 100644 --- a/tests/masonry/builders/test_wheel.py +++ b/tests/masonry/builders/test_wheel.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: + from pytest import LogCaptureFixture from pytest import MonkeyPatch from pytest_mock import MockerFixture @@ -469,3 +470,84 @@ def test_generated_script_file(tmp_path: Path) -> None: with zipfile.ZipFile(str(whl)) as z: assert "generated_script_file-0.1.data/scripts/script.sh" in z.namelist() + + +def test_dist_info_date_time_default_value(caplog: LogCaptureFixture) -> None: + import logging + + caplog.set_level(logging.INFO) + module_path = fixtures_dir / "complete" + WheelBuilder.make(Factory().create_poetry(module_path)) + + whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" + + default_date_time = (2016, 1, 1, 0, 0, 0) + + with zipfile.ZipFile(str(whl)) as z: + assert ( + z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time + ) + + assert ( + "SOURCE_DATE_EPOCH environment variable not set," + f" setting zipinfo date to default={default_date_time}" + ) in caplog.messages + + +def test_dist_info_date_time_value_from_envvar(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1727883000") + expected_date_time = (2024, 10, 2, 15, 30, 0) + module_path = fixtures_dir / "complete" + WheelBuilder.make(Factory().create_poetry(module_path)) + + whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" + + with zipfile.ZipFile(str(whl)) as z: + assert ( + z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time + == expected_date_time + ) + + +def test_dist_info_date_time_value_from_envvar_not_int( + monkeypatch: MonkeyPatch, caplog: LogCaptureFixture +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "october") + module_path = fixtures_dir / "complete" + WheelBuilder.make(Factory().create_poetry(module_path)) + + whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" + + default_date_time = (2016, 1, 1, 0, 0, 0) + + with zipfile.ZipFile(str(whl)) as z: + assert ( + z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time + ) + + assert ( + "SOURCE_DATE_EPOCH environment variable value" + f" is not an int, setting zipinfo date to default={default_date_time}" + ) in caplog.messages + + +def test_dist_info_date_time_value_from_envvar_older_than_1980( + monkeypatch: MonkeyPatch, caplog: LogCaptureFixture +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1000") + module_path = fixtures_dir / "complete" + WheelBuilder.make(Factory().create_poetry(module_path)) + + whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl" + + default_date_time = (2016, 1, 1, 0, 0, 0) + + with zipfile.ZipFile(str(whl)) as z: + assert ( + z.getinfo("my_package-1.2.3.dist-info/WHEEL").date_time == default_date_time + ) + + assert ( + "zipinfo date can't be earlier than 1980," + f" setting zipinfo date to default={default_date_time}" + ) in caplog.messages