diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 58e57532d..7f1194760 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -34,6 +34,7 @@ from pex.interpreter_constraints import InterpreterConstraints from pex.layout import Layout, ensure_installed from pex.orderedset import OrderedSet +from pex.os import WINDOWS from pex.pep_427 import InstallableType from pex.pep_723 import ScriptMetadata from pex.pex import PEX @@ -558,7 +559,7 @@ def __call__(self, parser, namespace, value, option_str=None): class InjectArgAction(Action): def __call__(self, parser, namespace, value, option_str=None): - self.default.extend(shlex.split(value)) + self.default.extend(shlex.split(value, posix=not WINDOWS)) group.add_argument( "--inject-python-args", diff --git a/pex/distutils/commands/bdist_pex.py b/pex/distutils/commands/bdist_pex.py index 7a2d9dbe8..3ce76c0ec 100644 --- a/pex/distutils/commands/bdist_pex.py +++ b/pex/distutils/commands/bdist_pex.py @@ -15,6 +15,7 @@ from pex.common import die from pex.compatibility import ConfigParser, string, to_unicode from pex.interpreter import PythonInterpreter +from pex.os import WINDOWS # Suppress checkstyle violations due to distutils command requirements. @@ -45,7 +46,7 @@ def initialize_options(self): self.pex_args = "" def finalize_options(self): - self.pex_args = shlex.split(self.pex_args) + self.pex_args = shlex.split(self.pex_args, posix=not WINDOWS) def parse_entry_points(self): def parse_entry_point_name(entry_point): diff --git a/pex/interpreter.py b/pex/interpreter.py index 231ba070d..b6d770c89 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -795,35 +795,33 @@ def include_system_site_packages(self): class PythonInterpreter(object): - _REGEXEN = ( - # NB: OSX ships python binaries named Python with a capital-P; so we allow for this. - re.compile(r"^Python{extension}$".format(extension=re.escape(EXE_EXTENSION))), - re.compile( - r""" - ^ - (?: - python | - pypy - ) - (?: - # Major version - [2-9] - (?:. - # Minor version - [0-9]+ - # Some distributions include a suffix on the interpreter name, similar to - # PEP-3149. For example, Gentoo has /usr/bin/python3.6m to indicate it was - # built with pymalloc - [a-z]? - )? + _REGEX = re.compile( + r""" + ^ + (?: + python | + pypy + ) + (?: + # Major version + [2-9] + (?:. + # Minor version + [0-9]+ + # Some distributions include a suffix on the interpreter name, similar to + # PEP-3149. For example, Gentoo has /usr/bin/python3.6m to indicate it was + # built with pymalloc + [a-z]? )? - {extension} - $ - """.format( - extension=re.escape(EXE_EXTENSION) - ), - flags=re.VERBOSE, + )? + {extension} + $ + """.format( + extension=re.escape(EXE_EXTENSION) ), + # NB: OSX ships python binaries named Python with a capital-P; so we allow for this as well + # as accommodating Windows which has DOS case insensitivity. + flags=re.IGNORECASE | re.VERBOSE, ) _PYTHON_INTERPRETER_BY_NORMALIZED_PATH = {} # type: Dict @@ -1214,8 +1212,7 @@ def from_binary( @classmethod def matches_binary_name(cls, path): # type: (str) -> bool - basefile = os.path.basename(path) - return any(matcher.match(basefile) is not None for matcher in cls._REGEXEN) + return cls._REGEX.match(os.path.basename(path)) is not None @overload @classmethod diff --git a/pex/sh_boot.py b/pex/sh_boot.py index 98b01a51d..345849c39 100644 --- a/pex/sh_boot.py +++ b/pex/sh_boot.py @@ -15,6 +15,7 @@ from pex.interpreter_constraints import InterpreterConstraints, iter_compatible_versions from pex.layout import Layout from pex.orderedset import OrderedSet +from pex.os import WINDOWS from pex.pep_440 import Version from pex.pex_info import PexInfo from pex.targets import Targets @@ -152,7 +153,8 @@ def create_sh_boot_script( # Drop leading `/usr/bin/env [args]?`. args = list( itertools.dropwhile( - lambda word: not PythonInterpreter.matches_binary_name(word), shlex.split(shebang) + lambda word: not PythonInterpreter.matches_binary_name(word), + shlex.split(shebang, posix=not WINDOWS), ) ) python = args[0] diff --git a/tests/integration/test_inject_python_args.py b/tests/integration/test_inject_python_args.py index 3679c9f88..cf526c239 100644 --- a/tests/integration/test_inject_python_args.py +++ b/tests/integration/test_inject_python_args.py @@ -3,12 +3,15 @@ import json import os.path +import shlex import shutil import sys from textwrap import dedent +from typing import Optional import pytest +from pex.os import WINDOWS from pex.typing import TYPE_CHECKING from testing import IS_PYPY, run_pex_command, subprocess @@ -58,12 +61,17 @@ def assert_exe_output( pex, # type: str warning_expected, # type: bool prefix_args=(), # type: Iterable[str] + custom_shebang=None, # type: Optional[str] ): - process = subprocess.Popen( - args=list(prefix_args) + [pex, "--foo", "bar"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + args = [] # type: List[str] + if prefix_args: + args.extend(prefix_args) + elif WINDOWS and custom_shebang: + # Windows doesn't do shebangs; so instead test a direct invocation with the shebang. + args.extend(shlex.split(custom_shebang, posix=False)) + args.extend((pex, "--foo", "bar")) + + process = subprocess.Popen(args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() error = stderr.decode("utf-8") assert 0 == process.returncode, error @@ -107,14 +115,9 @@ def test_python_args_passthrough( ] ) run_pex_command(args=args + ["-o", default_shebang_pex]).assert_success() + custom_shebang = "{python} -Wignore".format(python=sys.executable) run_pex_command( - args=args - + [ - "-o", - custom_shebang_pex, - "--python-shebang", - "{python} -Wignore".format(python=sys.executable), - ] + args=args + ["-o", custom_shebang_pex, "--python-shebang", custom_shebang] ).assert_success() # N.B.: We execute tests in doubles after a cache nuke to exercise both cold and warm runs @@ -124,8 +127,8 @@ def test_python_args_passthrough( shutil.rmtree(pex_root) assert_exe_output(default_shebang_pex, warning_expected=True) assert_exe_output(default_shebang_pex, warning_expected=True) - assert_exe_output(custom_shebang_pex, warning_expected=False) - assert_exe_output(custom_shebang_pex, warning_expected=False) + assert_exe_output(custom_shebang_pex, warning_expected=False, custom_shebang=custom_shebang) + assert_exe_output(custom_shebang_pex, warning_expected=False, custom_shebang=custom_shebang) # But they also should be able to be over-ridden. shutil.rmtree(pex_root) @@ -135,8 +138,18 @@ def test_python_args_passthrough( assert_exe_output( default_shebang_pex, prefix_args=[sys.executable, "-Wignore"], warning_expected=False ) - assert_exe_output(custom_shebang_pex, prefix_args=[sys.executable], warning_expected=True) - assert_exe_output(custom_shebang_pex, prefix_args=[sys.executable], warning_expected=True) + assert_exe_output( + custom_shebang_pex, + prefix_args=[sys.executable], + warning_expected=True, + custom_shebang=custom_shebang, + ) + assert_exe_output( + custom_shebang_pex, + prefix_args=[sys.executable], + warning_expected=True, + custom_shebang=custom_shebang, + ) @parametrize_execution_mode_args @@ -227,26 +240,26 @@ def test_issue_2422( run_pex_command(args=args).assert_success() shutil.rmtree(pex_root) - assert b"BufferedWriter\n" == subprocess.check_output(args=[sys.executable, exe]) - assert b"BufferedWriter\n" == subprocess.check_output( - args=[pex] + assert b"BufferedWriter" == subprocess.check_output(args=[sys.executable, exe]).rstrip() + assert ( + b"BufferedWriter" == subprocess.check_output(args=[pex]).rstrip() ), "Expected cold run to use buffered io." - assert b"BufferedWriter\n" == subprocess.check_output( - args=[pex] + assert ( + b"BufferedWriter" == subprocess.check_output(args=[pex]).rstrip() ), "Expected warm run to use buffered io." - assert b"FileIO\n" == subprocess.check_output( - args=[sys.executable, "-u", pex] + assert ( + b"FileIO" == subprocess.check_output(args=[sys.executable, "-u", pex]).rstrip() ), "Expected explicit Python arguments to be honored." run_pex_command(args=args + ["--inject-python-args=-u"]).assert_success() shutil.rmtree(pex_root) - assert b"FileIO\n" == subprocess.check_output(args=[sys.executable, "-u", exe]) - assert b"FileIO\n" == subprocess.check_output( - args=[pex] + assert b"FileIO" == subprocess.check_output(args=[sys.executable, "-u", exe]).rstrip() + assert ( + b"FileIO" == subprocess.check_output(args=[pex]).rstrip() ), "Expected cold run to use un-buffered io." - assert b"FileIO\n" == subprocess.check_output( - args=[pex] + assert ( + b"FileIO" == subprocess.check_output(args=[pex]).rstrip() ), "Expected warm run to use un-buffered io." process = subprocess.Popen( @@ -256,5 +269,5 @@ def test_issue_2422( ) stdout, stderr = process.communicate() assert 0 == process.returncode - assert b"FileIO\n" == stdout, "Expected injected Python arguments to be honored." + assert b"FileIO" == stdout.rstrip(), "Expected injected Python arguments to be honored." assert b"import " in stderr, "Expected explicit Python arguments to be honored as well."