From 77577f8d55497d497c57e02238273c47c3f3fd68 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 25 Oct 2018 00:59:10 -0700 Subject: [PATCH 01/12] Handle statically linked case in macOS as well It turned out macOS can have statically linked Python when it's installed via conda: https://github.com/JuliaPy/pyjulia/issues/150#issuecomment-432912833 So it seems `linked_libpython` (which calls `libdl`) is the only way to reliably detect if the Python executable is statically linked or not. Since cd2e4089c06c0945bbc4873f8dac6edf6b9cea44 implements it for Windows as well, we can now rely on `linked_libpython` everywhere which simplifies core.py. --- julia/core.py | 20 ++------------------ test/test_find_libpython.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/julia/core.py b/julia/core.py index 2f946910..9a9fffe8 100644 --- a/julia/core.py +++ b/julia/core.py @@ -39,7 +39,7 @@ # this is python 3.3 specific from types import ModuleType, FunctionType -from .find_libpython import find_libpython, normalize_path +from .find_libpython import find_libpython, linked_libpython, normalize_path #----------------------------------------------------------------------------- # Classes and funtions @@ -260,11 +260,7 @@ def isafunction(julia, julia_name, mod_name=""): def determine_if_statically_linked(): """Determines if this python executable is statically linked""" - # Windows and OS X are generally always dynamically linked - if not sys.platform.startswith('linux'): - return False - lddoutput = subprocess.check_output(["ldd",sys.executable]) - return not (b"libpython" in lddoutput) + return linked_libpython() is None JuliaInfo = namedtuple( @@ -356,18 +352,6 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _debug("libpython cannot be read from PyCall/deps/deps.jl") return False - if determine_if_statically_linked(): - _debug(sys.executable, "is statically linked.") - return False - - # Note that the following check is OK since statically linked case - # is already excluded. - if is_same_path(jlinfo.pyprogramname, sys.executable): - # In macOS and Windows, find_libpython does not work as good - # as in Linux. We add this shortcut so that PyJulia can work - # in those environments. - return True - py_libpython = find_libpython() jl_libpython = normalize_path(jlinfo.libpython) _debug("py_libpython =", py_libpython) diff --git a/test/test_find_libpython.py b/test/test_find_libpython.py index 31e30d53..910c4cce 100644 --- a/test/test_find_libpython.py +++ b/test/test_find_libpython.py @@ -1,5 +1,7 @@ +import subprocess +import sys + from julia.find_libpython import finding_libpython, linked_libpython -from julia.core import determine_if_statically_linked try: unicode @@ -14,6 +16,22 @@ def test_finding_libpython_yield_type(): # let's just check returned type of finding_libpython. +def determine_if_statically_linked(): + """Determines if this python executable is statically linked""" + if not sys.platform.startswith('linux'): + # Assuming that Windows and OS X are generally always + # dynamically linked. Note that this is not the case in + # Python installed via conda: + # https://github.com/JuliaPy/pyjulia/issues/150#issuecomment-432912833 + # However, since we do not use conda in our CI, this function + # is OK to use in tests. + return False + lddoutput = subprocess.check_output(["ldd", sys.executable]) + return not (b"libpython" in lddoutput) + + def test_linked_libpython(): + # TODO: Special-case conda (check `sys.version`). See the above + # comments in `determine_if_statically_linked. if not determine_if_statically_linked(): assert linked_libpython() is not None From bacf5e5f2a9851cb6666f0bdfb93f8431cf8a593 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 25 Oct 2018 16:09:49 -0700 Subject: [PATCH 02/12] Remove confusing python-jl --help sentence --- julia/python_jl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/julia/python_jl.py b/julia/python_jl.py index d637eb26..5f4aad5a 100644 --- a/julia/python_jl.py +++ b/julia/python_jl.py @@ -7,9 +7,7 @@ Deiban-based distribution such as Ubuntu and Python executable installed by Conda in Linux. -In Windows and macOS, this CLI is not necessary because those platforms do -not have the pre-compilation issue mentioned above. In fact, this CLI is -known to not work on Windows at the moment. +.. WARNING:: This CLI does not work on Windows. Although this script has -i option and it can do a basic REPL, contrl-c may crash the whole process. Consider using IPython >= 7 which can be launched From 2e12d24a1554868d9e41dd5e509238ebe092c164 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 25 Oct 2018 23:32:05 -0700 Subject: [PATCH 03/12] Fix is_compatible_exe; use linked_libpython, not find_libpython Since `if determine_if_statically_linked()` block was removed, we need additional check for statically linked executable. Furthermore, using `find_libpython()` here was not appropriate since `py_libpython` should be the libpython _used_ by the current executable. --- julia/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/julia/core.py b/julia/core.py index 9a9fffe8..d45b7ede 100644 --- a/julia/core.py +++ b/julia/core.py @@ -352,11 +352,11 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _debug("libpython cannot be read from PyCall/deps/deps.jl") return False - py_libpython = find_libpython() + py_libpython = linked_libpython() jl_libpython = normalize_path(jlinfo.libpython) _debug("py_libpython =", py_libpython) _debug("jl_libpython =", jl_libpython) - return py_libpython == jl_libpython + return py_libpython == jl_libpython and py_libpython is not None _separate_cache_error_common_header = """\ From 7217d0b8ca83ccbd6913b438d691e47bb77a0e73 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 26 Oct 2018 00:09:34 -0700 Subject: [PATCH 04/12] Fix typos and improve docstring --- julia/find_libpython.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index c5156179..7d88d1cd 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -263,8 +263,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): Parameters ---------- - path : str ot None + path : str or None A candidate path to a shared library. + + Returns + ------- + path : str or None + Normalized existing path or `None`. """ if not path: return None From f1036a6f55af5d20231639db33aa07f79208106b Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 26 Oct 2018 00:10:07 -0700 Subject: [PATCH 05/12] Further simplify is_compatible_exe Since `normalize_path(None)` is `None`, we don't need the explicit block `if jlinfo.libpython is None`. --- julia/core.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/julia/core.py b/julia/core.py index d45b7ede..a63382da 100644 --- a/julia/core.py +++ b/julia/core.py @@ -348,10 +348,6 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): A `JuliaInfo` object returned by `juliainfo` function. """ _debug("jlinfo.libpython =", jlinfo.libpython) - if jlinfo.libpython is None: - _debug("libpython cannot be read from PyCall/deps/deps.jl") - return False - py_libpython = linked_libpython() jl_libpython = normalize_path(jlinfo.libpython) _debug("py_libpython =", py_libpython) From 65f11f595581c937d2638cdd0282795685216fd0 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 26 Oct 2018 00:17:21 -0700 Subject: [PATCH 06/12] Document/comment is_compatible_exe more --- julia/core.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/julia/core.py b/julia/core.py index a63382da..e2d181b6 100644 --- a/julia/core.py +++ b/julia/core.py @@ -337,10 +337,9 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): Determine if Python used by PyCall.jl is compatible with this Python. Current Python executable is considered compatible if it is dynamically - linked to libpython (usually the case in macOS and Windows) and - both of them are using identical libpython. If this function returns - `True`, PyJulia use the same precompilation cache of PyCall.jl used by - Julia itself. + linked to libpython and both of them are using identical libpython. If + this function returns `True`, PyJulia use the same precompilation cache + of PyCall.jl used by Julia itself. Parameters ---------- @@ -353,6 +352,11 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _debug("py_libpython =", py_libpython) _debug("jl_libpython =", jl_libpython) return py_libpython == jl_libpython and py_libpython is not None + # `py_libpython is not None` here for checking if this Python + # executable is dynamically linked or not (`py_libpython is None` + # if it's statically linked). `jl_libpython` may be `None` if + # libpython used for PyCall is removed so we can't expect + # `jl_libpython` to be a `str` always. _separate_cache_error_common_header = """\ From 056a52a960a707058db7b99a57824b5829292b15 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 26 Oct 2018 20:43:46 -0700 Subject: [PATCH 07/12] Test invoking PyJulia with incompatible Python executable --- .travis.yml | 12 ++++ test/test_compatible_exe.py | 120 ++++++++++++++++++++++++++++++++++++ tox.ini | 3 + 3 files changed, 135 insertions(+) create mode 100644 test/test_compatible_exe.py diff --git a/.travis.yml b/.travis.yml index eca32f5c..9f49da0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - JULIA_VERSION=nightly global: - TOXENV=py + - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=/usr/bin/python2 matrix: # Python environment is not functional on OS X include: @@ -29,12 +30,14 @@ matrix: - language: generic env: - PYTHON=python3 + - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=python2 - JULIA_VERSION=1.0 # - JULIA_VERSION=nightly os: osx - language: generic env: - PYTHON=python3 + - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=python2 - JULIA_VERSION=0.6.4 - CROSS_VERSION=1 os: osx @@ -59,6 +62,15 @@ before_script: - julia --color=yes -e 'VERSION >= v"0.7.0-DEV.5183" && using Pkg; Pkg.add("PyCall")' script: + # Point PYJULIA_TEST_INCOMPATIBLE_PYTHONS to incompatible Python + # executable (see: test/test_compatible_exe.py). + - if [ "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS" = "$PYTHON" ]; then + PYJULIA_TEST_INCOMPATIBLE_PYTHONS=""; + elif ! which "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS"; then + PYJULIA_TEST_INCOMPATIBLE_PYTHONS=""; + fi + - echo "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS" + # "py,py27" below would be redundant when the main interpreter is # Python 2.7 but it simplifies the CI setup. - if [ "$CROSS_VERSION" = "1" ]; then diff --git a/test/test_compatible_exe.py b/test/test_compatible_exe.py new file mode 100644 index 00000000..9a34d7d6 --- /dev/null +++ b/test/test_compatible_exe.py @@ -0,0 +1,120 @@ +from __future__ import print_function + +import os +import subprocess +import sys +import textwrap + +import pytest + +from .test_core import julia +from julia.core import _enviorn, which + +is_linux = sys.platform.startswith("linux") +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + + +def _get_paths(path): + return filter(None, path.split(":")) + + +# Environment variable PYJULIA_TEST_INCOMPATIBLE_PYTHONS is the +# :-separated list of Python executables incompatible with the current +# Python: +incompatible_pythons = _get_paths(os.getenv("PYJULIA_TEST_INCOMPATIBLE_PYTHONS", "")) + + +try: + from types import SimpleNamespace +except ImportError: + from argparse import Namespace as SimpleNamespace + + +def _run_fallback(args, input=None, **kwargs): + process = subprocess.Popen(args, stdin=subprocess.PIPE, **kwargs) + stdout, stderr = process.communicate(input) + retcode = process.wait() + return SimpleNamespace(args=args, stdout=stdout, stderr=stderr, returncode=retcode) + + +try: + from subprocess import run +except ImportError: + run = _run_fallback + + +def runcode(python, code): + """Run `code` in `python`.""" + return run( + [python], + input=textwrap.dedent(code), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=dict( + _enviorn, + # Make PyJulia importable: + PYTHONPATH=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + ), + ) + + +def print_completed_proc(proc): + # Print output (pytest will hide it by default): + print("Ran:", *proc.args) + if proc.stdout: + print("# --- STDOUT from", *proc.args) + print(proc.stdout) + if proc.stderr: + print("# --- STDERR from", *proc.args) + print(proc.stderr) + print("# ---") + + +def is_dynamically_linked(executable): + path = which(executable) + assert os.path.exists(path) + if is_linux and which("ldd"): + proc = run( + ["ldd", path], stdout=subprocess.PIPE, env=_enviorn, universal_newlines=True + ) + print_completed_proc(proc) + return "libpython" in proc.stdout + elif is_apple and which("otool"): + proc = run( + ["otool", "-L", path], + stdout=subprocess.PIPE, + env=_enviorn, + universal_newlines=True, + ) + print_completed_proc(proc) + return "libpython" in proc.stdout or "/Python" in proc.stdout + # TODO: support Windows + return None + + +@pytest.mark.parametrize("python", incompatible_pythons) +def test_incompatible_python(python): + if julia.eval("(VERSION.major, VERSION.minor)") == (0, 6): + # Julia 0.6 implements mixed version + return + + python = which(python) + proc = runcode( + python, + """ + import os + from julia import Julia + Julia(runtime=os.getenv("JULIA_EXE"), debug=True) + """, + ) + print_completed_proc(proc) + + assert proc.returncode == 1 + assert "It seems your Julia and PyJulia setup are not supported." in proc.stderr + dynamic = is_dynamically_linked(python) + if dynamic is True: + assert "`libpython` have to match" in proc.stderr + elif dynamic is False: + assert "is statically linked to libpython" in proc.stderr diff --git a/tox.ini b/tox.ini index 2713953e..c3421a47 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,9 @@ passenv = PYJULIA_TEST_REBUILD JULIA_EXE + # See: test/test_compatible_exe.py + PYJULIA_TEST_INCOMPATIBLE_PYTHONS + # See: https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci TRAVIS TRAVIS_* From 2338a825426c5ed9deaf112faf3ecac62aa52ce1 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 27 Oct 2018 21:30:35 -0700 Subject: [PATCH 08/12] Refactor is_compatible_exe --- julia/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/julia/core.py b/julia/core.py index e2d181b6..bea3d607 100644 --- a/julia/core.py +++ b/julia/core.py @@ -351,7 +351,8 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): jl_libpython = normalize_path(jlinfo.libpython) _debug("py_libpython =", py_libpython) _debug("jl_libpython =", jl_libpython) - return py_libpython == jl_libpython and py_libpython is not None + dynamically_linked = py_libpython is not None + return dynamically_linked and py_libpython == jl_libpython # `py_libpython is not None` here for checking if this Python # executable is dynamically linked or not (`py_libpython is None` # if it's statically linked). `jl_libpython` may be `None` if From 32c5d933dfe7ebfc1c0cabc8b52127ba1255cf02 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 27 Oct 2018 21:52:52 -0700 Subject: [PATCH 09/12] More documentation in test_compatible_exe.py --- test/test_compatible_exe.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/test_compatible_exe.py b/test/test_compatible_exe.py index 9a34d7d6..34fab194 100644 --- a/test/test_compatible_exe.py +++ b/test/test_compatible_exe.py @@ -28,10 +28,12 @@ def _get_paths(path): try: from types import SimpleNamespace except ImportError: + # Python 2: from argparse import Namespace as SimpleNamespace def _run_fallback(args, input=None, **kwargs): + # A port of subprocess.run just enough to run the tests. process = subprocess.Popen(args, stdin=subprocess.PIPE, **kwargs) stdout, stderr = process.communicate(input) retcode = process.wait() @@ -73,6 +75,20 @@ def print_completed_proc(proc): def is_dynamically_linked(executable): + """ + Check if Python `executable` is (likely to be) dynamically linked. + + It returns three possible values: + + * `True`: Likely that it's dynamically linked. + * `False`: Likely that it's statically linked. + * `None`: Unsupported platform. + + It's only "likely" since the check is by simple occurrence of a + some substrings like "libpython". For example, if there is + another library existing on the path containing "libpython", this + function may return false-positive. + """ path = which(executable) assert os.path.exists(path) if is_linux and which("ldd"): From 44ae919682299fad7429d7e1d4263c8d912cdc63 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 27 Oct 2018 21:56:18 -0700 Subject: [PATCH 10/12] Simulate the case PyCall is configured with statically linked Python --- test/test_compatible_exe.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/test_compatible_exe.py b/test/test_compatible_exe.py index 34fab194..d8e0f2ac 100644 --- a/test/test_compatible_exe.py +++ b/test/test_compatible_exe.py @@ -16,7 +16,7 @@ def _get_paths(path): - return filter(None, path.split(":")) + return list(filter(None, path.split(":"))) # Environment variable PYJULIA_TEST_INCOMPATIBLE_PYTHONS is the @@ -134,3 +134,37 @@ def test_incompatible_python(python): assert "`libpython` have to match" in proc.stderr elif dynamic is False: assert "is statically linked to libpython" in proc.stderr + + +@pytest.mark.parametrize( + "python", + [ + p + for p in filter(None, map(which, incompatible_pythons)) + if is_dynamically_linked(p) is False + ], +) +def test_statically_linked(python): + """ + Simulate the case PyCall is configured with statically linked Python. + + In this case, `find_libpython()` would return the path identical + to the one in PyCall's deps.jl. `is_compatible_exe` should reject + it. + """ + python = which(python) + proc = runcode( + python, + """ + from __future__ import print_function + from julia.find_libpython import find_libpython + from julia.core import is_compatible_exe + + class jlinfo: + libpython = find_libpython() + + assert not is_compatible_exe(jlinfo, _debug=print) + """, + ) + print_completed_proc(proc) + assert proc.returncode == 0 From 5a7cc3c32972ccad300b19bda204ca19d6f95d6a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 27 Oct 2018 22:04:24 -0700 Subject: [PATCH 11/12] Use -v instead of -s in CI No-capturing mode -s is useful for remote debugging especially in case libjulia initialization fails. But it makes reading output almost impossible. Let's use -v instead since it provides what test is being executed in a more readable format. --- .travis.yml | 4 ++-- appveyor.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f49da0a..c2ba68b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,10 +74,10 @@ script: # "py,py27" below would be redundant when the main interpreter is # Python 2.7 but it simplifies the CI setup. - if [ "$CROSS_VERSION" = "1" ]; then - $PYTHON -m tox -e py,py27 -- -s; + $PYTHON -m tox -e py,py27 -- -v; fi - - PYJULIA_TEST_REBUILD=yes $PYTHON -m tox -- --cov=julia -s + - PYJULIA_TEST_REBUILD=yes $PYTHON -m tox -- --cov=julia -v after_success: - coveralls diff --git a/appveyor.yml b/appveyor.yml index 11d057f5..ee850d6f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,9 +91,9 @@ test_script: # Run cross-version tests but ignore the failures (from Python 2). # Once cross-version in Windows is fmixed, stop using # Invoke-Expression (which ignores the exit status). - - ps: if ($env:CROSS_VERSION -eq 1) { Invoke-Expression "tox -- -s" } - # - ps: if ($env:CROSS_VERSION -eq 1) { tox -- -s } + - ps: if ($env:CROSS_VERSION -eq 1) { Invoke-Expression "tox -- -v" } + # - ps: if ($env:CROSS_VERSION -eq 1) { tox -- -v } # Rebuild PyCall.ji for each Python interpreter before testing: - "SET PYJULIA_TEST_REBUILD=yes" - - tox -- -s + - tox -- -v From a387a3a3a65007b29d6b15249de282a1ec15dd9d Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 27 Oct 2018 22:18:37 -0700 Subject: [PATCH 12/12] Fix is_dynamically_linked for macOS See: https://travis-ci.org/JuliaPy/pyjulia/jobs/447296193#L1650 --- test/test_compatible_exe.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_compatible_exe.py b/test/test_compatible_exe.py index d8e0f2ac..0bd3f806 100644 --- a/test/test_compatible_exe.py +++ b/test/test_compatible_exe.py @@ -105,7 +105,11 @@ def is_dynamically_linked(executable): universal_newlines=True, ) print_completed_proc(proc) - return "libpython" in proc.stdout or "/Python" in proc.stdout + return ( + "libpython" in proc.stdout + or "/Python" in proc.stdout + or "/.Python" in proc.stdout + ) # TODO: support Windows return None