Handle statically linked case in macOS as well

.travis.yml

@@ -11,6 +11,7 @@ env:
     - JULIA_VERSION=nightly
   global:
     - TOXENV=py
+    - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=/usr/bin/python2
 matrix:
   # Python environment is not functional on OS X
   include:
@@ -29,12 +30,14 @@ matrix:
       - language: generic
         env:
           - PYTHON=python3
+          - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=python2
           - JULIA_VERSION=1.0
           # - JULIA_VERSION=nightly
         os: osx
       - language: generic
         env:
           - PYTHON=python3
+          - PYJULIA_TEST_INCOMPATIBLE_PYTHONS=python2
           - JULIA_VERSION=0.6.4
           - CROSS_VERSION=1
         os: osx
@@ -59,13 +62,22 @@ before_script:
   - julia --color=yes -e 'VERSION >= v"0.7.0-DEV.5183" && using Pkg; Pkg.add("PyCall")'
 script:
 
+  # Point PYJULIA_TEST_INCOMPATIBLE_PYTHONS to incompatible Python
+  # executable (see: test/test_compatible_exe.py).
+  - if [ "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS" = "$PYTHON" ]; then
+      PYJULIA_TEST_INCOMPATIBLE_PYTHONS="";
+    elif ! which "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS"; then
+      PYJULIA_TEST_INCOMPATIBLE_PYTHONS="";
+    fi
+  - echo "$PYJULIA_TEST_INCOMPATIBLE_PYTHONS"
+
   # "py,py27" below would be redundant when the main interpreter is
   # Python 2.7 but it simplifies the CI setup.
   - if [ "$CROSS_VERSION" = "1" ]; then
-    $PYTHON -m tox -e py,py27 -- -s;
+    $PYTHON -m tox -e py,py27 -- -v;
     fi
 
-  - PYJULIA_TEST_REBUILD=yes $PYTHON -m tox -- --cov=julia -s
+  - PYJULIA_TEST_REBUILD=yes $PYTHON -m tox -- --cov=julia -v
 
 after_success:
   - coveralls

appveyor.yml

@@ -91,9 +91,9 @@ test_script:
   # Run cross-version tests but ignore the failures (from Python 2).
   # Once cross-version in Windows is fmixed, stop using
   # Invoke-Expression (which ignores the exit status).
-  - ps: if ($env:CROSS_VERSION -eq 1) { Invoke-Expression "tox -- -s" }
-  # - ps: if ($env:CROSS_VERSION -eq 1) { tox -- -s }
+  - ps: if ($env:CROSS_VERSION -eq 1) { Invoke-Expression "tox -- -v" }
+  # - ps: if ($env:CROSS_VERSION -eq 1) { tox -- -v }
 
   # Rebuild PyCall.ji for each Python interpreter before testing:
   - "SET PYJULIA_TEST_REBUILD=yes"
-  - tox -- -s
+  - tox -- -v

julia/core.py

@@ -39,7 +39,7 @@
 # this is python 3.3 specific
 from types import ModuleType, FunctionType
 
-from .find_libpython import find_libpython, normalize_path
+from .find_libpython import find_libpython, linked_libpython, normalize_path
 
 #-----------------------------------------------------------------------------
 # Classes and funtions
@@ -260,11 +260,7 @@ def isafunction(julia, julia_name, mod_name=""):
 
 def determine_if_statically_linked():
     """Determines if this python executable is statically linked"""
-    # Windows and OS X are generally always dynamically linked
-    if not sys.platform.startswith('linux'):
-        return False
-    lddoutput = subprocess.check_output(["ldd",sys.executable])
-    return not (b"libpython" in lddoutput)
+    return linked_libpython() is None
 
 
 JuliaInfo = namedtuple(
@@ -341,38 +337,27 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None):
     Determine if Python used by PyCall.jl is compatible with this Python.
 
     Current Python executable is considered compatible if it is dynamically
-    linked to libpython (usually the case in macOS and Windows) and
-    both of them are using identical libpython.  If this function returns
-    `True`, PyJulia use the same precompilation cache of PyCall.jl used by
-    Julia itself.
+    linked to libpython and both of them are using identical libpython.  If
+    this function returns `True`, PyJulia use the same precompilation cache
+    of PyCall.jl used by Julia itself.
 
     Parameters
     ----------
     jlinfo : JuliaInfo
         A `JuliaInfo` object returned by `juliainfo` function.
     """
     _debug("jlinfo.libpython =", jlinfo.libpython)
-    if jlinfo.libpython is None:
-        _debug("libpython cannot be read from PyCall/deps/deps.jl")
-        return False
-
-    if determine_if_statically_linked():
-        _debug(sys.executable, "is statically linked.")
-        return False
-
-    # Note that the following check is OK since statically linked case
-    # is already excluded.
-    if is_same_path(jlinfo.pyprogramname, sys.executable):
-        # In macOS and Windows, find_libpython does not work as good
-        # as in Linux.  We add this shortcut so that PyJulia can work
-        # in those environments.
-        return True
-
-    py_libpython = find_libpython()
+    py_libpython = linked_libpython()
     jl_libpython = normalize_path(jlinfo.libpython)
     _debug("py_libpython =", py_libpython)
     _debug("jl_libpython =", jl_libpython)
-    return py_libpython == jl_libpython
+    dynamically_linked = py_libpython is not None
+    return dynamically_linked and py_libpython == jl_libpython
+    # `py_libpython is not None` here for checking if this Python
+    # executable is dynamically linked or not (`py_libpython is None`
+    # if it's statically linked).  `jl_libpython` may be `None` if
+    # libpython used for PyCall is removed so we can't expect
+    # `jl_libpython` to be a `str` always.
 
 
 _separate_cache_error_common_header = """\

julia/find_libpython.py

@@ -263,8 +263,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple):
 
     Parameters
     ----------
-    path : str ot None
+    path : str or None
         A candidate path to a shared library.
+
+    Returns
+    -------
+    path : str or None
+        Normalized existing path or `None`.
     """
     if not path:
         return None

julia/python_jl.py

@@ -7,9 +7,7 @@
 Deiban-based distribution such as Ubuntu and Python executable installed by
 Conda in Linux.
 
-In Windows and macOS, this CLI is not necessary because those platforms do
-not have the pre-compilation issue mentioned above.  In fact, this CLI is
-known to not work on Windows at the moment.
+.. WARNING:: This CLI does not work on Windows.
 
 Although this script has -i option and it can do a basic REPL, contrl-c may
 crash the whole process.  Consider using IPython >= 7 which can be launched

test/test_compatible_exe.py

@@ -0,0 +1,174 @@
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+from .test_core import julia
+from julia.core import _enviorn, which
+
+is_linux = sys.platform.startswith("linux")
+is_windows = os.name == "nt"
+is_apple = sys.platform == "darwin"
+
+
+def _get_paths(path):
+    return list(filter(None, path.split(":")))
+
+
+# Environment variable PYJULIA_TEST_INCOMPATIBLE_PYTHONS is the
+# :-separated list of Python executables incompatible with the current
+# Python:
+incompatible_pythons = _get_paths(os.getenv("PYJULIA_TEST_INCOMPATIBLE_PYTHONS", ""))
+
+
+try:
+    from types import SimpleNamespace
+except ImportError:
+    # Python 2:
+    from argparse import Namespace as SimpleNamespace
+
+
+def _run_fallback(args, input=None, **kwargs):
+    # A port of subprocess.run just enough to run the tests.
+    process = subprocess.Popen(args, stdin=subprocess.PIPE, **kwargs)
+    stdout, stderr = process.communicate(input)
+    retcode = process.wait()
+    return SimpleNamespace(args=args, stdout=stdout, stderr=stderr, returncode=retcode)
+
+
+try:
+    from subprocess import run
+except ImportError:
+    run = _run_fallback
+
+
+def runcode(python, code):
+    """Run `code` in `python`."""
+    return run(
+        [python],
+        input=textwrap.dedent(code),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+        env=dict(
+            _enviorn,
+            # Make PyJulia importable:
+            PYTHONPATH=os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
+        ),
+    )
+
+
+def print_completed_proc(proc):
+    # Print output (pytest will hide it by default):
+    print("Ran:", *proc.args)
+    if proc.stdout:
+        print("# --- STDOUT from", *proc.args)
+        print(proc.stdout)
+    if proc.stderr:
+        print("# --- STDERR from", *proc.args)
+        print(proc.stderr)
+    print("# ---")
+
+
+def is_dynamically_linked(executable):
+    """
+    Check if Python `executable` is (likely to be) dynamically linked.
+
+    It returns three possible values:
+
+    * `True`: Likely that it's dynamically linked.
+    * `False`: Likely that it's statically linked.
+    * `None`: Unsupported platform.
+
+    It's only "likely" since the check is by simple occurrence of a
+    some substrings like "libpython".  For example, if there is
+    another library existing on the path containing "libpython", this
+    function may return false-positive.
+    """
+    path = which(executable)
+    assert os.path.exists(path)
+    if is_linux and which("ldd"):
+        proc = run(
+            ["ldd", path], stdout=subprocess.PIPE, env=_enviorn, universal_newlines=True
+        )
+        print_completed_proc(proc)
+        return "libpython" in proc.stdout
+    elif is_apple and which("otool"):
+        proc = run(
+            ["otool", "-L", path],
+            stdout=subprocess.PIPE,
+            env=_enviorn,
+            universal_newlines=True,
+        )
+        print_completed_proc(proc)
+        return (
+            "libpython" in proc.stdout
+            or "/Python" in proc.stdout
+            or "/.Python" in proc.stdout
+        )
+    # TODO: support Windows
+    return None
+
+
+@pytest.mark.parametrize("python", incompatible_pythons)
+def test_incompatible_python(python):
+    if julia.eval("(VERSION.major, VERSION.minor)") == (0, 6):
+        # Julia 0.6 implements mixed version
+        return
+
+    python = which(python)
+    proc = runcode(
+        python,
+        """
+        import os
+        from julia import Julia
+        Julia(runtime=os.getenv("JULIA_EXE"), debug=True)
+        """,
+    )
+    print_completed_proc(proc)
+
+    assert proc.returncode == 1
+    assert "It seems your Julia and PyJulia setup are not supported." in proc.stderr
+    dynamic = is_dynamically_linked(python)
+    if dynamic is True:
+        assert "`libpython` have to match" in proc.stderr
+    elif dynamic is False:
+        assert "is statically linked to libpython" in proc.stderr
+
+
+@pytest.mark.parametrize(
+    "python",
+    [
+        p
+        for p in filter(None, map(which, incompatible_pythons))
+        if is_dynamically_linked(p) is False
+    ],
+)
+def test_statically_linked(python):
+    """
+    Simulate the case PyCall is configured with statically linked Python.
+
+    In this case, `find_libpython()` would return the path identical
+    to the one in PyCall's deps.jl.  `is_compatible_exe` should reject
+    it.
+    """
+    python = which(python)
+    proc = runcode(
+        python,
+        """
+        from __future__ import print_function
+        from julia.find_libpython import find_libpython
+        from julia.core import is_compatible_exe
+
+        class jlinfo:
+            libpython = find_libpython()
+
+        assert not is_compatible_exe(jlinfo, _debug=print)
+        """,
+    )
+    print_completed_proc(proc)
+    assert proc.returncode == 0

test/test_find_libpython.py

@@ -1,5 +1,7 @@
+import subprocess
+import sys
+
 from julia.find_libpython import finding_libpython, linked_libpython
-from julia.core import determine_if_statically_linked
 
 try:
     unicode
@@ -14,6 +16,22 @@ def test_finding_libpython_yield_type():
 # let's just check returned type of finding_libpython.
 
 
+def determine_if_statically_linked():
+    """Determines if this python executable is statically linked"""
+    if not sys.platform.startswith('linux'):
+        # Assuming that Windows and OS X are generally always
+        # dynamically linked.  Note that this is not the case in
+        # Python installed via conda:
+        # https://github.com/JuliaPy/pyjulia/issues/150#issuecomment-432912833
+        # However, since we do not use conda in our CI, this function
+        # is OK to use in tests.
+        return False
+    lddoutput = subprocess.check_output(["ldd", sys.executable])
+    return not (b"libpython" in lddoutput)
+
+
 def test_linked_libpython():
+    # TODO: Special-case conda (check `sys.version`).  See the above
+    # comments in `determine_if_statically_linked.
     if not determine_if_statically_linked():
         assert linked_libpython() is not None

tox.ini

@@ -31,6 +31,9 @@ passenv =
     PYJULIA_TEST_REBUILD
     JULIA_EXE
 
+    # See: test/test_compatible_exe.py
+    PYJULIA_TEST_INCOMPATIBLE_PYTHONS
+
     # See: https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci
     TRAVIS
     TRAVIS_*