Fix build lockfile (#466)

Move from `lockfile` library to `fasteners`

Author: MillionIntegrals

mujoco_py/builder.py

@@ -1,26 +1,26 @@
 import distutils
+import glob
 import os
 import shutil
+import subprocess
 import sys
 from distutils.core import Extension
 from distutils.dist import Distribution
 from distutils.sysconfig import customize_compiler
+from importlib.machinery import ExtensionFileLoader
 from os.path import abspath, dirname, exists, join, getmtime
 from random import choice
 from shutil import move
 from string import ascii_lowercase
-from importlib.machinery import ExtensionFileLoader
-import glob
 
+import fasteners
 import numpy as np
-from cffi import FFI
 from Cython.Build import cythonize
 from Cython.Distutils.old_build_ext import old_build_ext as build_ext
-from mujoco_py.version import get_version
-from lockfile import LockFile
-import subprocess
+from cffi import FFI
 
 from mujoco_py.utils import discover_mujoco, MISSING_KEY_MESSAGE
+from mujoco_py.version import get_version
 
 
 def get_nvidia_lib_dir():
@@ -86,7 +86,7 @@ def load_cython_ext(mujoco_path):
 
     lockpath = os.path.join(os.path.dirname(cext_so_path), 'mujocopy-buildlock')
 
-    with LockFile(lockpath):
+    with fasteners.InterProcessLock(lockpath):
         mod = None
         force_rebuild = os.environ.get('MUJOCO_PY_FORCE_REBUILD')
         if force_rebuild:
@@ -104,6 +104,7 @@ def load_cython_ext(mujoco_path):
         if mod is None:
             cext_so_path = builder.build()
             mod = load_dynamic_ext('cymj', cext_so_path)
+
     return mod
 
 
@@ -119,7 +120,7 @@ def _ensure_set_env_var(var_name, lib_path):
 
 
 def load_dynamic_ext(name, path):
-    ''' Load compiled shared object and return as python module. '''
+    """ Load compiled shared object and return as python module. """
     loader = ExtensionFileLoader(name, path)
     return loader.load_module()
 
@@ -144,7 +145,7 @@ def build_extensions(self):
 
 
 def fix_shared_library(so_file, name, library_path):
-    ''' Used to fixup shared libraries on Linux '''
+    """ Used to fixup shared libraries on Linux """
     subprocess.check_call(['patchelf', '--remove-rpath', so_file])
     ldd_output = subprocess.check_output(['ldd', so_file]).decode('utf-8')
 
@@ -154,7 +155,7 @@ def fix_shared_library(so_file, name, library_path):
 
 
 def manually_link_libraries(mujoco_path, raw_cext_dll_path):
-    ''' Used to fix mujoco library linking on Mac '''
+    """ Used to fix mujoco library linking on Mac """
     root, ext = os.path.splitext(raw_cext_dll_path)
     final_cext_dll_path = root + '_final' + ext
 
@@ -490,8 +491,6 @@ def build_callback_fn(function_string, userdata_names=[]):
     return module.lib.__fun
 
 
-
-
 def find_key():
     ''' Try to find the key file, if missing, print out a big message '''
     if exists(key_path):

mujoco_py/tests/test_cymj.py

@@ -1,19 +1,25 @@
-import numpy as np
-import pytest
+import glob
+import os
+import shutil
 import sys
+import time
 import unittest
-
 from io import BytesIO, StringIO
 from multiprocessing import get_context
 from numbers import Number
-from numpy.testing import assert_array_equal, assert_array_almost_equal
+
+import numpy as np
+import pytest
 # from threading import Thread, Event
 from PIL import Image
+from numpy.testing import assert_array_equal, assert_array_almost_equal
 
-from mujoco_py import (MjSim, load_model_from_xml,
-                       load_model_from_path, MjSimState,
-                       ignore_mujoco_warnings,
-                       load_model_from_mjb)
+from mujoco_py import (
+    MjSim, load_model_from_xml,
+    load_model_from_path, MjSimState,
+    ignore_mujoco_warnings,
+    load_model_from_mjb
+)
 from mujoco_py import const, cymj, functions
 from mujoco_py.tests.utils import compare_imgs
 
@@ -42,6 +48,26 @@
 """
 
 
+def remove_mujoco_build():
+    # Removes previously compiled mujoco_py files.
+    print("Removing previously compiled mujoco_py files.")
+    path = os.path.join(os.path.dirname(__file__), "generated")
+    for fname in glob.glob(f"{path}/*.so"):
+        os.remove(fname)
+    for dirname in glob.glob(f"{path}/_pyxbld*"):
+        shutil.rmtree(dirname, ignore_errors=True)
+    shutil.rmtree(f"{path}/__pycache__", ignore_errors=True)
+
+
+def remove_mujoco_build_and_lock():
+    # Removes previously compiled mujoco_py files.
+    remove_mujoco_build()
+    path = os.path.join(os.path.dirname(__file__), "..", "generated")
+    fname = f"{path}/mujocopy-buildlock.lock"
+    if os.path.exists(fname):
+        os.remove(fname)
+
+
 def test_nested():
     model = load_model_from_xml(BASIC_MODEL_XML)
     model.vis.global_.fovy
@@ -150,6 +176,7 @@ def udd_callback(sim):
     with pytest.raises(AssertionError):
         sim.step()
 
+
 def test_data_attribute_getters():
     model = load_model_from_xml(BASIC_MODEL_XML)
     sim = MjSim(model)
@@ -657,14 +684,15 @@ def test_high_res():
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="This test fails on windows.")
 def test_multiprocess():
-    '''
+    """
     Tests for importing mujoco_py from multiple processes.
-    '''
+    """
+    remove_mujoco_build_and_lock()
     ctx = get_context('spawn')
     processes = []
     times = 3
     queue = ctx.Queue()
-    for idx in range(3):
+    for idx in range(5):
         processes.append(ctx.Process(target=import_process, args=(queue, )))
     for p in processes:
         p.start()
@@ -674,6 +702,32 @@ def test_multiprocess():
         assert queue.get(), "One of processes failed."
 
 
+@pytest.mark.skipif(sys.platform.startswith("win"), reason="This test fails on windows.")
+def test_multiprocess_with_killing():
+    """
+    Kills a process in a middle of compilation and verifies that
+    other processes can resume compilation.
+    """
+    remove_mujoco_build_and_lock()
+    ctx = get_context('spawn')
+    processes = []
+    times = 3
+    queue = ctx.Queue()
+    for idx in range(times):
+        processes.append(ctx.Process(target=import_process, args=(queue, )))
+    processes[0].start()
+    # We wait 20s so the compilation already
+    # has started. Then we kill the process.
+    time.sleep(20)
+    for p in processes[1:]:
+        p.start()
+    processes[0].terminate()
+    for p in processes[1:]:
+        p.join()
+    for _ in range(times - 1):
+        assert queue.get(), "One of processes failed."
+
+
 def import_process(queue):
     try:
         from mujoco_py import builder

mujoco_py/version.py

@@ -1,12 +1,12 @@
 __all__ = ['__version__', 'get_version']
 
-version_info = (2, 0, 2, 6)
+version_info = (2, 0, 2, 7)
 # format:
 # ('mujoco_major', 'mujoco_minor', 'mujoco_py_major', 'mujoco_py_minor')
 
 
 def get_version():
-    "Returns the version as a human-format string."
+    """ Returns the version as a human-format string. """
     return '%d.%d.%d.%d' % version_info
 
 

pyproject.toml

@@ -6,5 +6,5 @@ requires = [
     "Cython>=0.27.2",
     "imageio>=2.1.2",
     "cffi>=1.10",
-    "lockfile>=0.12.2"
+    "fasteners~=0.15"
 ]

requirements.txt

@@ -4,4 +4,4 @@ numpy>=1.11
 Cython>=0.27.2
 imageio>=2.1.2
 cffi>=1.10
-lockfile>=0.12.2
+fasteners~=0.15