From a86eab99622dee9ad6c0e900cfe78c686449a7b8 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 10:49:23 +0200 Subject: [PATCH 01/26] Build universal wheel --- azure-pipelines.yml | 13 +++++++------ helpers/cibuild.yml | 10 ++-------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 42b3a0c5..1e2349b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,6 +5,8 @@ jobs: - job: linux displayName: Build on Linux pool: {vmImage: 'Ubuntu-16.04'} + variables: + plat_name: "manylinux1_x86_64" steps: - task: UsePythonVersion@0 inputs: @@ -17,6 +19,8 @@ jobs: - job: mac displayName: Build on macOS pool: {vmImage: 'macOS-10.14'} + variables: + plat_name: "macosx_10_9_x86_64" steps: - checkout: self - template: helpers/cibuild.yml @@ -24,6 +28,8 @@ jobs: - job: windows displayName: Build on Windows pool: {vmImage: 'vs2017-win2016'} + variables: + plat_name: "win_amd64" steps: - script: git config --global core.symlinks true - checkout: self @@ -70,11 +76,6 @@ jobs: python py_mini_racer/extension/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib displayName: Build the extension - bash: | - for i in cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38; do - sudo /opt/python/$i/bin/pip install wheel pytest - /opt/python/$i/bin/python setup.py bdist_wheel -d wheelhouse - sudo /opt/python/$i/bin/python setup.py develop --no-deps - /opt/python/$i/bin/pytest tests - done + python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 displayName: Build the Wheelhouse - template: helpers/release.yml diff --git a/helpers/cibuild.yml b/helpers/cibuild.yml index bd1352bf..04511ca5 100644 --- a/helpers/cibuild.yml +++ b/helpers/cibuild.yml @@ -5,17 +5,11 @@ steps: displayName: Build the extension - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - script: | - pip install git+https://github.com/joerick/cibuildwheel.git@0375e92110113689025a5f05376f77d220d8b5c9 - cibuildwheel --output-dir wheelhouse . - env: - CIBW_BEFORE_BUILD: "git clean -ffdx -e py_mini_racer/extension/out py_mini_racer/extension || true" - CIBW_SKIP: "*-manylinux1_i686 *-win32" - CIBW_TEST_REQUIRES: mock pytest pytest-azurepipelines - CIBW_TEST_COMMAND: pytest {project}/tests + python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse - script: | pip install auditwheel - python helpers/wheel_pymalloc.py wheelhouse + for i in wheelhouse/*.whl; do auditwheel show $i; done displayName: Patch the Wheelhouse - script: | pip install twine readme_renderer[md] From f2ae362c90a90ad4e3b8037b277388546e0acf49 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 11:11:41 +0200 Subject: [PATCH 02/26] Install wheel --- azure-pipelines.yml | 1 + helpers/cibuild.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1e2349b2..d3bbf2a9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,6 +76,7 @@ jobs: python py_mini_racer/extension/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib displayName: Build the extension - bash: | + python -m pip install wheel python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 displayName: Build the Wheelhouse - template: helpers/release.yml diff --git a/helpers/cibuild.yml b/helpers/cibuild.yml index 04511ca5..c5518771 100644 --- a/helpers/cibuild.yml +++ b/helpers/cibuild.yml @@ -5,10 +5,11 @@ steps: displayName: Build the extension - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - script: | + python -m pip install wheel python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse - script: | - pip install auditwheel + python -m pip install auditwheel for i in wheelhouse/*.whl; do auditwheel show $i; done displayName: Patch the Wheelhouse - script: | From 907d9a0c4c1d5fca26811b645cf683f4e4b8e96e Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 18:15:48 +0200 Subject: [PATCH 03/26] Build the extension out of setup.py --- .gitignore | 3 +- MANIFEST.in | 6 +- azure-pipelines.yml | 9 ++ helpers/build_package.py | 57 ++++++++ helpers/cibuild.yml | 12 +- .../extension => helpers}/v8_build.py | 29 ++-- py_mini_racer/py_mini_racer.py | 50 +++++-- setup.cfg | 14 -- setup.py | 126 ++---------------- 9 files changed, 142 insertions(+), 164 deletions(-) create mode 100644 helpers/build_package.py rename {py_mini_racer/extension => helpers}/v8_build.py (90%) delete mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 7f0d596e..61da2b06 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ __pycache__/ # C extensions *.so +*.dylib +*.dll # Distribution / packaging .Python @@ -68,7 +70,6 @@ py_mini_racer/extension/depot_tools py_mini_racer/extension/build py_mini_racer/extension/build_overrides py_mini_racer/extension/buildtools -py_mini_racer/extension/old_v8_test_wasm-js.git/ py_mini_racer/extension/testing py_mini_racer/extension/third_party py_mini_racer/extension/tools diff --git a/MANIFEST.in b/MANIFEST.in index f01288f4..2aa0efd2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,10 +3,12 @@ include CONTRIBUTING.rst include HISTORY.rst include LICENSE include README.rst -include py_mini_racer/extension/*.gn + +include py_mini_racer/libmini_racer.so +include py_mini_racer/libmini_racer.dylib +include py_mini_racer/mini_racer.dll recursive-include tests * -recursive-include requirements * recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3bbf2a9..ebe11d25 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,14 @@ jobs: steps: - checkout: self - template: helpers/cibuild.yml + - script: | + python -m pip install auditwheel + for i in wheelhouse/*.whl; do auditwheel show $i; done + displayName: Patch the Wheelhouse + - script: | + pip install twine readme_renderer[md] + twine check wheelhouse/*.whl + displayName: Check the wheels - template: helpers/release.yml - job: windows displayName: Build on Windows @@ -74,6 +82,7 @@ jobs: displayName: Prepare Aline Linux build environment - bash: | python py_mini_racer/extension/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib + cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so displayName: Build the extension - bash: | python -m pip install wheel diff --git a/helpers/build_package.py b/helpers/build_package.py new file mode 100644 index 00000000..0eeb9b16 --- /dev/null +++ b/helpers/build_package.py @@ -0,0 +1,57 @@ +import os +import sys +import glob +import shutil + +from setuptools.build_meta import ( + get_requires_for_build_wheel, + get_requires_for_build_sdist, + prepare_metadata_for_build_wheel, + build_wheel as setuptools_build_wheel, + build_sdist as setuptools_build_sdist, +) + +from v8_build import build_v8 + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + config_settings = config_settings or {} + # Our wheel is compatible both with python 2 & python 3 + config_settings["--global-option"] = options = ["--python-tag", "py2.py3"] + # Clean previous version of the lib + for pattern in ("py_mini_racer/*.so", "py_mini_racer/*.dylib", "py_mini_racer/*.dll"): + for filename in glob.glob(pattern): + print("removing {}".format(filename)) + os.unlink(filename) + # Build V8 + build_v8("py_mini_racer_shared_lib") + # Build the wheel + if os.name == "posix" and sys.platform == "darwin": + shutil.copyfile("py_mini_racer/extension/out/libmini_racer.dylib", "py_mini_racer/libmini_racer.dylib") + options.extend(["--plat-name", "macosx_10_9_x86_64"]) + elif sys.platform == "win32": + shutil.copyfile("py_mini_racer/extension/out/mini_racer.dll", "py_mini_racer/mini_racer.dll") + options.extend(["--plat-name", "win_amd64"]) + else: + shutil.copyfile("py_mini_racer/extension/out/libmini_racer.so", "py_mini_racer/libmini_racer.glibc.so") + options.extend(["--plat-name", "manylinux1_x86_64"]) + return setuptools_build_wheel(wheel_directory, config_settings=config_settings, metadata_directory=metadata_directory) + + +def build_sdist(sdist_directory, config_settings=None): + return setuptools_build_sdist(sdist_directory) + + +__all__ = [ + "get_requires_for_build_wheel", + "get_requires_for_build_sdist", + "prepare_metadata_for_build_wheel", + "build_wheel", + "build_sdist", +] + +if __name__ == "__main__": + if sys.argv[1] == "wheel": + build_wheel(sys.argv[2]) + else: + build_sdist(sys.argv[2]) diff --git a/helpers/cibuild.yml b/helpers/cibuild.yml index c5518771..d252f096 100644 --- a/helpers/cibuild.yml +++ b/helpers/cibuild.yml @@ -1,18 +1,10 @@ steps: +- {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" -- script: python setup.py build_ext +- script: python3 helpers/build_package.py wheel wheelhouse displayName: Build the extension -- {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - script: | python -m pip install wheel python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse -- script: | - python -m pip install auditwheel - for i in wheelhouse/*.whl; do auditwheel show $i; done - displayName: Patch the Wheelhouse -- script: | - pip install twine readme_renderer[md] - twine check wheelhouse/*.whl - displayName: Check the wheels diff --git a/py_mini_racer/extension/v8_build.py b/helpers/v8_build.py similarity index 90% rename from py_mini_racer/extension/v8_build.py rename to helpers/v8_build.py index 3181feae..394bb7f2 100644 --- a/py_mini_racer/extension/v8_build.py +++ b/helpers/v8_build.py @@ -26,13 +26,13 @@ def local_path(path="."): return os.path.abspath(os.path.join(ROOT_DIR, path)) -PATCHES_PATH = local_path('../../patches') +PATCHES_PATH = local_path('../patches') def call(cmd): LOGGER.debug("Calling: '%s' from working directory %s", cmd, os.getcwd()) current_env = os.environ - depot_tools_env = os.pathsep.join([local_path("depot_tools"), os.environ['PATH']]) + depot_tools_env = os.pathsep.join([local_path("../py_mini_racer/extension/depot_tools"), os.environ['PATH']]) current_env['PATH'] = depot_tools_env current_env['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' return subprocess.check_call(cmd, shell=True, env=current_env) @@ -55,9 +55,9 @@ def chdir(new_path, make=False): def install_depot_tools(): - if not os.path.isdir(local_path("depot_tools")): + if not os.path.isdir(local_path("../py_mini_racer/extension/depot_tools")): LOGGER.debug("Cloning depot tools") - call("git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git {}".format(local_path("depot_tools"))) + call("git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git {}".format(local_path("../py_mini_racer/extension/depot_tools"))) else: LOGGER.debug("Using already cloned depot tools") @@ -65,7 +65,7 @@ def install_depot_tools(): def prepare_workdir(): directories = ["build", "build_overrides", "buildtools", "testing", "third_party", "tools"] - with chdir(local_path()): + with chdir(local_path("../py_mini_racer/extension")): for item in directories: if not os.path.exists(item): symlink_force(os.path.join("v8", item), item) @@ -74,14 +74,14 @@ def prepare_workdir(): def ensure_v8_src(revision): """ Ensure that v8 src are presents and up-to-date """ - path = local_path() + path = local_path("../py_mini_racer/extension") - if not os.path.isfile(local_path(".gclient")): + if not os.path.isfile(local_path("../py_mini_racer/extension/.gclient")): fetch_v8(path) else: update_v8(path) - checkout_v8_version(local_path("v8"), revision) + checkout_v8_version(local_path("../py_mini_racer/extension/v8"), revision) dependencies_sync(path) @@ -119,7 +119,7 @@ def run_hooks(path): call("gclient runhooks") def gen_makefiles(build_path, no_sysroot=False): - with chdir(local_path()): + with chdir(local_path("../py_mini_racer/extension")): build_path = local_path(build_path) if not os.path.exists(build_path): os.makedirs(build_path) @@ -170,13 +170,14 @@ def gen_makefiles(build_path, no_sysroot=False): def make(build_path, target, cmd_prefix=""): """ Create a release of v8 """ - with chdir(local_path()): + with chdir(local_path("../py_mini_racer/extension")): call("{} ninja -vv -C {} {}".format(cmd_prefix, local_path(build_path), target)) + def patch_v8(): """ Apply patch on v8 """ - path = local_path("v8") + path = local_path("../py_mini_racer/extension/v8") patches_paths = PATCHES_PATH apply_patches(path, patches_paths) @@ -230,7 +231,7 @@ def apply_patches(path, patches_path): def patch_sysroot(): - with chdir(local_path("v8/build/linux/debian_sid_amd64-sysroot")): + with chdir(local_path("../py_mini_racer/extension/v8/build/linux/debian_sid_amd64-sysroot")): with open("usr/include/glob.h", "r") as f: header = f.read() s, e = header.split("sysroot-creator.sh.", 1) @@ -279,7 +280,7 @@ def build_v8(target=None, build_path=None, revision=None, no_build=False, target = "v8" if build_path is None: # Must be relative to local_path() - build_path = "out" + build_path = "../py_mini_racer/extension/out" if revision is None: revision = V8_VERSION install_depot_tools() @@ -290,7 +291,7 @@ def build_v8(target=None, build_path=None, revision=None, no_build=False, patch_sysroot() prepare_workdir() if not no_build: - checkout_path = local_path("v8") + checkout_path = local_path("../py_mini_racer/extension/v8") cmd_prefix = fixup_libtinfo(checkout_path) gen_makefiles(build_path, no_sysroot=no_sysroot) make(build_path, target, cmd_prefix) diff --git a/py_mini_racer/py_mini_racer.py b/py_mini_racer/py_mini_racer.py index 69c0e490..155409dc 100644 --- a/py_mini_racer/py_mini_racer.py +++ b/py_mini_racer/py_mini_racer.py @@ -9,19 +9,49 @@ import threading import datetime import fnmatch +import sysconfig + +try: + import pkg_resources +except ImportError: + pkg_resources = None # pragma: no cover + + +def _get_libc_name(): + """Return the libc of the system.""" + target = sysconfig.get_config_var("HOST_GNU_TYPE") + if target is not None and target.endswith("musl"): + return "muslc" + return "glibc" + + +def _get_lib_path(name): + """Return the path of the library called `name`.""" + if os.name == "posix" and sys.platform == "darwin": + prefix, ext = "lib", ".dylib" + elif sys.platform == "win32": + prefix, ext = "", ".dll" + else: + prefix, ext = "lib", ".{}.so".format(_get_libc_name()) + fn = None + meipass = getattr(sys, "_MEIPASS", None) + if meipass is not None: + fn = os.path.join(meipass, prefix + name + ext) + if fn is None and pkg_resources is not None: + fn = pkg_resources.resource_filename("py_mini_racer", prefix + name + ext) + if fn is None: + root_dir = os.path.dirname(os.path.abspath(__file__)) + fn = os.path.join(root_dir, prefix + name + ext) + return fn -from pkg_resources import resource_listdir, resource_filename # In python 3 the extension file name depends on the python version -try: - EXTENSION_NAME = fnmatch.filter(resource_listdir('py_mini_racer', '.'), '_v8*.so')[0] - EXTENSION_PATH = resource_filename('py_mini_racer', EXTENSION_NAME) -except NotImplementedError: - if not hasattr(sys, "_MEIPASS"): - raise - __location__ = os.path.join(sys._MEIPASS, "_v8") # pylint: disable=no-member, protected-access - EXTENSION_NAME = fnmatch.filter(os.listdir(__location__), '_v8*.so')[0] - EXTENSION_PATH = os.path.join(__location__, EXTENSION_NAME) +EXTENSION_PATH = _get_lib_path("mini_racer") +if EXTENSION_PATH is None or not os.path.exists(EXTENSION_PATH): + raise RuntimeError("Native library not available at {}".format(EXTENSION_PATH)) + +EXTENSION_NAME = os.path.basename(EXTENSION_PATH) + if sys.version_info[0] < 3: UNICODE_TYPE = unicode diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2b92bef3..00000000 --- a/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[bumpversion] -current_version = 0.1.0 -commit = True -tag = True - -[bumpversion:file:setup.py] - -[bumpversion:file:py_mini_racer/__init__.py] - -[wheel] -universal = 0 - -[aliases] -test=pytest diff --git a/setup.py b/setup.py index aa702afa..9851fcaf 100755 --- a/setup.py +++ b/setup.py @@ -2,131 +2,36 @@ # -*- coding: utf-8 -*- import os -import sys import io -import pkg_resources -import traceback - -from itertools import chain -from subprocess import check_call, check_output, STDOUT -from os.path import dirname, abspath, join, isfile, isdir, basename - -from distutils.file_util import copy_file try: - from setuptools import setup, Extension, Command - from setuptools.command.build_ext import build_ext - from setuptools.command.install import install + from setuptools import setup except ImportError: - from distutils.core import setup, Extension, Command - from distutils.command.build_ext import build_ext - from distutils.command.install import install + from distutils.core import setup + -from py_mini_racer import __version__ -from py_mini_racer.extension.v8_build import build_v8, \ - local_path as local_path_v8 +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +about = {} # type: ignore -with io.open('README.rst', 'r', encoding='utf8') as readme_file: +# Read the version from the source +with io.open(os.path.join(ROOT_DIR, "py_mini_racer", "__init__.py"), encoding="utf-8") as f: + exec(f.read(), about) + +with io.open(os.path.join(ROOT_DIR, 'README.rst'), 'r', encoding='utf8') as readme_file: readme = readme_file.read() # Convert local image links by their github equivalent readme = readme.replace(".. image:: data/", ".. image:: https://github.com/sqreen/PyMiniRacer/raw/master/data/") -with io.open('HISTORY.rst', 'r', encoding='utf8') as history_file: +with io.open(os.path.join(ROOT_DIR, 'HISTORY.rst'), 'r', encoding='utf8') as history_file: history = history_file.read().replace('.. :changelog:', '') -def check_python_version(): - """ Check that the python executable is Python 2.7. - """ - output = check_output(['python', '--version'], stderr=STDOUT) - if not output.strip().decode().startswith('Python 2.7'): - msg = """py_mini_racer cannot build V8 in the current configuration. - The V8 build system requires the python executable to be Python 2.7. - See also: https://github.com/sqreen/PyMiniRacer#build""" - raise Exception(msg) - -def lib_filename(name, static=False): - if os.name == "posix" and sys.platform == "darwin": - prefix = "lib" - if static: - ext = ".a" - else: - ext = ".dylib" - elif sys.platform == "win32": - prefix = "" - if static: - ext = ".lib" - else: - ext = ".dll" - else: - prefix = "lib" - if static: - ext = ".a" - else: - ext = ".so" - return prefix + name + ext - - -class V8Extension(Extension): - - def __init__(self, dest_module, target, lib, sources=[], **kwa): - Extension.__init__(self, dest_module, sources=sources, **kwa) - self.target = target - self.lib = lib - - -class MiniRacerBuildExt(build_ext): - - def get_ext_filename(self, name): - # XXX the filename is the same for all platforms for now - ext = ".so" - parts = name.split(".") - last = parts.pop(-1) + ext - return os.path.join(*(parts + [last])) - - def build_extensions(self): - self.debug = True - try: - for ext in self.extensions: - src = os.path.join(ext.lib) - dest = self.get_ext_fullpath(ext.name) - if not os.path.isfile(dest) and not os.path.isfile(src): - check_python_version() - print("building {}".format(ext.target)) - build_v8(ext.target) - if not os.path.isfile(dest): - dest_dir = os.path.dirname(dest) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - copy_file(src, dest) - else: - print("extension was already built") - except Exception as e: - traceback.print_exc() - - # Alter message - err_msg = """py_mini_racer failed to build, ensure you have an up-to-date pip (>= 8.1) to use the wheel instead - To update pip: 'pip install -U pip' - See also: https://github.com/sqreen/PyMiniRacer#binary-builds-availability - - Original error: %s""" - - raise Exception(err_msg % repr(e)) - - -PY_MINI_RACER_EXTENSION = V8Extension( - "py_mini_racer._v8", - "py_mini_racer_shared_lib", - local_path_v8(os.path.join("out", lib_filename("mini_racer"))) -) - - setup( name='py_mini_racer', - version=__version__, + version=about["__version__"], description="Minimal, modern embedded V8 for Python.", long_description=readme + '\n\n' + history, author='Sqreen', @@ -134,9 +39,7 @@ def build_extensions(self): url='https://github.com/sqreen/PyMiniRacer', packages=[ 'py_mini_racer', - 'py_mini_racer.extension' ], - ext_modules=[PY_MINI_RACER_EXTENSION], package_dir={'py_mini_racer': 'py_mini_racer'}, include_package_data=True, @@ -162,8 +65,5 @@ def build_extensions(self): "tox", "six", "pytest", - ], - cmdclass={ - "build_ext": MiniRacerBuildExt, - } + ] ) From d04da194944f3237ae2b1c774147346ca47e760b Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 18:29:41 +0200 Subject: [PATCH 04/26] Fix CI --- MANIFEST.in | 2 +- azure-pipelines.yml | 39 +++++++++++++++++++++++++++++---------- helpers/cibuild.yml | 10 ---------- 3 files changed, 30 insertions(+), 21 deletions(-) delete mode 100644 helpers/cibuild.yml diff --git a/MANIFEST.in b/MANIFEST.in index 2aa0efd2..b99a907a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include HISTORY.rst include LICENSE include README.rst -include py_mini_racer/libmini_racer.so +include py_mini_racer/libmini_racer.glibc.so py_mini_racer/libmini_racer.muslc.so include py_mini_racer/libmini_racer.dylib include py_mini_racer/mini_racer.dll diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ebe11d25..3b4a94e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,7 +14,20 @@ jobs: architecture: x64 - checkout: self - bash: python setup.py sdist --dist-dir wheelhouse - - template: helpers/cibuild.yml + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - script: | + python3 -m pip install setuptools wheel + python3 helpers/build_package.py wheel wheelhouse + displayName: Build the Wheelhouse + - script: | + python -m pip install auditwheel + for i in wheelhouse/*.whl; do auditwheel show $i; done + displayName: Patch the Wheelhouse + - script: | + python -m pip install twine readme_renderer[md] + twine check wheelhouse/*.whl + displayName: Check the wheels - template: helpers/release.yml - job: mac displayName: Build on macOS @@ -23,15 +36,12 @@ jobs: plat_name: "macosx_10_9_x86_64" steps: - checkout: self - - template: helpers/cibuild.yml - - script: | - python -m pip install auditwheel - for i in wheelhouse/*.whl; do auditwheel show $i; done - displayName: Patch the Wheelhouse + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - script: | - pip install twine readme_renderer[md] - twine check wheelhouse/*.whl - displayName: Check the wheels + python3 -m pip install setuptools wheel + python3 helpers/build_package.py wheel wheelhouse + displayName: Build the wheelhouse - template: helpers/release.yml - job: windows displayName: Build on Windows @@ -45,7 +55,16 @@ jobs: Remove-Item C:\ProgramData\Chocolatey\bin\python2.7.exe -force Remove-Item C:\ProgramData\Chocolatey\bin\python2.exe -force displayName: Remove extraneous Python Interpreters - - template: helpers/cibuild.yml + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" + - script: | + python py_mini_racer/extension/v8_build.py + cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll + displayName: Build the extension + - script: | + python -m pip install wheel + python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) + displayName: Build the Wheelhouse - template: helpers/release.yml - job: fetch_sources_for_alpine displayName: Prepare build for Alpine Linux diff --git a/helpers/cibuild.yml b/helpers/cibuild.yml deleted file mode 100644 index d252f096..00000000 --- a/helpers/cibuild.yml +++ /dev/null @@ -1,10 +0,0 @@ -steps: -- {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} -- {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} -- bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" -- script: python3 helpers/build_package.py wheel wheelhouse - displayName: Build the extension -- script: | - python -m pip install wheel - python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) - displayName: Build the Wheelhouse From d985c8af89cec9ff03c6fb6dd15034366c428720 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 18:56:04 +0200 Subject: [PATCH 05/26] Fix CI --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b4a94e8..2282addc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ jobs: python3 helpers/build_package.py wheel wheelhouse displayName: Build the Wheelhouse - script: | - python -m pip install auditwheel + python3 -m pip install auditwheel for i in wheelhouse/*.whl; do auditwheel show $i; done displayName: Patch the Wheelhouse - script: | @@ -58,7 +58,7 @@ jobs: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - script: | - python py_mini_racer/extension/v8_build.py + python helpers/v8_build.py cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll displayName: Build the extension - script: | @@ -75,7 +75,7 @@ jobs: versionSpec: '2.7' architecture: x64 - checkout: self - - bash: python py_mini_racer/extension/v8_build.py --no-build --no-sysroot + - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources - bash: tar -czf v8.tar.gz py_mini_racer/extension displayName: Archive V8 sources @@ -100,7 +100,7 @@ jobs: rm -f py_mini_racer/extension/depot_tools/ninja displayName: Prepare Aline Linux build environment - bash: | - python py_mini_racer/extension/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib + python helpers/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so displayName: Build the extension - bash: | From cabf5b3acedadd17e3f081e8858926e9fb49d97d Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 19:12:20 +0200 Subject: [PATCH 06/26] Create the wheelhouse directory --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2282addc..e6c0e752 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,6 +40,7 @@ jobs: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - script: | python3 -m pip install setuptools wheel + mkdir wheelhouse python3 helpers/build_package.py wheel wheelhouse displayName: Build the wheelhouse - template: helpers/release.yml @@ -63,6 +64,7 @@ jobs: displayName: Build the extension - script: | python -m pip install wheel + mkdir wheelhouse python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse - template: helpers/release.yml @@ -105,6 +107,7 @@ jobs: displayName: Build the extension - bash: | python -m pip install wheel + mkdir wheelhouse python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 displayName: Build the Wheelhouse - template: helpers/release.yml From 215e03924548974dc5ce523ef4fb9c2272332d69 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 20:01:34 +0200 Subject: [PATCH 07/26] Fix CI --- helpers/v8_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/v8_build.py b/helpers/v8_build.py index 394bb7f2..6ab640b0 100644 --- a/helpers/v8_build.py +++ b/helpers/v8_build.py @@ -300,7 +300,7 @@ def build_v8(target=None, build_path=None, revision=None, no_build=False, if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--target", default="v8", help="Ninja target") - parser.add_argument("--build-path", default="out", help="Build destination directory (relative to the path)") + parser.add_argument("--build-path", default="../py_mini_racer/extension/out", help="Build destination directory (relative to the current path)") parser.add_argument("--v8-revision", default=V8_VERSION) parser.add_argument("--no-build", action="store_true", help="Only prepare workdir") parser.add_argument("--no-update", action="store_true", help="Do not update the workdir") From 59ba6516a7ab1c4e06132f2dce7079ec7ec5c7c6 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Tue, 15 Sep 2020 23:14:38 +0200 Subject: [PATCH 08/26] Fix CI --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e6c0e752..cfc74e6c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,6 +60,7 @@ jobs: - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - script: | python helpers/v8_build.py + ls -l py_mini_racer/extension/out/ cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll displayName: Build the extension - script: | @@ -106,8 +107,8 @@ jobs: cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so displayName: Build the extension - bash: | - python -m pip install wheel + /opt/python/cp36-cp36m/bin/python -m pip install wheel mkdir wheelhouse - python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 + /opt/python/cp36-cp36m/bin/python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 displayName: Build the Wheelhouse - template: helpers/release.yml From 4df2c982553a80e832e41e9c0931e040a9a66d03 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 16 Sep 2020 10:16:57 +0200 Subject: [PATCH 09/26] Fix windows target --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cfc74e6c..3974f864 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,6 @@ jobs: versionSpec: '3.6' architecture: x64 - checkout: self - - bash: python setup.py sdist --dist-dir wheelhouse - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - script: | @@ -59,8 +58,7 @@ jobs: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - script: | - python helpers/v8_build.py - ls -l py_mini_racer/extension/out/ + python helpers/v8_build.py py_mini_racer_shared_lib cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll displayName: Build the extension - script: | @@ -111,4 +109,6 @@ jobs: mkdir wheelhouse /opt/python/cp36-cp36m/bin/python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 displayName: Build the Wheelhouse + - bash: /opt/python/cp36-cp36m/bin/python setup.py sdist --dist-dir wheelhouse + displayName: Build the source package - template: helpers/release.yml From 0d23adaaf151b17da1b1e4966ae3147cab57e8d5 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 16 Sep 2020 10:25:12 +0200 Subject: [PATCH 10/26] Fix windows target --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3974f864..02f91600 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,7 +58,7 @@ jobs: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - script: | - python helpers/v8_build.py py_mini_racer_shared_lib + python helpers/v8_build.py --target py_mini_racer_shared_lib cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll displayName: Build the extension - script: | From 115ba5f139a468f10e88875a7a931fd349d2ed41 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 16 Sep 2020 10:52:00 +0200 Subject: [PATCH 11/26] Use Python 3 to build wheelhouse on windows --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02f91600..c274d953 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ jobs: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - script: | python3 -m pip install setuptools wheel + mkdir wheelhouse python3 helpers/build_package.py wheel wheelhouse displayName: Build the Wheelhouse - script: | @@ -61,6 +62,7 @@ jobs: python helpers/v8_build.py --target py_mini_racer_shared_lib cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll displayName: Build the extension + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - script: | python -m pip install wheel mkdir wheelhouse From fdc15ba9cca9f7de6cf1f60af5b7a0aef4e33082 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Wed, 16 Sep 2020 17:45:00 +0200 Subject: [PATCH 12/26] Move to .azure/pipelines/build.yml --- .azure/pipelines/build.yml | 198 +++++++++++++++++++++++++++++++++++++ azure-pipelines.yml | 116 ---------------------- helpers/release.yml | 15 --- py_mini_racer/__init__.py | 2 - 4 files changed, 198 insertions(+), 133 deletions(-) create mode 100644 .azure/pipelines/build.yml delete mode 100644 azure-pipelines.yml delete mode 100644 helpers/release.yml diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml new file mode 100644 index 00000000..c7ad8ff7 --- /dev/null +++ b/.azure/pipelines/build.yml @@ -0,0 +1,198 @@ +trigger: +- master + +stages: +- stage: build + jobs: + - job: linux + displayName: Build on Linux + pool: {vmImage: 'Ubuntu-16.04'} + variables: + plat_name: "manylinux1_x86_64" + steps: + - checkout: self + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - script: | + python3 -m pip install setuptools wheel + mkdir wheelhouse + python3 helpers/build_package.py wheel wheelhouse + displayName: Build the Wheelhouse + - script: | + python3 -m pip install auditwheel + for i in wheelhouse/*.whl; do auditwheel show $i; done + displayName: Patch the Wheelhouse + - script: | + python -m pip install twine readme_renderer[md] + twine check wheelhouse/*.whl + displayName: Check the wheels + - publish: wheelhouse + artifact: wheelhouse + - job: mac + displayName: Build on macOS + pool: {vmImage: 'macOS-10.14'} + variables: + plat_name: "macosx_10_9_x86_64" + steps: + - checkout: self + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - script: | + python3 -m pip install setuptools wheel + mkdir wheelhouse + python3 helpers/build_package.py wheel wheelhouse + displayName: Build the wheelhouse + - publish: wheelhouse + artifact: wheelhouse + - job: windows + displayName: Build on Windows + pool: {vmImage: 'vs2017-win2016'} + variables: + plat_name: "win_amd64" + steps: + - script: git config --global core.symlinks true + - checkout: self + - powershell: | + Remove-Item C:\ProgramData\Chocolatey\bin\python2.7.exe -force + Remove-Item C:\ProgramData\Chocolatey\bin\python2.exe -force + displayName: Remove extraneous Python Interpreters + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" + - script: | + python helpers/v8_build.py --target py_mini_racer_shared_lib + cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll + displayName: Build the extension + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - script: | + python -m pip install wheel + mkdir wheelhouse + python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) + displayName: Build the Wheelhouse + - publish: wheelhouse + artifact: wheelhouse + - job: fetch_sources_for_alpine + displayName: Prepare build for Alpine Linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '2.7' + architecture: x64 + - checkout: self + - bash: python helpers/v8_build.py --no-build --no-sysroot + displayName: Download V8 sources + - bash: tar -czf v8.tar.gz py_mini_racer/extension + displayName: Archive V8 sources + - publish: v8.tar.gz + artifact: sources-for-alpine + - job: alpine + displayName: Build on Alpine Linux + dependsOn: + - fetch_sources_for_alpine + container: nicolassqreen/azure-pipelines-container-alpine-python:latest + steps: + - checkout: self + - download: current + artifact: sources-for-alpine + - bash: | + echo Extracting in $PWD + tar -xf $(Pipeline.Workspace)/sources-for-alpine/v8.tar.gz + displayName: Extract V8 sources + - bash: | + sudo apk add ninja llvm lld linux-headers + cp -f /usr/local/bin/gn py_mini_racer/extension/v8/buildtools/linux64/gn + rm -f py_mini_racer/extension/depot_tools/ninja + displayName: Prepare Aline Linux build environment + - bash: | + python helpers/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib + cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so + displayName: Build the extension + - bash: | + /opt/python/cp36-cp36m/bin/python -m pip install wheel + mkdir wheelhouse + /opt/python/cp36-cp36m/bin/python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 + displayName: Build the Wheelhouse + - bash: /opt/python/cp36-cp36m/bin/python setup.py sdist --dist-dir wheelhouse + displayName: Build the source package + - publish: wheelhouse + artifact: wheelhouse + +- stage: test + jobs: + - job: linux + displayName: Test on Linux + pool: {vmImage: 'Ubuntu-16.04'} + variables: + plat_name: "manylinux1_x86_64" + steps: + - checkout: self + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - download: current + artifact: wheelhouse + - script: | + python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install pytest + pytest tests + displayName: Test the Wheel + - job: mac + displayName: Build on macOS + pool: {vmImage: 'macOS-10.14'} + variables: + plat_name: "macosx_10_9_x86_64" + steps: + - checkout: self + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - download: current + artifact: wheelhouse + - script: | + python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install pytest + pytest tests + displayName: Test the Wheel + - job: windows + displayName: Build on Windows + pool: {vmImage: 'vs2017-win2016'} + variables: + plat_name: "win_amd64" + steps: + - script: git config --global core.symlinks true + - checkout: self + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - download: current + artifact: wheelhouse + - script: | + python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install pytest + pytest tests + displayName: Test the Wheel + - job: alpine + displayName: Test on Alpine Linux + container: nicolassqreen/azure-pipelines-container-alpine-python:latest + steps: + - checkout: self + - download: current + artifact: wheelhouse + - bash: | + /opt/python/cp36-cp36m/bin/python -m pip wheelhouse/*.tar.gz + /opt/python/cp36-cp36m/bin/python -m pip install pytest + /opt/python/cp36-cp36m/bin/pytest tests + displayName: Test the Source Dist + +- stage: release + condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master')) + jobs: + - job: release + steps: + - download: current + artifact: wheelhouse + - task: S3Upload@1 + displayName: Publish Wheelhouse to S3 + inputs: + awsCredentials: AWS + bucketName: sqreen-download-private + globExpressions: | + *.whl + *.tar.gz + targetFolder: python/py-mini-racer/ + sourceFolder: wheelhouse + flattenFolders: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c274d953..00000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,116 +0,0 @@ -trigger: -- master - -jobs: -- job: linux - displayName: Build on Linux - pool: {vmImage: 'Ubuntu-16.04'} - variables: - plat_name: "manylinux1_x86_64" - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - architecture: x64 - - checkout: self - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - script: | - python3 -m pip install setuptools wheel - mkdir wheelhouse - python3 helpers/build_package.py wheel wheelhouse - displayName: Build the Wheelhouse - - script: | - python3 -m pip install auditwheel - for i in wheelhouse/*.whl; do auditwheel show $i; done - displayName: Patch the Wheelhouse - - script: | - python -m pip install twine readme_renderer[md] - twine check wheelhouse/*.whl - displayName: Check the wheels - - template: helpers/release.yml -- job: mac - displayName: Build on macOS - pool: {vmImage: 'macOS-10.14'} - variables: - plat_name: "macosx_10_9_x86_64" - steps: - - checkout: self - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - script: | - python3 -m pip install setuptools wheel - mkdir wheelhouse - python3 helpers/build_package.py wheel wheelhouse - displayName: Build the wheelhouse - - template: helpers/release.yml -- job: windows - displayName: Build on Windows - pool: {vmImage: 'vs2017-win2016'} - variables: - plat_name: "win_amd64" - steps: - - script: git config --global core.symlinks true - - checkout: self - - powershell: | - Remove-Item C:\ProgramData\Chocolatey\bin\python2.7.exe -force - Remove-Item C:\ProgramData\Chocolatey\bin\python2.exe -force - displayName: Remove extraneous Python Interpreters - - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - - script: | - python helpers/v8_build.py --target py_mini_racer_shared_lib - cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll - displayName: Build the extension - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - - script: | - python -m pip install wheel - mkdir wheelhouse - python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) - displayName: Build the Wheelhouse - - template: helpers/release.yml -- job: fetch_sources_for_alpine - displayName: Prepare build for Alpine Linux - pool: {vmImage: 'Ubuntu-16.04'} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '2.7' - architecture: x64 - - checkout: self - - bash: python helpers/v8_build.py --no-build --no-sysroot - displayName: Download V8 sources - - bash: tar -czf v8.tar.gz py_mini_racer/extension - displayName: Archive V8 sources - - publish: v8.tar.gz - artifact: sources-for-alpine -- job: alpine - displayName: Build on Alpine Linux - dependsOn: - - fetch_sources_for_alpine - container: nicolassqreen/azure-pipelines-container-alpine-python:latest - steps: - - checkout: self - - download: current - artifact: sources-for-alpine - - bash: | - echo Extracting in $PWD - tar -xf $(Pipeline.Workspace)/sources-for-alpine/v8.tar.gz - displayName: Extract V8 sources - - bash: | - sudo apk add ninja llvm lld linux-headers - cp -f /usr/local/bin/gn py_mini_racer/extension/v8/buildtools/linux64/gn - rm -f py_mini_racer/extension/depot_tools/ninja - displayName: Prepare Aline Linux build environment - - bash: | - python helpers/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib - cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so - displayName: Build the extension - - bash: | - /opt/python/cp36-cp36m/bin/python -m pip install wheel - mkdir wheelhouse - /opt/python/cp36-cp36m/bin/python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name linux_x86_64 - displayName: Build the Wheelhouse - - bash: /opt/python/cp36-cp36m/bin/python setup.py sdist --dist-dir wheelhouse - displayName: Build the source package - - template: helpers/release.yml diff --git a/helpers/release.yml b/helpers/release.yml deleted file mode 100644 index 663f2a43..00000000 --- a/helpers/release.yml +++ /dev/null @@ -1,15 +0,0 @@ -steps: -- task: PublishBuildArtifacts@1 - inputs: {pathtoPublish: "wheelhouse"} -- task: S3Upload@1 - condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master')) - displayName: Publish Wheelhouse to S3 - inputs: - awsCredentials: AWS - bucketName: sqreen-download-private - globExpressions: | - *.whl - *.tar.gz - targetFolder: python/py-mini-racer/ - sourceFolder: wheelhouse - flattenFolders: true diff --git a/py_mini_racer/__init__.py b/py_mini_racer/__init__.py index 7a908a92..841fbdb5 100755 --- a/py_mini_racer/__init__.py +++ b/py_mini_racer/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - __author__ = 'Sqreen' __email__ = 'hey@sqreen.io' __version__ = '0.3.0' From 1086683ff4d423ba0c0f564f471a7774f687f18d Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 18 Sep 2020 10:19:41 +0200 Subject: [PATCH 13/26] Remove artifact name --- .azure/pipelines/build.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index c7ad8ff7..d790bd22 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -27,7 +27,6 @@ stages: twine check wheelhouse/*.whl displayName: Check the wheels - publish: wheelhouse - artifact: wheelhouse - job: mac displayName: Build on macOS pool: {vmImage: 'macOS-10.14'} @@ -43,7 +42,6 @@ stages: python3 helpers/build_package.py wheel wheelhouse displayName: Build the wheelhouse - publish: wheelhouse - artifact: wheelhouse - job: windows displayName: Build on Windows pool: {vmImage: 'vs2017-win2016'} @@ -69,7 +67,6 @@ stages: python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse - publish: wheelhouse - artifact: wheelhouse - job: fetch_sources_for_alpine displayName: Prepare build for Alpine Linux pool: {vmImage: 'Ubuntu-16.04'} @@ -115,7 +112,6 @@ stages: - bash: /opt/python/cp36-cp36m/bin/python setup.py sdist --dist-dir wheelhouse displayName: Build the source package - publish: wheelhouse - artifact: wheelhouse - stage: test jobs: @@ -128,7 +124,6 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current - artifact: wheelhouse - script: | python -m pip install wheelhouse/*$(plat_name).whl python -m pip install pytest @@ -143,7 +138,6 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current - artifact: wheelhouse - script: | python -m pip install wheelhouse/*$(plat_name).whl python -m pip install pytest @@ -159,7 +153,6 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current - artifact: wheelhouse - script: | python -m pip install wheelhouse/*$(plat_name).whl python -m pip install pytest @@ -171,7 +164,6 @@ stages: steps: - checkout: self - download: current - artifact: wheelhouse - bash: | /opt/python/cp36-cp36m/bin/python -m pip wheelhouse/*.tar.gz /opt/python/cp36-cp36m/bin/python -m pip install pytest @@ -184,7 +176,6 @@ stages: - job: release steps: - download: current - artifact: wheelhouse - task: S3Upload@1 displayName: Publish Wheelhouse to S3 inputs: From c8704943d767be5813479a378f599516a6496a80 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 18 Sep 2020 13:49:53 +0200 Subject: [PATCH 14/26] Download only build artifacts --- .azure/pipelines/build.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index d790bd22..45289df2 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -124,8 +124,9 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current + artifact: build.linux - script: | - python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install py_mini_racer-*$(plat_name).whl python -m pip install pytest pytest tests displayName: Test the Wheel @@ -138,8 +139,9 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current + artifact: build.mac - script: | - python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install py_mini_racer-*$(plat_name).whl python -m pip install pytest pytest tests displayName: Test the Wheel @@ -153,8 +155,9 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current + artifact: build.windows - script: | - python -m pip install wheelhouse/*$(plat_name).whl + python -m pip install py_mini_racer-*$(plat_name).whl python -m pip install pytest pytest tests displayName: Test the Wheel @@ -164,8 +167,9 @@ stages: steps: - checkout: self - download: current + artifact: alpine - bash: | - /opt/python/cp36-cp36m/bin/python -m pip wheelhouse/*.tar.gz + /opt/python/cp36-cp36m/bin/python -m pip install py_mini_racer-*.tar.gz /opt/python/cp36-cp36m/bin/python -m pip install pytest /opt/python/cp36-cp36m/bin/pytest tests displayName: Test the Source Dist @@ -176,14 +180,20 @@ stages: - job: release steps: - download: current + artifact: build.linux + - download: current + artifact: build.mac + - download: current + artifact: build.windows + - download: current + artifact: build.alpine - task: S3Upload@1 displayName: Publish Wheelhouse to S3 inputs: awsCredentials: AWS bucketName: sqreen-download-private globExpressions: | - *.whl - *.tar.gz + py_mini_racer-*.whl + py_mini_racer-*.tar.gz targetFolder: python/py-mini-racer/ - sourceFolder: wheelhouse flattenFolders: true From f7eb9bbfed4d9c5190eaa88b9fd38d6c2e3bb97d Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Fri, 18 Sep 2020 14:34:34 +0200 Subject: [PATCH 15/26] Paths --- .azure/pipelines/build.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index 45289df2..69de9ddd 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -126,9 +126,9 @@ stages: - download: current artifact: build.linux - script: | - python -m pip install py_mini_racer-*$(plat_name).whl + python -m pip install $(Pipeline.Workspace)/build.linux/*.whl python -m pip install pytest - pytest tests + pytest $(Build.SourcesDirectory)/tests displayName: Test the Wheel - job: mac displayName: Build on macOS @@ -141,9 +141,9 @@ stages: - download: current artifact: build.mac - script: | - python -m pip install py_mini_racer-*$(plat_name).whl + python -m pip install $(Pipeline.Workspace)/build.mac/*.whl python -m pip install pytest - pytest tests + pytest $(Build.SourcesDirectory)/tests displayName: Test the Wheel - job: windows displayName: Build on Windows @@ -157,9 +157,9 @@ stages: - download: current artifact: build.windows - script: | - python -m pip install py_mini_racer-*$(plat_name).whl + python -m pip install $(Pipeline.Workspace)/build.windows/*.whl python -m pip install pytest - pytest tests + pytest $(Build.SourcesDirectory)/tests displayName: Test the Wheel - job: alpine displayName: Test on Alpine Linux @@ -167,15 +167,15 @@ stages: steps: - checkout: self - download: current - artifact: alpine + artifact: build.alpine - bash: | - /opt/python/cp36-cp36m/bin/python -m pip install py_mini_racer-*.tar.gz + /opt/python/cp36-cp36m/bin/python -m pip install $(Pipeline.Workspace)/build.alpine/*.tar.gz /opt/python/cp36-cp36m/bin/python -m pip install pytest /opt/python/cp36-cp36m/bin/pytest tests displayName: Test the Source Dist - stage: release - condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master')) + condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/tags/')) jobs: - job: release steps: @@ -193,7 +193,8 @@ stages: awsCredentials: AWS bucketName: sqreen-download-private globExpressions: | - py_mini_racer-*.whl - py_mini_racer-*.tar.gz + **/py_mini_racer-*.whl + **/py_mini_racer-*.tar.gz targetFolder: python/py-mini-racer/ + sourceFolder: $(Pipeline.Workspace) flattenFolders: true From 5ace0a29a31fc8228ecc9d9563c53612af74cf16 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 20 Sep 2020 12:58:00 +0200 Subject: [PATCH 16/26] Fix test and cache sources --- .azure/pipelines/build.yml | 48 ++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index 69de9ddd..c1493e0b 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -13,6 +13,16 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - task: Cache@2 + inputs: + key: 'depot_tools | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/depot_tools + displayName: Cache V8 depot tools + - task: Cache@2 + inputs: + key: 'v8 | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/v8 + displayName: Cache V8 sources - script: | python3 -m pip install setuptools wheel mkdir wheelhouse @@ -36,6 +46,16 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - task: Cache@2 + inputs: + key: 'depot_tools | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/depot_tools + displayName: Cache V8 depot tools + - task: Cache@2 + inputs: + key: 'v8 | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/v8 + displayName: Cache V8 sources - script: | python3 -m pip install setuptools wheel mkdir wheelhouse @@ -56,6 +76,16 @@ stages: displayName: Remove extraneous Python Interpreters - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" + - task: Cache@2 + inputs: + key: 'depot_tools | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/depot_tools + displayName: Cache V8 depot tools + - task: Cache@2 + inputs: + key: 'v8 | $(plat_name) | helpers/v8_build.py' + path: py_mini_racer/extension/v8 + displayName: Cache V8 sources - script: | python helpers/v8_build.py --target py_mini_racer_shared_lib cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll @@ -75,6 +105,16 @@ stages: inputs: versionSpec: '2.7' architecture: x64 + - task: Cache@2 + inputs: + key: 'depot_tools | manylinux1_x86_64 | helpers/v8_build.py' + path: py_mini_racer/extension/depot_tools + displayName: Cache V8 depot tools + - task: Cache@2 + inputs: + key: 'v8 | manylinux1_x86_64 | helpers/v8_build.py' + path: py_mini_racer/extension/v8 + displayName: Cache V8 sources - checkout: self - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources @@ -131,7 +171,7 @@ stages: pytest $(Build.SourcesDirectory)/tests displayName: Test the Wheel - job: mac - displayName: Build on macOS + displayName: Test on macOS pool: {vmImage: 'macOS-10.14'} variables: plat_name: "macosx_10_9_x86_64" @@ -146,7 +186,7 @@ stages: pytest $(Build.SourcesDirectory)/tests displayName: Test the Wheel - job: windows - displayName: Build on Windows + displayName: Test on Windows pool: {vmImage: 'vs2017-win2016'} variables: plat_name: "win_amd64" @@ -156,7 +196,7 @@ stages: - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - download: current artifact: build.windows - - script: | + - bash: | python -m pip install $(Pipeline.Workspace)/build.windows/*.whl python -m pip install pytest pytest $(Build.SourcesDirectory)/tests @@ -171,7 +211,7 @@ stages: - bash: | /opt/python/cp36-cp36m/bin/python -m pip install $(Pipeline.Workspace)/build.alpine/*.tar.gz /opt/python/cp36-cp36m/bin/python -m pip install pytest - /opt/python/cp36-cp36m/bin/pytest tests + /opt/python/cp36-cp36m/bin/python -m pytest $(Build.SourcesDirectory)/tests displayName: Test the Source Dist - stage: release From 8bb03255fb4b7cd7ef1bbf0bb5c838f764348145 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 20 Sep 2020 13:32:49 +0200 Subject: [PATCH 17/26] Fix checkout --- .azure/pipelines/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index c1493e0b..4de6eff1 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -85,6 +85,7 @@ stages: inputs: key: 'v8 | $(plat_name) | helpers/v8_build.py' path: py_mini_racer/extension/v8 + continueOnError: true displayName: Cache V8 sources - script: | python helpers/v8_build.py --target py_mini_racer_shared_lib @@ -101,6 +102,7 @@ stages: displayName: Prepare build for Alpine Linux pool: {vmImage: 'Ubuntu-16.04'} steps: + - checkout: self - task: UsePythonVersion@0 inputs: versionSpec: '2.7' @@ -115,7 +117,6 @@ stages: key: 'v8 | manylinux1_x86_64 | helpers/v8_build.py' path: py_mini_racer/extension/v8 displayName: Cache V8 sources - - checkout: self - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources - bash: tar -czf v8.tar.gz py_mini_racer/extension From 0c0f70feedffb78a15a1247578f82070d1c07fe5 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Sun, 20 Sep 2020 13:44:15 +0200 Subject: [PATCH 18/26] No cache --- .azure/pipelines/build.yml | 41 -------------------------------------- 1 file changed, 41 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index 4de6eff1..c21ac225 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -13,16 +13,6 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - task: Cache@2 - inputs: - key: 'depot_tools | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/depot_tools - displayName: Cache V8 depot tools - - task: Cache@2 - inputs: - key: 'v8 | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/v8 - displayName: Cache V8 sources - script: | python3 -m pip install setuptools wheel mkdir wheelhouse @@ -46,16 +36,6 @@ stages: - checkout: self - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - task: Cache@2 - inputs: - key: 'depot_tools | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/depot_tools - displayName: Cache V8 depot tools - - task: Cache@2 - inputs: - key: 'v8 | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/v8 - displayName: Cache V8 sources - script: | python3 -m pip install setuptools wheel mkdir wheelhouse @@ -76,17 +56,6 @@ stages: displayName: Remove extraneous Python Interpreters - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\py_mini_racer\\extension\\depot_tools\\" - - task: Cache@2 - inputs: - key: 'depot_tools | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/depot_tools - displayName: Cache V8 depot tools - - task: Cache@2 - inputs: - key: 'v8 | $(plat_name) | helpers/v8_build.py' - path: py_mini_racer/extension/v8 - continueOnError: true - displayName: Cache V8 sources - script: | python helpers/v8_build.py --target py_mini_racer_shared_lib cp py_mini_racer/extension/out/mini_racer.dll py_mini_racer/mini_racer.dll @@ -107,16 +76,6 @@ stages: inputs: versionSpec: '2.7' architecture: x64 - - task: Cache@2 - inputs: - key: 'depot_tools | manylinux1_x86_64 | helpers/v8_build.py' - path: py_mini_racer/extension/depot_tools - displayName: Cache V8 depot tools - - task: Cache@2 - inputs: - key: 'v8 | manylinux1_x86_64 | helpers/v8_build.py' - path: py_mini_racer/extension/v8 - displayName: Cache V8 sources - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources - bash: tar -czf v8.tar.gz py_mini_racer/extension From 01a7a75eab6b406d4f75f57bf072702f9ca7d239 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 10:55:56 +0200 Subject: [PATCH 19/26] Try relative paths --- .azure/pipelines/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index c21ac225..3ebca710 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -157,9 +157,9 @@ stages: - download: current artifact: build.windows - bash: | - python -m pip install $(Pipeline.Workspace)/build.windows/*.whl + python -m pip install ../build.windows/*.whl python -m pip install pytest - pytest $(Build.SourcesDirectory)/tests + pytest tests displayName: Test the Wheel - job: alpine displayName: Test on Alpine Linux @@ -171,7 +171,7 @@ stages: - bash: | /opt/python/cp36-cp36m/bin/python -m pip install $(Pipeline.Workspace)/build.alpine/*.tar.gz /opt/python/cp36-cp36m/bin/python -m pip install pytest - /opt/python/cp36-cp36m/bin/python -m pytest $(Build.SourcesDirectory)/tests + /opt/python/cp36-cp36m/bin/pytest $(Build.SourcesDirectory)/tests displayName: Test the Source Dist - stage: release From dc941b3fcd905de6d646c637e5c86749c55157f0 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 13:38:01 +0200 Subject: [PATCH 20/26] Try with sudo --- .azure/pipelines/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index 3ebca710..b5058eb1 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -169,8 +169,8 @@ stages: - download: current artifact: build.alpine - bash: | - /opt/python/cp36-cp36m/bin/python -m pip install $(Pipeline.Workspace)/build.alpine/*.tar.gz - /opt/python/cp36-cp36m/bin/python -m pip install pytest + sudo /opt/python/cp36-cp36m/bin/python -m pip install $(Pipeline.Workspace)/build.alpine/*.tar.gz + sudo /opt/python/cp36-cp36m/bin/python -m pip install pytest /opt/python/cp36-cp36m/bin/pytest $(Build.SourcesDirectory)/tests displayName: Test the Source Dist From 5d82902e12d0c7e2573ef26cc11f18b091442af8 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 14:35:05 +0200 Subject: [PATCH 21/26] Try to target host for alpine build --- .azure/pipelines/build.yml | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index b5058eb1..a6a60df7 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -67,34 +67,20 @@ stages: python setup.py bdist_wheel --dist-dir wheelhouse --python-tag py2.py3 --plat-name $(plat_name) displayName: Build the Wheelhouse - publish: wheelhouse - - job: fetch_sources_for_alpine - displayName: Prepare build for Alpine Linux + - job: alpine + displayName: Build on Alpine Linux pool: {vmImage: 'Ubuntu-16.04'} + container: nicolassqreen/azure-pipelines-container-alpine-python:latest steps: - checkout: self - task: UsePythonVersion@0 inputs: versionSpec: '2.7' architecture: x64 + target: host - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources - - bash: tar -czf v8.tar.gz py_mini_racer/extension - displayName: Archive V8 sources - - publish: v8.tar.gz - artifact: sources-for-alpine - - job: alpine - displayName: Build on Alpine Linux - dependsOn: - - fetch_sources_for_alpine - container: nicolassqreen/azure-pipelines-container-alpine-python:latest - steps: - - checkout: self - - download: current - artifact: sources-for-alpine - - bash: | - echo Extracting in $PWD - tar -xf $(Pipeline.Workspace)/sources-for-alpine/v8.tar.gz - displayName: Extract V8 sources + target: host - bash: | sudo apk add ninja llvm lld linux-headers cp -f /usr/local/bin/gn py_mini_racer/extension/v8/buildtools/linux64/gn From 6e85055806b60248389d0c3215ae36906ae1ca81 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 14:51:20 +0200 Subject: [PATCH 22/26] Try with default python binary --- .azure/pipelines/build.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index a6a60df7..76918046 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -73,11 +73,6 @@ stages: container: nicolassqreen/azure-pipelines-container-alpine-python:latest steps: - checkout: self - - task: UsePythonVersion@0 - inputs: - versionSpec: '2.7' - architecture: x64 - target: host - bash: python helpers/v8_build.py --no-build --no-sysroot displayName: Download V8 sources target: host @@ -166,13 +161,6 @@ stages: - job: release steps: - download: current - artifact: build.linux - - download: current - artifact: build.mac - - download: current - artifact: build.windows - - download: current - artifact: build.alpine - task: S3Upload@1 displayName: Publish Wheelhouse to S3 inputs: From 386e500703cc07e9ce48e394a1782bcd14c5dd34 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 15:15:44 +0200 Subject: [PATCH 23/26] Update README and bump version --- README.rst | 27 +++++++++++---------------- py_mini_racer/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index 1bd1de91..551d0dd7 100644 --- a/README.rst +++ b/README.rst @@ -138,14 +138,6 @@ We built Python wheels (prebuilt binaries) for macOS 64 bits, Linux 64 bits and $ pip install py-mini-racer -If you're running Alpine, Pypi isn't able to host the dedicated wheels. -We're making them available with the following command. - -.. code:: bash - - $ pip install --index-url https://download.sqreen.io/python/alpine py-mini-racer - - Build ----- @@ -161,20 +153,17 @@ You can build the extension with the following command: .. code:: bash - $ python setup.py build_ext + $ python helpers/v8_build.py You can generate a wheel for whatever Python version with the command: .. code:: bash - $ python setup.py build_ext # (for Python 2 and Python 3) - $ python setup.py bdist_wheel # (for Python 2 only) - $ python3 setup.py bdist_wheel # (for Python 3 only) + $ python3 helpers/build_package.py wheel dist It will then build V8, the extension, and generates a wheel for your current Python version. The V8 builds are cached in the ``py_mini_racer/extension/v8/`` -directory. Please note that you can build Python 3 wheels by reusing the -cached version of V8 built with Python 2.7. +directory. Notes for building on macOS ''''''''''''''''''''''''''' @@ -187,11 +176,17 @@ They will allow to build a wheel compatible with former OSX versions. Tests ----- -If you want to run the tests, you need to build the extension first, then launch: +If you want to run the tests, you need to build the extension first, first install pytest: + +.. code-block:: bash + + $ python -m pip install pytest + +Then launch: .. code:: bash - $ python setup.py test --addopts tests + $ python -m pytest tests Credits ------- diff --git a/py_mini_racer/__init__.py b/py_mini_racer/__init__.py index 841fbdb5..ab4d3bc2 100755 --- a/py_mini_racer/__init__.py +++ b/py_mini_racer/__init__.py @@ -1,5 +1,5 @@ __author__ = 'Sqreen' __email__ = 'hey@sqreen.io' -__version__ = '0.3.0' +__version__ = '0.4.0' __all__ = ['py_mini_racer'] diff --git a/setup.py b/setup.py index 9851fcaf..3d88359f 100755 --- a/setup.py +++ b/setup.py @@ -34,8 +34,8 @@ version=about["__version__"], description="Minimal, modern embedded V8 for Python.", long_description=readme + '\n\n' + history, - author='Sqreen', - author_email='hey@sqreen.io', + author=about["__author__"], + author_email=about["__email__"], url='https://github.com/sqreen/PyMiniRacer', packages=[ 'py_mini_racer', From 90441ccc362dc57f0d3fadddd66231b35e955343 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 15:16:55 +0200 Subject: [PATCH 24/26] Test release --- .azure/pipelines/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index 76918046..e4d26107 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -156,7 +156,7 @@ stages: displayName: Test the Source Dist - stage: release - condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/tags/')) + #condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/tags/')) jobs: - job: release steps: From b0d4ff3e7b5b4605d1ccc75c9debb5b06697fd87 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 15:46:52 +0200 Subject: [PATCH 25/26] Restore release condition --- .azure/pipelines/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/build.yml b/.azure/pipelines/build.yml index e4d26107..76918046 100644 --- a/.azure/pipelines/build.yml +++ b/.azure/pipelines/build.yml @@ -156,7 +156,7 @@ stages: displayName: Test the Source Dist - stage: release - #condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/tags/')) + condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/tags/')) jobs: - job: release steps: From 64cdc43e5357ac127a0bd0878115844e5507b592 Mon Sep 17 00:00:00 2001 From: Nicolas Vivet Date: Mon, 21 Sep 2020 16:02:23 +0200 Subject: [PATCH 26/26] Do not raise when module is loading --- py_mini_racer/py_mini_racer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/py_mini_racer/py_mini_racer.py b/py_mini_racer/py_mini_racer.py index 155409dc..950252ac 100644 --- a/py_mini_racer/py_mini_racer.py +++ b/py_mini_racer/py_mini_racer.py @@ -47,10 +47,7 @@ def _get_lib_path(name): # In python 3 the extension file name depends on the python version EXTENSION_PATH = _get_lib_path("mini_racer") -if EXTENSION_PATH is None or not os.path.exists(EXTENSION_PATH): - raise RuntimeError("Native library not available at {}".format(EXTENSION_PATH)) - -EXTENSION_NAME = os.path.basename(EXTENSION_PATH) +EXTENSION_NAME = os.path.basename(EXTENSION_PATH) if EXTENSION_PATH is not None else None if sys.version_info[0] < 3: @@ -122,6 +119,8 @@ def _fetch_ext_handle(): if _ext_handle: return _ext_handle + if EXTENSION_PATH is None or not os.path.exists(EXTENSION_PATH): + raise RuntimeError("Native library not available at {}".format(EXTENSION_PATH)) _ext_handle = ctypes.CDLL(EXTENSION_PATH) _ext_handle.mr_init_context.restype = ctypes.c_void_p