From 67ddda830274474186f98f044c8334ddf02a59a0 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 16 Apr 2020 13:51:48 +0200 Subject: [PATCH 1/9] Install numpy This is required by the unittests --- scripts/build/Dockerfile.debian | 3 +++ scripts/build/Dockerfile.manylinux | 2 ++ scripts/build/build.sh | 1 + 3 files changed, 6 insertions(+) diff --git a/scripts/build/Dockerfile.debian b/scripts/build/Dockerfile.debian index 73d84a6..c512cc5 100644 --- a/scripts/build/Dockerfile.debian +++ b/scripts/build/Dockerfile.debian @@ -37,5 +37,8 @@ RUN apt-get update && apt-get install -y locales \ RUN mkdir /work RUN mkdir /pylon_installer +RUN mkdir /worker_home && chmod go+rwx /worker_home +ENV HOME=/worker_home + RUN uname -a diff --git a/scripts/build/Dockerfile.manylinux b/scripts/build/Dockerfile.manylinux index 79dd3c1..cb6e5d5 100644 --- a/scripts/build/Dockerfile.manylinux +++ b/scripts/build/Dockerfile.manylinux @@ -19,5 +19,7 @@ RUN mkdir /build && \ RUN mkdir /work RUN mkdir /pylon_installer +RUN mkdir /worker_home && chmod go+rwx /worker_home +ENV HOME=/worker_home RUN uname -a \ No newline at end of file diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 6f1dc23..6feb162 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -78,6 +78,7 @@ export PYLON_ROOT $PYTHON setup.py clean if [ -z "$DISABLE_TESTS" ]; then + $PYTHON -m pip install --user numpy #For now failed tests are accepted until all are fixed $PYTHON setup.py test || true fi From b9386a9a1c755ba4befeb4776a2efdb035070c83 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 16 Apr 2020 16:44:38 +0200 Subject: [PATCH 2/9] Add french locale to manylinux images This is required in the unittests --- scripts/build/Dockerfile.manylinux | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build/Dockerfile.manylinux b/scripts/build/Dockerfile.manylinux index cb6e5d5..ab7e05f 100644 --- a/scripts/build/Dockerfile.manylinux +++ b/scripts/build/Dockerfile.manylinux @@ -16,6 +16,8 @@ RUN mkdir /build && \ ./configure --with-python3 && make -j2 && make install && \ rm -rf /build +# one genicam unittest requires a french locale +RUN yum -y reinstall glibc-common RUN mkdir /work RUN mkdir /pylon_installer From 3c0203fb90ff8c8be9665eaec40793a00f2f2cc5 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Mon, 27 Apr 2020 17:42:19 +0200 Subject: [PATCH 3/9] Allow manylinux to fail silently --- .travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8554f4b..b69ad0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,6 +81,27 @@ jobs: env: P=linux_aarch64 A=cp37m - arch: arm64 env: P=linux_aarch64 A=cp38 + allow_failures: + # manylinux fails, because the runtime dependencies of current pylon are higher than manylinux2014. + # The jobs are still kept in here to keep the mechanics alive. + # manylinux2014_x86_64 + - arch: amd64 + env: P=manylinux2014_x86_64 A=cp35m + - arch: amd64 + env: P=manylinux2014_x86_64 A=cp36m + - arch: amd64 + env: P=manylinux2014_x86_64 A=cp37m + - arch: amd64 + env: P=manylinux2014_x86_64 A=cp38 + # manylinux2014_i686 + - arch: amd64 + env: P=manylinux2014_i686 A=cp35m + - arch: amd64 + env: P=manylinux2014_i686 A=cp36m + - arch: amd64 + env: P=manylinux2014_i686 A=cp37m + - arch: amd64 + env: P=manylinux2014_i686 A=cp38 install: From 0dfa0309eab112ffe39e3cff464b96b3af4fbca6 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Tue, 21 Apr 2020 16:51:33 +0200 Subject: [PATCH 4/9] Switch to pylon 6.1 on linux The new pylon also comes with newer requirements to libc and libstdc++. To be able to run the unittests, we need to upgrade the docker base images. Additionally mechanics for architecture dependent default pylon versions where needed. --- .travis.yml | 5 +++-- VersionInfo.py | 5 ++++- scripts/build/build-arch.sh | 8 ++++---- setup.py | 9 +++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b69ad0f..dc58160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -108,9 +108,10 @@ install: - cd $TRAVIS_BUILD_DIR - mkdir pylon_installer && cd pylon_installer - | - for PYLON_ARCH in armhf arm64 x86 x86_64 ; do - travis_retry curl -sSfL -O "${PYLON_DOWNLOAD_URL_BASE}pylon-5.2.0.13457-${PYLON_ARCH}.tar.gz" + for PYLON_VERSION in 6.1.0.19674_x86 6.1.1.19861_x86_64 6.1.3.20159_armhf 6.1.3.20159_aarch64 ; do + travis_retry curl -sSfL -O "${PYLON_DOWNLOAD_URL_BASE}pylon_${PYLON_VERSION}_setup.tar.gz" done + - echo "Pylon Installer:" && ls script: diff --git a/VersionInfo.py b/VersionInfo.py index 57d326f..ba46be6 100644 --- a/VersionInfo.py +++ b/VersionInfo.py @@ -1,6 +1,9 @@ # The pylon version this source tree was designed for, by platform ReferencePylonVersion = { "Windows": "6.1.0", - "Linux": "5.2.0", + # ATTENTION: This version is the pylon core version reported by pylon-config, + # which is not equal to the version on the outer tar.gz + "Linux": "6.1.0", + "Linux_x86_64": "6.1.1", "Darwin": "5.1.1" } diff --git a/scripts/build/build-arch.sh b/scripts/build/build-arch.sh index 7491ad4..2d5e953 100755 --- a/scripts/build/build-arch.sh +++ b/scripts/build/build-arch.sh @@ -54,10 +54,10 @@ if [ $BUILD_DISTRO = "debian" ]; then #Note: Be careful when changing the base image. Not every image is available for every architecture. case $ABI_TAG in - cp27m) BASE_IMAGE="python:2.7.16-jessie" ;; - cp34m) BASE_IMAGE="python:3.4.8-jessie" ;; - cp35m) BASE_IMAGE="python:3.5.5-jessie" ;; - cp36m) BASE_IMAGE="python:3.6.5-jessie" ;; + cp27m) BASE_IMAGE="python:2.7.16-stretch" ;; + cp34m) BASE_IMAGE="python:3.4.8-stretch" ;; + cp35m) BASE_IMAGE="python:3.5.5-stretch" ;; + cp36m) BASE_IMAGE="python:3.6.5-stretch" ;; cp37m) BASE_IMAGE="python:3.7.5-stretch" ;; cp38) BASE_IMAGE="python:3.8.2-buster" ;; *) diff --git a/setup.py b/setup.py index e0a66b7..8e78065 100755 --- a/setup.py +++ b/setup.py @@ -39,6 +39,9 @@ def get_machinewidth(): def get_platform(): return platform.system() +def get_machine(): + return platform.machine() + class BuildSupport(object): # --- Constants --- @@ -291,6 +294,12 @@ def get_version(self): pylon_version_tag = match.group(2) reference_version = VersionInfo.ReferencePylonVersion[get_platform()] + + # check for a more specialized reference version + platform_machine = get_platform() + "_" + get_machine() + if platform_machine in VersionInfo.ReferencePylonVersion: + reference_version = VersionInfo.ReferencePylonVersion[platform_machine] + if ( pylon_version_no_build == reference_version and pylon_version_tag == '' From 2283395e5bc6ed4ac6357bee0d9a21c427d45aec Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Fri, 26 Jun 2020 11:14:54 +0200 Subject: [PATCH 5/9] Fix armv7 build on travis with native armv8 machines We switched to native arm machines on travis for speed reasons. The python build detected aarch64 as target architecture even inside the armv7 containers. This lead to armv7 wheels that were incorrectly labelled and published as aarch64. The real aarch64 packages where in turn never published because the name was already taken. --- scripts/build/Dockerfile.debian | 9 ++++++++- scripts/build/build-arch.sh | 13 +++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/build/Dockerfile.debian b/scripts/build/Dockerfile.debian index c512cc5..f57f973 100644 --- a/scripts/build/Dockerfile.debian +++ b/scripts/build/Dockerfile.debian @@ -5,6 +5,7 @@ ARG DOCKER_BASE_IMAGE FROM multiarch/qemu-user-static:4.2.0-6 as qemu FROM $DOCKER_BASE_IMAGE +ARG CMD_WRAPPER COPY --from=qemu /usr/bin/* /usr/bin/ # Quick fix for the now archived debian jessie. Security updates are also no longer provided for arm64. @@ -41,4 +42,10 @@ RUN mkdir /worker_home && chmod go+rwx /worker_home ENV HOME=/worker_home -RUN uname -a +# run everything wrapped using CMD_WRAPPER +# In most cases the wrapper is linux64/32. +# This is for example required when running the armv7 container on armv8 hardware to ensure that python really builds for armv7 +RUN echo "#!/bin/sh" > /entrypoint.sh; echo exec $CMD_WRAPPER \"\$@\" >> /entrypoint.sh; chmod +x /entrypoint.sh +ENTRYPOINT [ "/entrypoint.sh" ] + +RUN /entrypoint.sh uname -a diff --git a/scripts/build/build-arch.sh b/scripts/build/build-arch.sh index 2d5e953..a765e76 100755 --- a/scripts/build/build-arch.sh +++ b/scripts/build/build-arch.sh @@ -77,13 +77,18 @@ else esac fi +# When running a armv7 container on a native aarch64 machine, uname -a still outputs aarch64 instead of armv7 +# Python assumes it should buid for aarch64. Wrapping everything with linux32 fixes the issue. +# linux64 is the default +CMD_WRAPPER=linux64 + case $PLATFORM_TAG in linux_x86_64) QEMU_ARCH="x86_64"; BASE_IMAGE="amd64/$BASE_IMAGE"; PYLON_ARCH=x86_64 ;; - linux_i686) QEMU_ARCH="i386"; BASE_IMAGE="i386/$BASE_IMAGE"; PYLON_ARCH=x86 ;; - linux_armv7l) QEMU_ARCH="arm"; BASE_IMAGE="arm32v7/$BASE_IMAGE"; PYLON_ARCH=armhf ;; + linux_i686) QEMU_ARCH="i386"; BASE_IMAGE="i386/$BASE_IMAGE"; PYLON_ARCH=x86 ; CMD_WRAPPER=linux32 ;; + linux_armv7l) QEMU_ARCH="arm"; BASE_IMAGE="arm32v7/$BASE_IMAGE"; PYLON_ARCH=armhf CMD_WRAPPER=linux32 ;; linux_aarch64) QEMU_ARCH="aarch64"; BASE_IMAGE="arm64v8/$BASE_IMAGE"; PYLON_ARCH=aarch64 ;; manylinux2014_x86_64) QEMU_ARCH="x86_64"; BASE_IMAGE="quay.io/pypa/manylinux2014_x86_64"; PYLON_ARCH=x86_64 ;; - manylinux2014_i686) QEMU_ARCH="i386"; BASE_IMAGE="quay.io/pypa/manylinux2014_i686"; PYLON_ARCH=x86 ;; + manylinux2014_i686) QEMU_ARCH="i386"; BASE_IMAGE="quay.io/pypa/manylinux2014_i686"; PYLON_ARCH=x86 ; CMD_WRAPPER=linux32 ;; manylinux2014_aarch64) QEMU_ARCH="aarch64"; BASE_IMAGE="quay.io/pypa/manylinux2014_aarch64"; PYLON_ARCH=aarch64 ;; *) echo "Unsupported platform tag '$PLATFORM_TAG'. Supported platforms: linux_x86_64, linux_i686, linux_armv7l, linux_aarch64, manylinux2014_x86_64, manylinux2014_i686, manylinux2014_aarch64" @@ -117,11 +122,11 @@ if [ ! -f "$PYLON" ]; then exit 1 fi - DOCKER_TAG="pypylon-$PLATFORM_TAG-$(date +%s)" docker build --network host \ --build-arg "QEMU_TARGET_ARCH=$QEMU_ARCH" \ --build-arg "DOCKER_BASE_IMAGE=$BASE_IMAGE" \ + --build-arg "CMD_WRAPPER=$CMD_WRAPPER" \ --tag "$DOCKER_TAG" - < $THISDIR/Dockerfile.$BUILD_DISTRO From 3626d942ecb17ef6a824a7c177b5607928dea4ff Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Wed, 19 Aug 2020 09:16:20 +0200 Subject: [PATCH 6/9] Updated to newest pylon on windows --- .appveyor.yml | 2 +- VersionInfo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c67a23b..a5f3ddb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -39,7 +39,7 @@ install: - 7z x swigwin.zip -oC:\ > NUL - set PATH=C:\swigwin-4.0.1;%PATH% # install pylon - - appveyor-retry curl -sSfL -o pylon_installer.exe %PYLON_DOWNLOAD_URL_BASE%Basler_pylon_6.1.0.19674.exe + - appveyor-retry curl -sSfL -o pylon_installer.exe %PYLON_DOWNLOAD_URL_BASE%Basler_pylon_6.1.1.19832.exe - pylon_installer.exe /quiet /install # PYLON_DEV_DIR is not available in the shell after installation, so we set it manually - set PYLON_DEV_DIR=%PROGRAMFILES%\Basler\pylon 6\development diff --git a/VersionInfo.py b/VersionInfo.py index ba46be6..3371094 100644 --- a/VersionInfo.py +++ b/VersionInfo.py @@ -1,6 +1,6 @@ # The pylon version this source tree was designed for, by platform ReferencePylonVersion = { - "Windows": "6.1.0", + "Windows": "6.1.1", # ATTENTION: This version is the pylon core version reported by pylon-config, # which is not equal to the version on the outer tar.gz "Linux": "6.1.0", From cb45ea4dc3094439587df5c92d5a08b0e19c4105 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 16 Apr 2020 09:42:42 +0200 Subject: [PATCH 7/9] Enable tests on linux --- .travis.yml | 2 +- scripts/build/build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc58160..fdc13c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -120,7 +120,7 @@ script: if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then docker run --rm --privileged multiarch/qemu-user-static:register --reset fi -- "./scripts/build/build-arch.sh --platform-tag $P --abi-tag $A --pylon-dir ./pylon_installer --disable-tests" +- "./scripts/build/build-arch.sh --platform-tag $P --abi-tag $A --pylon-dir ./pylon_installer" deploy: provider: releases diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 6feb162..594da80 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -80,7 +80,7 @@ $PYTHON setup.py clean if [ -z "$DISABLE_TESTS" ]; then $PYTHON -m pip install --user numpy #For now failed tests are accepted until all are fixed - $PYTHON setup.py test || true + $PYTHON setup.py test fi $PYTHON setup.py bdist_wheel From 11832207fdcc5de4daeeb23565f5fcd7eeb0ce89 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 16 Apr 2020 17:01:14 +0200 Subject: [PATCH 8/9] Enable tests on appveyor build --- .appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index a5f3ddb..bf26bd6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -43,8 +43,11 @@ install: - pylon_installer.exe /quiet /install # PYLON_DEV_DIR is not available in the shell after installation, so we set it manually - set PYLON_DEV_DIR=%PROGRAMFILES%\Basler\pylon 6\development + # install numpy for the unittests + - "%PYTHON%\\python.exe -m pip install numpy" build_script: + - "scripts\\build\\msvc_build_wrapper.cmd %PYTHON%\\python.exe setup.py test" - "scripts\\build\\msvc_build_wrapper.cmd %PYTHON%\\python.exe setup.py bdist_wheel" test_script: From 9b18629053f83ad400fc7d0dd66c91c6e07d55e1 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Wed, 19 Aug 2020 15:40:29 +0200 Subject: [PATCH 9/9] Updated changelog --- changelog.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/changelog.txt b/changelog.txt index 13958a1..09624a0 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,7 +1,9 @@ -Version ?.?.? - - Date ????-??-?? - - Fix issues with callbacks due to intptr_t truncation on linux - - Fix unittests that failed in python 3.8 +Version 1.6.0 + - Date 2020-08-20 + - Fixed issues with callbacks due to intptr_t truncation on linux + - Fixed unittests that failed in python 3.8 + - Updated to current pylon version for windows and linux + - Enabled unittests in CI-builds Version 1.5.4 - Date 2020-04-16