yongtang · yongtang · Feb 9, 2019 · Feb 8, 2019 · Feb 9, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -1,14 +1,7 @@
-dist: xenial
+dist: trusty
 sudo: required
 language: minimal
 
-addons:
-  apt:
-    update: true
-
-env:
-  - BAZEL_VERSION=0.20.0
-
 services:
   - docker
 
@@ -18,74 +11,16 @@ before_script:
 
 matrix:
   include:
-    - name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 16.04 Python 2.7 Tests"
-      script:
-        - docker pull ubuntu:16.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 16.04 Python 3.5 Tests"
-      script:
-        - docker pull ubuntu:16.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 18.04 Python 2.7 Tests"
-      script:
-        - docker pull ubuntu:18.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 18.04 Python 3.6 Tests"
-      script:
-        - docker pull ubuntu:18.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
-    - name: "Ubuntu 16.04 Python 2.7 Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:16.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
-    - name: "Ubuntu 16.04 Python 3.5 Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:16.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
-    - name: "Ubuntu 18.04 Python 2.7 Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:18.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
-    - name: "Ubuntu 18.04 Python 3.6 Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:18.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
-    - name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:16.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh &&  apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
-    - name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests"
-      script:
-        - docker pull tensorflow/tensorflow:custom-op
-        - docker pull ubuntu:18.04
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
-        - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh &&  apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
+  - name: "Python 2.7 Build"
+    env: PYTHON_VERSION=2.7
+  - name: "Python 3.4 Build"
+    env: PYTHON_VERSION=3.4
+  - name: "Python 3.5 Build"
+    env: PYTHON_VERSION=3.5
+  - name: "Python 3.6 Build"
+    env: PYTHON_VERSION=3.6
+script:
+  - docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION}
 
 notifications:
   email: false
diff --git a/.travis/bazel.install.sh b/.travis/bazel.install.sh
@@ -0,0 +1,9 @@
+# Install bazel
+set -e -x
+
+BAZEL_OS=${1}
+BAZEL_VERSION=${2}
+curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
+chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
+./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
+rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
diff --git a/.travis/pip.install.sh b/.travis/pip.install.sh
@@ -0,0 +1,24 @@
+# Pin wheel==0.31.1 to work around issue
+# https://github.com/pypa/auditwheel/issues/102
+pip install -q wheel==0.31.1
+
+# Install last working version of setuptools. This must happen before we install
+# absl-py, which uses install_requires notation introduced in setuptools 20.5.
+pip install -q --upgrade setuptools==39.1.0
+
+# Install six.
+pip install -q --upgrade six==1.10.0
+
+# Install protobuf.
+pip install -q --upgrade protobuf==3.6.0
+
+# numpy needs to be installed from source to fix segfaults. See:
+# https://github.com/tensorflow/tensorflow/issues/6968
+# This workaround isn't needed for Ubuntu 16.04 or later.
+pip install -q --no-binary=:all: --upgrade numpy==1.14.5
+
+# Keras
+pip install -q keras_applications==1.0.6 --no-deps
+pip install -q keras_preprocessing==1.0.5 --no-deps
+
+pip install auditwheel==1.5.0
diff --git a/.travis/python.install.sh b/.travis/python.install.sh
@@ -0,0 +1,58 @@
+#/bin/bash
+set -x -e
+
+apt-get -y -qq update
+apt-get -y -qq install lsb-core
+if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then
+  apt-get -y -qq install libav-tools
+  if [[ ${1} == "2.7" ]]; then
+    rm -f /usr/local/bin/pip
+    ln -s /usr/local/bin/pip2 /usr/local/bin/pip
+    pip install auditwheel==1.5.0
+  elif [[ ${1} == "3.4" ]]; then
+    rm -f /usr/local/bin/pip
+    ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+    rm -f /usr/bin/python
+    ln -s /usr/bin/python3 /usr/bin/python
+    pip install auditwheel==1.5.0
+  elif [[ ${1} == "3.5" ]]; then
+    curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
+    chmod +x install_python3.5_pip_packages.sh
+    sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh
+    sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh
+    sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh
+    ./install_python3.5_pip_packages.sh
+    rm -f install_python3.5_pip_packages.sh 
+    rm -f /usr/bin/python
+    ln -s /usr/bin/python3.5 /usr/bin/python
+    rm -f /usr/local/bin/pip
+    ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip
+  elif [[ ${1} == "3.6" ]]; then
+    rm -f /usr/local/bin/pip3
+    curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
+    chmod +x install_python3.6_pip_packages.sh
+    sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh
+    sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh
+    sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh
+    sed -i 's/pip3 install/pip3 -q install/g' install_python3.6_pip_packages.sh
+    sed -i 's/tar xvf/tar xf/g' install_python3.6_pip_packages.sh
+    sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh
+    sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh
+    sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh
+    ./install_python3.6_pip_packages.sh
+    rm -f install_python3.6_pip_packages.sh make.log
+    rm -rf Python-3.6.1*
+    rm -f /usr/bin/python
+    ln -s /usr/local/bin/python3.6 /usr/bin/python
+    rm -f /usr/local/bin/pip
+    ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+  else
+    echo Python ${1} not supported!
+    exit 1
+  fi
+else
+  echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
+fi
+python --version
+pip --version
+pip freeze
diff --git a/.travis/python.release.sh b/.travis/python.release.sh
@@ -13,24 +13,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-set -e
-set -x
+set -e -x
+
 
 # Release:
-# docker run -it -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir  tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6>
+# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6>
+# Nightly:
+# export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S')
+# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> <tensorflow-version> <project-name> $TENSORFLOW_IO_VERSION
 
 if [[ -z "${1}" ]]; then
   echo "usage:" $0 "<2.7|3.4|3.5|3.6>"
   exit 1
 fi
 
-export PYTHON_VERSION=$1 
-export BAZEL_VERSION=${BAZEL_VERSION}
-export CUSTOM_OP=True
-export BAZEL_OS=linux
-.travis/python.configure.sh
-bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg
+BAZEL_VERSION=0.20.0
+PYTHON_VERSION=${1}
+shift
+TENSORFLOW_VERSION=1.12.0
+if [[ ! -z ${1} ]]; then
+  TENSORFLOW_VERSION=${1}
+  shift
+fi
 
+.travis/bazel.install.sh linux ${BAZEL_VERSION}
+
+.travis/python.install.sh ${PYTHON_VERSION}
+
+pip install -q tensorflow==${TENSORFLOW_VERSION}
+./configure.sh
+bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/...
+bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg
 bazel-bin/build_pip_pkg artifacts
 
 exit 0
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -0,0 +1,30 @@
+trigger:
+- azure-pipelines
+
+pool:
+  vmImage: 'Ubuntu-16.04'
+
+steps:
+- script: |
+    sudo apt-get -y -qq update
+    sudo apt-get -y -qq install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo apt-key fingerprint 0EBFCD88
+    sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    sudo apt-get -y -qq update
+    sudo apt-get -y -qq install docker-ce docker-ce-cli containerd.io
+    echo Success!
+  displayName: 'Install Docker CE'
+- script: |
+    pip install twine
+    echo Success!
+  displayName: 'Install Twine'
+- script: |
+    sudo bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
+    sudo bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
+    echo Success!
+  displayName: 'Setup Ignite/Kafka'
+- script: |
+    sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
+    echo Success!
+  displayName: 'Build TensorFlow IO'