Simplify Android build and bring up to date (#273)

* Use latest Android platform and build-tools versions

* Require ANDROID_NDK environment variable instead of ANDROID_NDK_ROOT

* Upgrade gradle wrapper and build.gradle to support gradle 4.4.1 and Android gradle plugin 3.0.1

* Add BoringSSL as a submodule according to https://boringssl.googlesource.com/boringssl/+/HEAD/INCORPORATING.md

* Add build.gradle and dummy manifest for BoringSSL

* Add BoringSSL as a dependency to the main Android themis build.gradle

* Update Circle CI configuration to support updated Android build

* Limit JVM memory for gradle to 1GB

Otherwise, builds are failing with oom

* Disable parallel build for gradle and limit number of workers

* Split Android test into separate build and test stages

Circle CI fails with out of memory. To reduce memory usage we will compile Themis before starting the emulator. This should also fail the build early if there are any issues with Android build without having to wait for emulator to boot (which takes a while).

* Disable gradle deamon in CI

* Limit CMake/Ninja build concurrency for Circle CI builds of BoringSSL for Android

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/boringssl/src"]
+	path = third_party/boringssl/src
+	url = https://boringssl.googlesource.com/boringssl

build.gradle

@@ -1,20 +1,23 @@
-import org.apache.tools.ant.taskdefs.condition.Os
+apply plugin: 'com.android.library'
 
 buildscript {
     repositories {
-        mavenCentral()
+        google()
+        jcenter()
     }
 
     dependencies {
-        classpath 'com.android.tools.build:gradle:1.1.3'
+        classpath 'com.android.tools.build:gradle:3.0.1'
     }
 }
 
-apply plugin: 'com.android.library'
+dependencies {
+    implementation project(":boringssl")
+}
 
 android {
-    compileSdkVersion 22
-    buildToolsVersion "22.0.1"
+    compileSdkVersion 27
+    buildToolsVersion "27.0.3"
 
     defaultConfig {
     	minSdkVersion 16
@@ -33,37 +36,12 @@ android {
 
 	}
 
-    task ndkBuild(type: Exec) {
-	workingDir 'jni'
-        commandLine getNdkBuildCmd() 
-	if (project.hasProperty('boringssl_path')){
-	    args = ["BORINGSSL_PATH=$boringssl_path"]
-	}
-    }
-
-    tasks.withType(JavaCompile) {
-        compileTask -> compileTask.dependsOn ndkBuild
-    }
+    // ensure we execute boringssl tasks first
+    tasks.whenTaskAdded({Task task -> task.dependsOn('boringssl:' + task.name)})
 
-    task cleanNative(type: Exec) {
-		workingDir 'jni'
-        commandLine getNdkBuildCmd(), 'clean'
-    }
-
-    clean.dependsOn cleanNative
-}
-
-def getNdkDir() {
-    if (System.env.ANDROID_NDK_ROOT != null)
-        return System.env.ANDROID_NDK_ROOT
-
-    throw new GradleException("NDK location not found. Define location with an ANDROID_NDK_ROOT environment variable.")
-}
-
-def getNdkBuildCmd() {
-    def ndkbuild = getNdkDir() + "/ndk-build"
-    if (Os.isFamily(Os.FAMILY_WINDOWS))
-        ndkbuild += ".cmd"
-
-    return ndkbuild
+	externalNativeBuild {
+	    ndkBuild {
+	        path "jni/Android.mk"
+	    }
+	}
 }

circle.yml

@@ -1,4 +1,6 @@
 machine:
+  java:
+    version: openjdk8
   environment:
     ANDROID_NDK_ROOT: $ANDROID_NDK
     GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
@@ -7,7 +9,7 @@ machine:
     PATH: $GOROOT/bin:$PATH
     VALGRIND_BUILD_PATH: $HOME/valgrind
     # to avoid OOM killer (https://circleci.com/docs/1.0/oom/#out-of-memory-errors-in-android-builds)
-    GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError"'
+    GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError"'
     # add define that turn off one nist test (tests/soter/soter_rand_test.c:190) that always fail on ci machine but ok on real machine
     CFLAGS: "-DCIRICLE_TEST"
     BORINGSSL_PATH: "$HOME/boringssl"
@@ -29,8 +31,13 @@ dependencies:
     # link from http://valgrind.org/downloads/current.html
     # don't fetch if was cached
     - if [ ! -d $VALGRIND_BUILD_PATH ]; then wget ftp://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2 && tar -xjf valgrind-3.13.0.tar.bz2 && cd valgrind-3.13.0 && ./configure --prefix=$VALGRIND_BUILD_PATH && make && sudo make install; fi
+    # install Android SDK packages
+    - echo y | android update sdk --no-ui --all --filter 'tool'
+    - yes | $ANDROID_HOME/tools/bin/sdkmanager 'tools' 'platform-tools' 'build-tools;27.0.3' 'platforms;android-27' 'ndk-bundle' 'system-images;android-22;default;armeabi-v7a'
+    - $ANDROID_HOME/tools/bin/avdmanager create avd --name nexus --device "Nexus 5" --package 'system-images;android-22;default;armeabi-v7a'
 
   override:
+    - git submodule update --init
     - make
     - make JAVA_HOME=/usr/lib/jvm/default-java themis_jni
     - sudo make install
@@ -39,15 +46,21 @@ dependencies:
     - sudo make rubythemis_install
     - sudo make phpthemis_install
     - if [ ! -d $BORINGSSL_PATH ]; then cd $HOME && git clone https://boringssl.googlesource.com/boringssl && cd boringssl && git checkout chromium-stable && mkdir build && cd build && cmake .. && make && cp decrepit/libdecrepit.a crypto/; fi
-    - if [ ! -d $BORINGSSL_PATH/build-armeabi-v7a ]; then cd $BORINGSSL_PATH && mkdir build-armeabi-v7a && cd build-armeabi-v7a && cmake -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
-    - if [ ! -d $BORINGSSL_PATH/build-arm64-v8a ]; then cd $BORINGSSL_PATH && mkdir build-arm64-v8a && cd build-arm64-v8a && cmake -DANDROID_ABI=arm64-v8a -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
-    - if [ ! -d $BORINGSSL_PATH/build-x86 ]; then cd $BORINGSSL_PATH && mkdir build-x86 && cd build-x86 && cmake -DANDROID_ABI=x86 -DCMAKE_TOOLCHAIN_FILE=../third_party/android-cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=16 -GNinja .. && ninja -j 20; fi
     - make ENGINE=boringssl ENGINE_INCLUDE_PATH=$HOME/boringssl/include ENGINE_LIB_PATH=$HOME/boringssl/build/crypto BUILD_PATH=build_with_boringssl prepare_tests_basic
     - make BUILD_PATH=cover_build COVERAGE=y prepare_tests_basic
     - make prepare_tests_all
     - mkdir -p $HOME/gopath/src/$GOTHEMIS_IMPORT
     - rsync -auv gothemis/ $HOME/gopath/src/$GOTHEMIS_IMPORT/
     - lcov --directory . --zerocounters
+
+compile:
+  override:
+    # limit CMake/Ninja build concurrency when building BoringSSL
+    # otherwise we hit the 4GB memory limit for the build container
+    - echo 'set_property(GLOBAL APPEND PROPERTY JOB_POOLS circleci_job_pool=4)' >> third_party/boringssl/src/CMakeLists.txt
+    - sed -i 's/"-GNinja"/"-DCMAKE_JOB_POOL_COMPILE=circleci_job_pool", "-GNinja"/g' third_party/boringssl/build.gradle
+    - ./gradlew --no-daemon --no-parallel --max-workers=2 build
+
 ## Customize test commands
 test:
   override:
@@ -73,10 +86,11 @@ test:
     - ${VALGRIND_BUILD_PATH}/bin/valgrind build_with_boringssl/tests/soter_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
     - ${VALGRIND_BUILD_PATH}/bin/valgrind build_with_boringssl/tests/themis_test 2>&1 | grep "ERROR SUMMARY\|definitely lost\|indirectly lost\|possibly lost" | awk '{sum += $4} END {print $0; if ( sum > 0 ) { exit 1 } }'
     # - tests/check_ios_test.sh `tests/start_ios_test.sh`
-    # start Android emulator
-    - emulator -avd circleci-android22 -no-audio -no-window:
+    # start Android emulator (we use explicit path, because the update above
+    # installs new emulator version, which overrides Circle CIs one )
+    - $ANDROID_HOME/emulator/emulator -avd nexus -noaudio -no-window -gpu off -verbose -qemu:
         background: true
     # wait for it to have booted
     - circle-android wait-for-boot
     # run Android tests
-    - ./gradlew --info -Pboringssl_path=$HOME/boringssl connectedAndroidTest
+    - ./gradlew --no-daemon --no-parallel --max-workers=2 connectedAndroidTest

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Mon Jun 01 10:43:23 EEST 2015
+#Wed Dec 27 12:10:05 UTC 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip

jni/Android.mk

@@ -14,12 +14,14 @@
 # limitations under the License.
 #
 
-LOCAL_PATH := $(call my-dir)
+BUILD_TYPE := release
 
-ifeq ($(BORINGSSL_PATH),)
-    $(error "boringSSL path must be set")
+ifeq ($(NDK_DEBUG),1)
+  BUILD_TYPE := debug
 endif
 
+LOCAL_PATH := $(call my-dir)
+
 include $(CLEAR_VARS)
 
 LOCAL_MODULE := libsoter
@@ -28,10 +30,10 @@ LOCAL_SRC_FILES := $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/*.c
 LOCAL_SRC_FILES += $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/boringssl/*.c))
 LOCAL_SRC_FILES += $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/soter/ed25519/*.c))
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
 LOCAL_EXPORT_CFLAGS := -DBORINGSSL
-LOCAL_STATIC_LIBRARIES := libcrypto libdecrepit
+LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 
 include $(BUILD_STATIC_LIBRARY)
 
@@ -41,8 +43,8 @@ LOCAL_MODULE := libthemis
 
 LOCAL_SRC_FILES := $(patsubst jni/%,%, $(wildcard $(LOCAL_PATH)/../src/themis/*.c))
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src 
-LOCAL_STATIC_LIBRARIES := libsoter libcrypto libdecrepit
+LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 
 include $(BUILD_STATIC_LIBRARY)
 
@@ -54,8 +56,8 @@ LOCAL_SRC_FILES := themis_jni.c themis_message.c themis_keygen.c themis_cell.c t
 LOCAL_SRC_FILES += themis_compare.c
 LOCAL_CFLAGS := -DBORINGSSL -DCRYPTO_ENGINE_PATH=boringssl
 LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/../third_party/boringssl/src/include
 LOCAL_STATIC_LIBRARIES := libthemis libsoter libcrypto libdecrepit
-# LOCAL_LDFLAGS := -llog
 
 include $(BUILD_SHARED_LIBRARY)
 
@@ -85,26 +87,24 @@ LOCAL_SHARED_LIBRARIES := libthemis_jni
 
 LOCAL_C_INCLUDES := $(LOCAL_PATH)/../src $(LOCAL_PATH)/../tests
 
-
 include $(BUILD_EXECUTABLE)
 
-# this is needed to disable targets not present in the NDK otherwise error happens
-#BUILD_HOST_EXECUTABLE := 
-#BUILD_HOST_STATIC_LIBRARY := 
-
-#include $(LOCAL_PATH)/external/openssl/Android.mk
-
-# We are making a separate target with slightly modified script to support building libcrypto with NDK
-# We put it here to make minimal changes in original Google scripts to make importing new versions easier
 include $(CLEAR_VARS)
 
-#LOCAL_MODULE_TAGS := optional
 LOCAL_MODULE := libcrypto
-LOCAL_SRC_FILES := $(BORINGSSL_PATH)/build-$(TARGET_ARCH_ABI)/crypto/libcrypto.a
-LOCAL_EXPORT_C_INCLUDES := $(BORINGSSL_PATH)/include
-include $(PREBUILT_STATIC_LIBRARY)
+LOCAL_SRC_FILES := $(LOCAL_PATH)/../third_party/boringssl/.externalNativeBuild/cmake/$(BUILD_TYPE)/$(TARGET_ARCH_ABI)/crypto/libcrypto.a
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/../third_party/boringssl/src/include
+
+ifneq (,$(wildcard $(LOCAL_SRC_FILES)))
+  include $(PREBUILT_STATIC_LIBRARY)
+endif
+
+include $(CLEAR_VARS)
 
 LOCAL_MODULE := libdecrepit
-LOCAL_SRC_FILES := $(BORINGSSL_PATH)/build-$(TARGET_ARCH_ABI)/decrepit/libdecrepit.a
-include $(PREBUILT_STATIC_LIBRARY)
+LOCAL_SRC_FILES := $(LOCAL_PATH)/../third_party/boringssl/.externalNativeBuild/cmake/$(BUILD_TYPE)/$(TARGET_ARCH_ABI)/decrepit/libdecrepit.a
+LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/../third_party/boringssl/src/include
 
+ifneq (,$(wildcard $(LOCAL_SRC_FILES)))
+  include $(PREBUILT_STATIC_LIBRARY)
+endif

jni/Application.mk

@@ -14,6 +14,11 @@
 # limitations under the License.
 #
 
-APP_PLATFORM := android-16
+APP_PLATFORM := android-21
 APP_MODULES := libthemis_jni
-APP_ABI := armeabi-v7a arm64-v8a x86
+# we needs this to prevent ndk-build complaining about missing
+# prebuilt boringssl static libraries
+APP_ALLOW_MISSING_DEPS := true
+# currently boringssl does not build well with clang for all archs
+# so we use GCC for Themis as well for maximum compatibility
+NDK_TOOLCHAIN_VERSION := 4.9

settings.gradle

@@ -0,0 +1,2 @@
+include ':boringssl'
+project(':boringssl').projectDir = new File('third_party/boringssl')

third_party/boringssl/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.cossacklabs.themis.boringssl" android:versionCode="1" android:versionName="0.9.6">
+</manifest>

third_party/boringssl/build.gradle

@@ -0,0 +1,43 @@
+buildscript {
+    repositories {
+        google()
+        jcenter()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.0.1'
+    }
+}
+
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 27
+    buildToolsVersion "27.0.3"
+
+    defaultConfig {
+        minSdkVersion 16
+        targetSdkVersion 16
+        externalNativeBuild {
+		    cmake {
+		        arguments "-DCMAKE_TOOLCHAIN_FILE=" + android.ndkDirectory + "/build/cmake/android.toolchain.cmake",
+		                  "-DANDROID_NATIVE_API_LEVEL=21",
+		                  "-DANDROID_TOOLCHAIN=gcc",
+		                  "-DCMAKE_BUILD_TYPE=Release",
+		                  "-GNinja"
+		    }
+	    }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path "src/CMakeLists.txt"
+        }
+    }
+
+	sourceSets {
+        main {
+            manifest.srcFile 'AndroidManifest.xml'
+        }
+	}
+}