Update Pony to use LLVM 12.0.0

- Update the LLVM submodule to `llvmorg-12.0.0` - Update the JIT code to work with LLVM 12. - Update custom optimizations to work with LLVM 12. - Move old JIT and optimization code to separate files for historical reference - Update handling LLVM library dependencies: - Consolidate LLVM library configuration in the root `CMakeLists.txt` file. - Deprecate the `llvm_config` cmake macro in favour of using individual components with `llvm_map_components_to_libnames`. - Include the in-progress work on Mac M1: - Update CMakeLists.txt to require C++14 (LLVM now does). - Add missing header to cpu.c that is required on Apple M1. - Fixes #1930
ponylang · Jul 13, 2021 · c657540 · c657540
1 parent 665e729
commit c657540
Show file tree

Hide file tree

Showing 32 changed files with 2,637 additions and 599 deletions.
diff --git a/.ci-dockerfiles/x86-64-pc-windows-msvc-builder/Dockerfile b/.ci-dockerfiles/x86-64-pc-windows-msvc-builder/Dockerfile
@@ -4,7 +4,7 @@ FROM cirrusci/windowsservercore:2019
 
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
-ADD https://download.visualstudio.microsoft.com/download/pr/9b3476ff-6d0a-4ff8-956d-270147f21cd4/ccfb9355f4f753315455542f966025f96de734292d3908c8c3717e9685b709f0/vs_BuildTools.exe C:\vs_BuildTools.exe
+ADD https://download.visualstudio.microsoft.com/download/pr/3105fcfe-e771-41d6-9a1c-fc971e7d03a7/67584dbce4956291c76a5deabbadc15c9ad6a50d6a5f4458765b6bd3a924aead/vs_BuildTools.exe C:\vs_BuildTools.exe
 
 RUN C:\vs_BuildTools.exe --quiet --wait --norestart --nocache --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended
 RUN choco install cmake -y --no-progress --installargs 'ADD_CMAKE_TO_PATH=System'

diff --git a/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile b/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile
@@ -16,7 +16,7 @@ RUN apk update \
   zlib-dev \
   bash \
   curl \
-  py-pip \
+  py3-pip \
 && pip install cloudsmith-cli
 
 # add user pony in order to not run tests as root

diff --git a/.ci-dockerfiles/x86-64-unknown-linux-ubuntu18.04-builder/Dockerfile b/.ci-dockerfiles/x86-64-unknown-linux-ubuntu18.04-builder/Dockerfile
@@ -8,13 +8,14 @@ RUN apt-get update \
   curl \
   git \
   make \
-  python-pip \
+  python3-pip \
+  python3-setuptools \
   xz-utils \
   zlib1g-dev \
  && rm -rf /var/lib/apt/lists/* \
  && apt-get -y autoremove --purge \
  && apt-get -y clean \
- && pip install cloudsmith-cli
+ && pip3 install cloudsmith-cli
 
 # install a newer cmake
 RUN curl --output cmake-3.15.3-Linux-x86_64.sh https://cmake.org/files/v3.15/cmake-3.15.3-Linux-x86_64.sh \

diff --git a/.cirrus.yml b/.cirrus.yml
@@ -29,9 +29,9 @@ task:
       TRIPLE_OS: linux-gnu
   - name: "PR: x86-64-unknown-linux-ubuntu18.04"
     container:
-      image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+      image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
     environment:
-      CACHE_BUSTER: 20210224
+      CACHE_BUSTER: 20210427
       TRIPLE_VENDOR: unknown
       TRIPLE_OS: linux-ubuntu18.04
   - name: "PR: x86-64-unknown-linux-ubuntu20.04"
@@ -43,7 +43,7 @@ task:
       TRIPLE_OS: linux-ubuntu20.04
   - name: "PR: x86-64-unknown-linux-musl"
     container:
-      image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+      image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
     environment:
       CACHE_BUSTER: 20210224
       TRIPLE_VENDOR: unknown
@@ -143,17 +143,19 @@ task:
   only_if: $CIRRUS_PR != ''
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210430
     os_version: 2019
     cpu: 8
     memory: 24
 
   name: "PR: x86-64-pc-windows-msvc"
 
+  timeout_in: 120m
+
   libs_cache:
     folder: build/libs
     fingerprint_script:
-      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20201208"
+      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20210430"
     populate_script:
       - ps: .\make.ps1 -Command libs -Generator "Visual Studio 16 2019"
 
@@ -168,7 +170,7 @@ task:
   only_if: $CIRRUS_PR != ''
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210430
     os_version: 2019
     cpu: 8
     memory: 24
@@ -180,7 +182,7 @@ task:
   libs_cache:
     folder: build/libs
     fingerprint_script:
-      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20201208"
+      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20210430"
     populate_script:
       - ps: .\make.ps1 -Command libs -Generator "Visual Studio 16 2019"
 
@@ -191,58 +193,58 @@ task:
   test_script:
     - ps: .\make.ps1 -Command test -Config Debug -Generator "Visual Studio 16 2019"
 
-
-task:
-  only_if: $CIRRUS_PR != ''
-
-  container:
-    image: ponylang/ponyc-ci-cross-arm:20210430
-    cpu: 8
-    memory: 24
-
-  name: "PR: cross-compile: arm-unknown-linux-gnueabi"
-
-  libs_cache:
-    folder: build/libs
-    fingerprint_script: echo "`md5sum lib/CMakeLists.txt` cross arm 20210430"
-    populate_script: make libs build_flags=-j8 llvm_archs="X86;ARM"
-
-  configure_script:
-    - make configure
-  build_script:
-    - make build build_flags=-j8
-  test_script:
-    - make test-ci
-  libponyrt_cross_script:
-    - make cross-libponyrt CC=arm-linux-gnueabi-gcc CXX=arm-linux-gnueabi-g++ arch=armv7-a tune=cortex-a9 llc_arch=arm
-  test_stdlib_cross_script:
-    - make test-cross-ci PONYPATH=../armv7-a/release cross_triple=arm-unknown-linux-gnueabi cross_arch=armv7-a cross_cpu=cortex-a9 cross_linker=arm-linux-gnueabi-gcc cross_runner="qemu-arm-static -cpu cortex-a9 -L /usr/local/arm-linux-gnueabi/libc"
-
-task:
-  only_if: $CIRRUS_PR != ''
-
-  container:
-    image: ponylang/ponyc-ci-cross-armhf:20210430
-    cpu: 8
-    memory: 24
-
-  name: "PR: cross-compile: arm-linux-gnueabihf-gcc"
-
-  libs_cache:
-    folder: build/libs
-    fingerprint_script: echo "`md5sum lib/CMakeLists.txt` cross arm 20210430"
-    populate_script: make libs build_flags=-j8 llvm_archs="X86;ARM"
-
-  configure_script:
-    - make configure
-  build_script:
-    - make build build_flags=-j8
-  test_script:
-    - make test-ci
-  libponyrt_cross_script:
-    - make cross-libponyrt CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ arch=armv7-a tune=cortex-a9 llc_arch=arm
-  test_stdlib_cross_script:
-    - make test-cross-ci PONYPATH=../armv7-a/release cross_triple=arm-unknown-linux-gnueabihf cross_arch=armv7-a cross_cpu=cortex-a9 cross_linker=arm-linux-gnueabihf-gcc cross_runner="qemu-arm-static -cpu cortex-a9 -L /usr/local/arm-linux-gnueabihf/libc"
+# TODO: re-enable these (and add aarch64) once 32-bit ARM and M1 fixes have been made
+# task:
+#   only_if: $CIRRUS_PR != ''
+
+#   container:
+#     image: ponylang/ponyc-ci-cross-arm:20210430
+#     cpu: 8
+#     memory: 24
+
+#   name: "PR: cross-compile: arm-unknown-linux-gnueabi"
+
+#   libs_cache:
+#     folder: build/libs
+#     fingerprint_script: echo "`md5sum lib/CMakeLists.txt` cross arm 20210430"
+#     populate_script: make libs build_flags=-j8
+
+#   configure_script:
+#     - make configure
+#   build_script:
+#     - make build build_flags=-j8
+#   test_script:
+#     - make test-ci
+#   libponyrt_cross_script:
+#     - make cross-libponyrt CC=arm-linux-gnueabi-gcc CXX=arm-linux-gnueabi-g++ arch=armv7-a cross_cflags="-march=armv7-a -mtune=cortex-a9" cross_lflags="-O3;-march=arm"
+#   test_stdlib_cross_script:
+#     - make test-cross-ci PONYPATH=../armv7-a/release cross_triple=arm-unknown-linux-gnueabi cross_arch=armv7-a cross_cpu=cortex-a9 cross_linker=arm-linux-gnueabi-gcc cross_runner="qemu-arm-static -cpu cortex-a9 -L /usr/local/arm-linux-gnueabi/libc"
+
+# task:
+#   only_if: $CIRRUS_PR != ''
+
+#   container:
+#     image: ponylang/ponyc-ci-cross-armhf:20210430
+#     cpu: 8
+#     memory: 24
+
+#   name: "PR: cross-compile: arm-linux-gnueabihf-gcc"
+
+#   libs_cache:
+#     folder: build/libs
+#     fingerprint_script: echo "`md5sum lib/CMakeLists.txt` cross arm 20210430"
+#     populate_script: make libs build_flags=-j8
+
+#   configure_script:
+#     - make configure
+#   build_script:
+#     - make build build_flags=-j8
+#   test_script:
+#     - make test-ci
+#   libponyrt_cross_script:
+#     - make cross-libponyrt CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ arch=armv7-a cross_cflags="-march=armv7-a -mtune=cortex-a9" cross_lflags="-O3;-march=arm"
+#   test_stdlib_cross_script:
+#     - make test-cross-ci PONYPATH=../armv7-a/release cross_triple=arm-unknown-linux-gnueabihf cross_arch=armv7-a cross_cpu=cortex-a9 cross_linker=arm-linux-gnueabihf-gcc cross_runner="qemu-arm-static -cpu cortex-a9 -L /usr/local/arm-linux-gnueabihf/libc"
 
 #
 # Nightly build tasks
@@ -275,9 +277,9 @@ task:
         TRIPLE_OS: linux-gnu
     - name: "nightly: x86-64-unknown-linux-ubuntu18.04"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
       environment:
-        CACHE_BUSTER: 20210224
+        CACHE_BUSTER: 20210427
         TRIPLE_VENDOR: unknown
         TRIPLE_OS: linux-ubuntu18.04
     - name: "nightly: x86-64-unknown-linux-ubuntu20.04"
@@ -289,7 +291,7 @@ task:
         TRIPLE_OS: linux-ubuntu20.04
     - name: "nightly: x86-64-unknown-linux-musl"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
       environment:
         CACHE_BUSTER: 20210224
         TRIPLE_VENDOR: unknown
@@ -369,7 +371,7 @@ task:
   only_if: $CIRRUS_CRON == "nightly"
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210430
     os_version: 2019
     cpu: 8
     memory: 24
@@ -382,7 +384,7 @@ task:
   libs_cache:
     folder: build/libs
     fingerprint_script:
-      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20201208"
+      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20210430"
     populate_script:
       - ps: .\make.ps1 -Command libs -Generator "Visual Studio 16 2019"
 
@@ -428,9 +430,9 @@ task:
         TRIPLE_OS: linux-gnu
     - name: "release: x86-64-unknown-linux-ubuntu18.04"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
       environment:
-        CACHE_BUSTER: 20200507
+        CACHE_BUSTER: 20210427
         TRIPLE_VENDOR: unknown
         TRIPLE_OS: linux-ubuntu18.04
     - name: "release: x86-64-unknown-linux-ubuntu20.04"
@@ -442,7 +444,7 @@ task:
         TRIPLE_OS: linux-ubuntu20.04
     - name: "release: x86-64-unknown-linux-musl"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
       environment:
         CACHE_BUSTER: 20200421
         TRIPLE_VENDOR: unknown
@@ -522,7 +524,7 @@ task:
   only_if: $CIRRUS_TAG =~ '^\d+\.\d+\.\d+$'
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210430
     os_version: 2019
     cpu: 8
     memory: 24
@@ -535,7 +537,7 @@ task:
   libs_cache:
     folder: build/libs
     fingerprint_script:
-      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20201208"
+      - ps: (Get-FileHash -Path lib\CMakeLists.txt).Hash + "Windows 20210430"
     populate_script:
       - ps: .\make.ps1 -Command libs -Generator "Visual Studio 16 2019"
 

diff --git a/.release-notes/3745.md b/.release-notes/3745.md
@@ -0,0 +1,5 @@
+## Update to LLVM 12.0.1
+
+We've updated the LLVM used to build pony to LLVM 12.0.1 and in the process, we've dropped support for 32-bit ARM as supported platform.
+
+We might bring 32-bit Arm back as a supported platform if the Arm fixes we need to do to get ponyc working on M1 also fix the current issues we have with 32-bit Arm with LLVM 12. The errors currently present the same so we assume that adding M1 support will bring 32-bit ARM along with it. If fixing the M1 issues doesn't fix 32-bit Arm issues then we'll have to make a decision about whether we bring back 32-bit Arm support.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,6 +18,10 @@ endif()
 # Uncomment this to show build commands
 # set(CMAKE_VERBOSE_MAKEFILE ON)
 
+# We require C++14 (because LLVM does)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 # We require LLVM, Google Test and Google Benchmark
 if(NOT PONY_CROSS_LIBPONYRT)
     find_package(LLVM REQUIRED CONFIG PATHS "build/libs/lib/cmake/llvm" "build/libs/lib64/cmake/llvm" NO_DEFAULT_PATH)
@@ -103,6 +107,33 @@ if("${_compiler_arch}" STREQUAL "")
     set(_compiler_arch ${CMAKE_SYSTEM_PROCESSOR})
 endif()
 
+# LLVM component setup
+message("Compiler architecture is ${_compiler_arch}")
+
+if(NOT PONY_CROSS_LIBPONYRT)
+    set(LLVM_COMPONENTS
+        core
+        demangle
+        objcarcopts
+        orcjit
+        x86asmparser
+        x86codegen
+        x86desc
+        x86info
+        aarch64asmparser
+        aarch64codegen
+        aarch64desc
+        aarch64info
+        armasmparser
+        armcodegen
+        armdesc
+        arminfo
+    )
+
+    llvm_map_components_to_libnames(PONYC_LLVM_LIBS ${LLVM_COMPONENTS})
+    # message("PONYC_LLVM_LIBS: ${PONYC_LLVM_LIBS}")
+endif()
+
 # Required definitions.  We use these generators so that the defines are correct for both *nix (where the config applies at configuration time) and Windows (where the config applies at build time).
 add_compile_definitions(
     BUILD_COMPILER="${CMAKE_C_COMPILER_VERSION}"
@@ -175,9 +206,6 @@ if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "DragonFly")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/local/cxx_atomics")
 endif()
 
-
-
-
 set(CMAKE_STATIC_LIBRARY_PREFIX "")
 
 if (PONY_CROSS_LIBPONYRT)

diff --git a/Makefile b/Makefile
@@ -2,7 +2,7 @@ config ?= release
 arch ?= native
 tune ?= generic
 build_flags ?= -j2
-llvm_archs ?= X86
+llvm_archs ?= X86;ARM;AArch64
 llvm_config ?= Release
 llc_arch ?= x86-64
 
@@ -166,7 +166,7 @@ crossBuildDir := $(srcDir)/build/$(arch)/build_$(config)
 
 cross-libponyrt:
 	$(SILENT)mkdir -p $(crossBuildDir)
-	$(SILENT)cd '$(crossBuildDir)' && env CC=$(CC) CXX=$(CXX) cmake -B '$(crossBuildDir)' -S '$(srcDir)' -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=$(arch) -DCMAKE_C_COMPILER=$(CC) -DCMAKE_CXX_COMPILER=$(CXX) -DPONY_CROSS_LIBPONYRT=true -DCMAKE_BUILD_TYPE=$(config) -DCMAKE_C_FLAGS="-march=$(arch) -mtune=$(tune)" -DCMAKE_CXX_FLAGS="-march=$(arch) -mtune=$(tune)" -DPONYC_VERSION=$(version) -DLL_FLAGS="-O3;-march=$(llc_arch);-mcpu=$(tune)"  $(CMAKE_FLAGS)
+	$(SILENT)cd '$(crossBuildDir)' && env CC=$(CC) CXX=$(CXX) cmake -B '$(crossBuildDir)' -S '$(srcDir)' -DCMAKE_CROSSCOMPILING=true -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=$(arch) -DCMAKE_C_COMPILER=$(CC) -DCMAKE_CXX_COMPILER=$(CXX) -DPONY_CROSS_LIBPONYRT=true -DCMAKE_BUILD_TYPE=$(config) -DCMAKE_C_FLAGS="$(cross_cflags)" -DCMAKE_CXX_FLAGS="$(cross_cflags)" -DPONY_ARCH=$(arch) -DPONYC_VERSION=$(version) -DLL_FLAGS="$(cross_llc_flags)"  $(CMAKE_FLAGS)
 	$(SILENT)cd '$(crossBuildDir)' && env CC=$(CC) CXX=$(CXX) cmake --build '$(crossBuildDir)' --config $(config) --target libponyrt -- $(build_flags)
 
 test: all test-core test-stdlib-release test-examples

diff --git a/benchmark/libponyc/CMakeLists.txt b/benchmark/libponyc/CMakeLists.txt
@@ -13,8 +13,6 @@ target_include_directories(libponyc.benchmarks
     PRIVATE ../../build/libs/include
 )
 
-llvm_config(libponyc.benchmarks all)
-
 target_link_directories(libponyc.benchmarks
     PRIVATE ../../build/libs/lib
 )
@@ -33,14 +31,12 @@ if (MSVC)
         PRIVATE Ws2_32
         PRIVATE Shlwapi
     )
-
-    get_target_property(TARGET_LIBRARIES libponyc.benchmarks LINK_LIBRARIES)
-    LIST(REMOVE_ITEM TARGET_LIBRARIES LTO)
-    set_property(TARGET libponyc.benchmarks PROPERTY LINK_LIBRARIES ${TARGET_LIBRARIES})
 elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
     target_link_libraries(libponyc.benchmarks PRIVATE dl)
-elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "BSD|DragonFly")
+elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "BSD")
     target_link_libraries(libponyc.benchmarks PRIVATE execinfo)
+elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "DragonFly")
+    target_link_libraries(libponyc.benchmarks PRIVATE execinfo atomic)
 else()
     target_link_libraries(libponyc.benchmarks PRIVATE atomic dl)
     target_link_options(libponyc.benchmarks PRIVATE "-static-libstdc++")
@@ -49,3 +45,7 @@ endif ()
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     target_link_options(libponyc.benchmarks PRIVATE "-static-libgcc")
 endif()
+
+if(NOT PONY_CROSS_LIBPONYRT)
+    target_link_libraries(libponyc.benchmarks PRIVATE ${PONYC_LLVM_LIBS})
+endif()