Bugfix/issue 923 cross compile fail

.github/workflows/manual_trigger_build.yml

@@ -27,8 +27,8 @@ on: # yamllint disable-line rule:truthy
         required: true
         type: string
         default: 'manual'
-      enable_native_build_for_macOS:
-        description: 'Experimental try to build macos binary native'
+      build_for_macOS:
+        description: 'build macos binary, upload to s3 bucket'
         required: false
         default: 'false'
         type: choice
@@ -39,7 +39,7 @@ on: # yamllint disable-line rule:truthy
 jobs:
   prepare_sanitizer:
     runs-on: ubuntu-latest
-    if: github.event.inputs.enable_native_build_for_macOS == 'false' && github.event.inputs.build_type != 'release'
+    if: github.event.inputs.build_for_macOS == 'false' && github.event.inputs.build_type != 'release'
     outputs:
       command: |
         cd $GITHUB_WORKSPACE
@@ -87,7 +87,7 @@ jobs:
           echo 'command: ${{ steps.set_command.outputs.command }}'
   prepare_release:
     runs-on: ubuntu-latest
-    if:  github.event.inputs.enable_native_build_for_macOS == 'false' && github.event.inputs.build_type == 'release'
+    if:  github.event.inputs.build_for_macOS == 'false' && github.event.inputs.build_type == 'release'
     outputs:
       command: |
         cd $GITHUB_WORKSPACE
@@ -220,7 +220,7 @@ jobs:
     env:
       build_directory: ${{ github.workspace }}/build
       build_type: RelWithDebInfo
-    if: github.event.inputs.enable_native_build_for_macOS == 'true' && github.event.inputs.arch == 'arm'
+    if: github.event.inputs.build_for_macOS == 'true' && github.event.inputs.arch == 'arm'
     steps:
       - name: Checkout
         uses: actions/checkout@v4.1.7
@@ -264,3 +264,95 @@ jobs:
           ARCH=$(uname -m)
           STRIPPED_BINARY=${{ env.build_directory }}/programs/proton_${COMMIT_HASH}_${ARCH}
           aws s3 cp --no-progress $STRIPPED_BINARY s3://tp-internal/proton/native_build_macOS/
+  Build_macOS_X86_64:
+    if: github.event.inputs.build_for_macOS == 'true' && github.event.inputs.arch == 'x64'
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: c7i.8xlarge
+      ec2-image-id: ami-042a37e33a285c22b
+      submodules: 'recursive'
+      run_mode: 'start'
+      command: |
+        cd $GITHUB_WORKSPACE
+
+        # git config
+        git config user.name "proton-robot"
+        git config user.email "proton_robot@timeplus.io"
+
+        mkdir -p $GITHUB_WORKSPACE/ccache    
+        # Download pre-built v8 binary for cross-compilation
+        aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
+        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
+        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
+
+        # compiling
+        ./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-19-darwin
+
+        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
+          echo "Compiling proton Failed"
+          exit 127
+        fi
+
+        # get proton tag and rename binary
+        COMMIT_HASH=$(git rev-parse --short HEAD)
+        PROTON_BINARY=proton-${COMMIT_HASH}-Darwin-x86_64
+        echo "PROTON_BINARY=${PROTON_BINARY}" >> $GITHUB_ENV
+
+        # Only strip if strip_binary is true  
+        echo "Stripping the binary..."
+        ls -lh $GITHUB_WORKSPACE/output/proton
+        docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-19 /usr/bin/llvm-strip-19 proton
+        ls -lh $GITHUB_WORKSPACE/output/proton
+
+        # Calculate checksum
+        sha256sum $GITHUB_WORKSPACE/output/proton > $GITHUB_WORKSPACE/output/proton.sha256
+        cat $GITHUB_WORKSPACE/output/proton.sha256
+
+        # Rename binary
+        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/output/${PROTON_BINARY}
+
+        # S3 upload with advanced retry mechanism
+        max_attempts=5
+        attempt=1
+        wait_time=15
+
+        while [ $attempt -le $max_attempts ]; do
+          echo "S3 upload attempt $attempt of $max_attempts"
+
+          if aws s3 cp \
+             --no-progress \
+             --cli-connect-timeout 30 \
+             --cli-read-timeout 600 \
+             "$GITHUB_WORKSPACE/output/${PROTON_BINARY}" \
+             "s3://tp-internal/proton/cross-compile/macOS-x86/"; then
+            echo "Binary upload successful"
+            break
+          else
+            upload_status=$?
+            echo "Binary upload failed on attempt $attempt with exit code $upload_status"
+
+            if [ $attempt -eq $max_attempts ]; then
+              echo "All binary upload attempts failed"
+              exit 1
+            fi
+
+            echo "Retrying in $wait_time seconds..."
+            sleep $wait_time
+            wait_time=$((wait_time * 2))  # Exponential backoff
+            attempt=$((attempt + 1))
+          fi
+        done
+
+        echo "Build and upload complete for ${PROTON_BINARY}"
+        echo "S3 location: s3://tp-internal/proton/cross-compile/macOS-x86/${PROTON_BINARY}"
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

docker/packager/binary/Dockerfile

@@ -1,4 +1,4 @@
-# rebuild in https://github.com/timeplus-io/proton/pull/921
+# rebuild in https://github.com/timeplus-io/proton/pull/924
 # docker build -t timeplus/proton-binary-builder .
 FROM ubuntu:20.04
 
@@ -41,7 +41,7 @@ RUN apt-get update \
         build-essential \
         ccache \
         clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
+        libdispatch-dev \
         cmake \
         fakeroot \
         gdb \
@@ -69,18 +69,33 @@ RUN apt-get update \
 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
 
-# This symlink is required by packager script
-RUN ln -s /usr/bin/clang-tidy-${LLVM_VERSION} /usr/bin/clang-tidy
-
-# python udf require
-RUN ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-tblgen /usr/bin/llvm-tblgen
-RUN ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-profdata /usr/bin/llvm-profdata
-
 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 
-# Copy pre-built cctools from the clickhouse/cctools image
-COPY --from=clickhouse/cctools:d9e3596e706b /cctools /cctools
+# libtapi is required to support .tbh format from recent MacOS SDKs
+RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
+    && cd apple-libtapi \
+    && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
+    && INSTALLPREFIX=/cctools ./build.sh \
+    && ./install.sh \
+    && cd .. \
+    && rm -rf apple-libtapi
+
+# Build and install tools for cross-linking to Darwin (x86-64)
+# Build and install tools for cross-linking to Darwin (aarch64)
+RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=x86_64-apple-darwin \
+    && make install -j$(nproc) \
+    && make clean \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=aarch64-apple-darwin \
+    && make install -j$(nproc) \
+    && cd ../.. \
+    && rm -rf cctools-port
+
 
 # Download toolchain and SDK for Darwin
 RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz

docker/packager/cctools/Dockerfile

@@ -0,0 +1,107 @@
+# docker build -t timeplus/cctools .
+
+# This is a hack to significantly reduce the build time of the timeplus/binary-builder
+# It's based on the assumption that we don't care of the cctools version so much
+# It event does not depend on the timeplus/fasttest in the `docker/images.json`
+FROM ubuntu:20.04 AS builder
+
+# ARG for quick switch to a given ubuntu mirror
+ARG apt_archive="http://archive.ubuntu.com"
+ARG LLVM_VERSION=17
+ARG LIBTAPI_COMMIT=15dfc2a8c9a2a89d06ff227560a69f5265b692f9
+ARG CCTOOLS_COMMIT=2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7
+
+RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get install \
+        apt-transport-https \
+        apt-utils \
+        ca-certificates \
+        dnsutils \
+        gnupg \
+        iputils-ping \
+        lsb-release \
+        wget \
+        gpg \
+        curl \
+        --yes --no-install-recommends --verbose-versions \
+    && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
+    && wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
+    && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
+    && apt-key add /tmp/llvm-snapshot.gpg.key \
+    && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
+    && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
+        /etc/apt/sources.list \
+    && curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg \
+    && echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
+
+# initial packages
+RUN apt-get update \
+    && apt-get install \
+        bash \
+        build-essential \
+        clang-${LLVM_VERSION} \
+        cmake \
+        fakeroot \
+        zlib1g-dev \
+        git \
+        lld-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION}-dev \
+        libclang-rt-${LLVM_VERSION}-dev \
+        moreutils \
+        ninja-build \
+        pigz \
+        rename \
+        software-properties-common \
+        tzdata \
+        automake \
+        m4 \
+        texinfo \
+        libtool \
+        --yes --no-install-recommends \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
+
+# This symlink required by gcc to find lld compiler
+RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+
+ENV CC=clang-${LLVM_VERSION}
+ENV CXX=clang++-${LLVM_VERSION}
+
+# Build and install libtapi (required for cctools)
+RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
+    && cd apple-libtapi \
+    && git checkout ${LIBTAPI_COMMIT} \
+    && INSTALLPREFIX=/cctools ./build.sh \
+    && ./install.sh \
+    && cd .. \
+    && rm -rf apple-libtapi
+
+# Build and install tools for cross-linking to Darwin (x86-64 and aarch64)
+RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && git checkout ${CCTOOLS_COMMIT} \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=x86_64-apple-darwin \
+    && make install -j$(nproc) \
+    && make clean \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=aarch64-apple-darwin \
+    && make install -j$(nproc) \
+    && cd ../.. \
+    && rm -rf cctools-port
+
+# Verify the installation by checking if key binaries exist
+RUN ls -la /cctools/bin/x86_64-apple-darwin-* && \
+    ls -la /cctools/bin/aarch64-apple-darwin-* && \
+    echo "Verification complete: cctools built successfully"
+
+# Final stage: copy only the cctools directory
+FROM scratch
+COPY --from=builder /cctools /cctools

docker/test/proton_unit_test/Dockerfile

@@ -1,6 +1,5 @@
 # docker build -t ghcr.io/timeplus-io/proton-unit-test .
 ARG FROM_TAG=fae41f7606bb
-# https://github.com/timeplus-io/proton-enterprise/issues/5737
 FROM clickhouse/unit-test:$FROM_TAG
 
 COPY unit_tests_dbms /