[CI] add py3 sot ci yaml

.github/workflows/CI.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+permissions: read-all
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+  # PR_REF: ${{ github.event.pull_request.head.ref }}
+
+jobs:
+  Clone:
+    uses: ./.github/workflows/_Clone-linux.yml
+
+  SOT:
+    uses: ./.github/workflows/_SOT.yml
+    needs: Clone
+
+  Py3:
+    uses: ./.github/workflows/_Py3.yml
+    needs: Clone

.github/workflows/_Clone-linux.yml

@@ -0,0 +1,82 @@
+name: Clone-linux
+
+on:
+  workflow_call:
+
+permissions: read-all
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+
+jobs:
+  Clone:
+    # Don't run on forked repos.
+    if: github.repository_owner == 'PaddlePaddle'
+    runs-on:
+      group: HK-Clone
+
+    steps:
+    - name: Clone paddle
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.base.ref }}
+        submodules: 'recursive'
+        fetch-depth: 1000
+
+    - name: Merge PR to test branch
+      run: |
+        git config --unset http.https://github.com/.extraheader
+        git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
+        git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
+        git switch develop
+        set +e
+        git branch -D test
+        set -e
+        git gc
+        git switch -c test
+        git config --global user.name "PaddleCI"
+        git config --global user.email "paddle_ci@example.com"
+        git fetch origin pull/${{ github.event.pull_request.number }}/head:pr
+        git merge --no-ff pr
+        git branch -d pr
+        bash ${{ github.workspace }}/paddle/scripts/paddle_build/third_party_tag.sh
+
+    - name: Download bos client
+      env:
+        home_path: "/home/paddle/actions-runner/"
+        bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
+      run: |
+        if [ ! -f "${bos_file}" ]; then
+          wget -q --no-proxy -O ${home_path}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
+          mkdir ${home_path}/bos
+          tar xf ${home_path}/bos_new.tar.gz -C ${home_path}/bos
+        fi
+
+    - name: Push paddle-action.tar.gz to bos
+      env:
+        AK: paddle
+        SK: paddle
+        bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
+      run: |
+        cd ..
+        tar -zcf Paddle.tar.gz Paddle
+        echo "::group::Install bce-python-sdk"
+        python -m pip install bce-python-sdk==0.8.74
+        echo "::endgroup::"
+        python ${bos_file} Paddle.tar.gz paddle-github-action/PR/Paddle/${PR_ID}/${COMMIT_ID}
+        rm Paddle.tar.gz
+        cd -
+        git switch develop
+        git branch -D test
+        git gc
+
+    # - name: Clean environment
+    #   if: always()
+    #   run: |
+    #     cd ${{ github.workspace }}
+    #     rm -rf * .[^.]*

.github/workflows/_Py3.yml

@@ -0,0 +1,175 @@
+name: PR-CI-Py3
+
+on:
+  workflow_call:
+
+env:
+  dockerfile: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
+  docker_image: 077ca344c322
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+  container_name: paddle-CI-${{ github.event.pull_request.number }}-py3
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-test:
+    runs-on:
+      group: GZ_BD-CPU
+
+    steps:
+    - name: Check docker image and run container
+      env:
+        BRANCH: develop
+        work_dir: ${{ github.workspace }}
+        PADDLE_ROOT: ${{ github.workspace }}
+        WITH_SHARED_PHI: "ON"
+        FLAGS_fraction_of_gpu_memory_to_use: 0.15
+        CTEST_OUTPUT_ON_FAILURE: 1
+        CTEST_PARALLEL_LEVEL: 2
+        WITH_GPU: "OFF"
+        WITH_CACHE: "ON"
+        WITH_AVX: "OFF"
+        WITH_MKL: "OFF"
+        WITH_TESTING: "ON"
+        WITH_COVERAGE: "OFF"
+        COVERALLS_UPLOAD: "OFF"
+        GIT_PR_ID: ${{ github.event.pull_request.number }}
+        PADDLE_VERSION: 0.0.0
+        CMAKE_BUILD_TYPE: Release
+        PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
+        WITH_DISTRIBUTE: "ON"
+        RUN_TEST: "ON"
+        PRECISION_TEST: "OFF"
+        PREC_SUFFIX: .py3
+        WITH_UNITY_BUILD: "ON"
+        TIMEOUT_DEBUG_HEL: "OFF"
+        PY_VERSION: 3.9
+        CI_SKIP_CPP_TEST: "OFF"
+        PROC_RUN: 12
+        FLAGS_enable_eager_mode: 1
+        WITH_TENSORRT: "OFF"
+        WITH_NEWIR: "ON"
+        GENERATOR: "Ninja"
+        CCACHE_MAXSIZE: 150G
+        CCACHE_LIMIT_MULTIPLE: 0.8
+      run: |
+        # bash ${{ github.workspace }}/paddle/scripts/paddle_build/check_docker_image.sh
+        docker run -d -t --name ${container_name} \
+          -v "/home/data/cfs:/home/data/cfs" \
+          -v "/home/data/gzcfs:/home/data/gzcfs" \
+          -v "/dev/shm:/dev/shm" \
+          -v "${{ github.workspace }}/../../../..:${{ github.workspace }}/../../../.." \
+          -e BRANCH \
+          -e PR_ID \
+          -e COMMIT_ID \
+          -e work_dir \
+          -e PADDLE_ROOT \
+          -e WITH_SHARED_PHI \
+          -e FLAGS_fraction_of_gpu_memory_to_use \
+          -e CTEST_OUTPUT_ON_FAILURE \
+          -e CTEST_PARALLEL_LEVEL \
+          -e WITH_GPU \
+          -e WITH_CACHE \
+          -e WITH_AVX \
+          -e WITH_MKL \
+          -e WITH_TESTING \
+          -e WITH_COVERAGE \
+          -e COVERALLS_UPLOAD \
+          -e GIT_PR_ID \
+          -e PADDLE_VERSION \
+          -e CMAKE_BUILD_TYPE \
+          -e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
+          -e WITH_DISTRIBUTE \
+          -e RUN_TEST \
+          -e PRECISION_TEST \
+          -e PREC_SUFFIX \
+          -e WITH_UNITY_BUILD \
+          -e TIMEOUT_DEBUG_HEL \
+          -e PY_VERSION \
+          -e CI_SKIP_CPP_TEST \
+          -e PROC_RUN \
+          -e FLAGS_enable_eager_mode \
+          -e WITH_TENSORRT \
+          -e WITH_NEWIR \
+          -e GENERATOR \
+          -e WITH_INFERENCE_API_TEST \
+          -e CCACHE_MAXSIZE \
+          -e CCACHE_LIMIT_MULTIPLE \
+          -e GITHUB_ENV \
+          -w ${{ github.workspace }} --network host ${docker_image}
+
+    - name: Download paddle.tar.gz and merge develop
+      env:
+        work_dir: ${{ github.workspace }}
+      run: |
+        docker exec -t ${container_name} /bin/bash -c '
+        rm -rf * .[^.]*
+        rm -rf /root/.cache && ln -s /home/data/cfs/.cache/python35-cpu /root/.cache
+        wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
+        tar xf Paddle.tar.gz --strip-components=1
+        bash ${work_dir}/paddle/scripts/paddle_build/config_paddle.sh
+        . ../../../proxy
+        git checkout develop
+        git pull upstream develop
+        git fetch upstream develop
+        git checkout test
+        git merge --no-edit develop
+        '
+
+    - name: Configure cache and ccache
+      env:
+        work_dir: ${{ github.workspace }}
+      run: |
+        docker exec -t ${container_name} /bin/bash -c 'bash ${work_dir}/paddle/scripts/paddle_build/config_cfs.sh py3 ${{ github.env }}'
+
+    - name: Build
+      env:
+        work_dir: ${{ github.workspace }}
+      run: |
+        docker exec -t ${container_name} /bin/bash -c '
+        export CACHE_DIR=${{ env.CACHE_DIR }}
+        export CCACHE_DIR=${{ env.CCACHE_DIR }}
+        . ../../../proxy
+        bash ${work_dir}/paddle/scripts/paddle_build/run_setup.sh ${PYTHON_ABI:-""} bdist_wheel ${parallel_number:-""} py3
+        '
+
+    - name: Test
+      env:
+        work_dir: ${{ github.workspace }}
+      run: |
+        docker exec -t ${container_name} /bin/bash -c '
+        . ../../../proxy
+        bash ${work_dir}/paddle/scripts/paddle_build/run_linux_cpu_test.sh
+        source paddle/scripts/paddle_build/utils.sh; clean_build_files
+        echo "::group::Install bce-python-sdk"
+        python -m pip install bce-python-sdk==0.8.74
+        echo "::endgroup::"
+        '
+
+    - name: Upload paddle_whl
+      env:
+        home_path: ${{ github.workspace }}/..
+        bos_file: ${{ github.workspace }}/../bos/BosClient.py
+        paddle_whl: paddlepaddle-0.0.0-cp39-cp39-linux_x86_64.whl
+      run: |
+        docker exec -t ${container_name} /bin/bash -c '
+        export AK=paddle
+        export SK=paddle
+        if [ ! -f "${{ env.bos_file }}" ]; then
+          wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
+          mkdir ${{ env.home_path }}/bos
+          tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
+        fi
+        cd ${{ github.workspace }}/dist
+        python3.9 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
+        '
+
+    - name: Terminate and delete the container
+      if: always()
+      run: |
+        docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
+        docker stop ${container_name}
+        docker rm ${container_name}