native_jni_s3_paddle.yml

name: Native JNI S3 Paddle

on:
  workflow_dispatch:
  schedule:
    - cron: '0 10 * * *'

jobs:
  build-paddle-jni-cpu:
    if: github.repository == 'deepjavalibrary/djl'
    runs-on: ${{ matrix.operating-system }}
    strategy:
      matrix:
        operating-system: [ macos-latest, windows-latest ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          distribution: 'corretto'
          java-version: 17
      - uses: actions/cache@v3
        with:
          path: ~/.gradle/caches
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
          restore-keys: |
            ${{ runner.os }}-gradle-
      - name: Release JNI prep
        run: ./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI
      - name: Upload compiled jni library
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: jnilib-${{ runner.os }}
          path: engines/paddlepaddle/paddlepaddle-native/jnilib

  # A few things tried to build GPU jni (2.0.2) shared library:
  # 1. cpu JNI .so file works fine with cu101 and cu102
  # 2. cu101 and cu102 generate identical .so file, no point to build both of them
  # 3. centos:7 + GCC 8 is broken: symbol not found
  # 4. amazonlinux:2 + GCC 7.3.1 is broken: symbol not found (CPU works fine)
  # 5. ubuntu:18.04 + GCC 7.5.1 works fine
  # 6. ubuntu:16.04 + GCC 8 works fine (old pipeline use this, assume this works)
  build-paddle-jni-linux:
    if: github.repository == 'deepjavalibrary/djl'
    runs-on: ubuntu-latest
    container: amazonlinux:2
    steps:
      - name: Install Environment
        run: |
          yum -y update
          yum -y groupinstall "Development Tools"
          yum -y install patch cmake3
          ln -sf /usr/bin/cmake3 /usr/bin/cmake
          pip3 install awscli --upgrade
      - uses: actions/checkout@v3
      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: 'corretto'
          java-version: 17
      - uses: actions/cache@v3
        with:
          path: ~/.gradle/caches
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
          restore-keys: |
            ${{ runner.os }}-gradle-
      - name: Release JNI prep
        run: |
          ./gradlew :engines:paddlepaddle:paddlepaddle-native:compileJNI
      - name: Upload compiled jni library
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: jnilib-${{ runner.os }}
          path: engines/paddlepaddle/paddlepaddle-native/jnilib

  publish:
    if: github.repository == 'deepjavalibrary/djl'
    runs-on: ubuntu-latest
    needs: [ build-paddle-jni-cpu, build-paddle-jni-linux ]
    steps:
      - uses: actions/checkout@v3
      - name: Download compiledJNI Mac
        uses: actions/download-artifact@v3
        with:
          name: jnilib-macOS
          path: jnilib
      - name: Download compiledJNI Windows
        uses: actions/download-artifact@v3
        with:
          name: jnilib-Windows
          path: jnilib
      - name: Download compiledJNI Linux CPU
        uses: actions/download-artifact@v3
        with:
          name: jnilib-Linux
          path: jnilib
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1-node16
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Copy files to S3 with the AWS CLI
        run: |
          PADDLE_VERSION="$(cat gradle.properties | awk -F '=' '/paddlepaddle_version/ {print $2}')"
          aws s3 sync jnilib s3://djl-ai/publish/paddlepaddle-${PADDLE_VERSION}/jnilib
          aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/paddlepaddle-${PADDLE_VERSION}/jnilib*"