Add log forwarding Pebble layer to sparkd #85
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GPU integration CI pipeline | |
| on: | |
| pull_request: | |
| workflow_call: | |
| jobs: | |
| build-rock: | |
| uses: ./.github/workflows/build_gpu.yaml | |
| start-runner: | |
| name: Start self-hosted EC2 runner | |
| needs: [build-rock] | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2.3.6 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.WELPAOLO_GITHUB_TOKEN }} | |
| ec2-image-id: ${{ vars.AWS_EC2_IMAGE_ID }} | |
| ec2-instance-type: ${{ vars.AWS_EC2_INSTANCE_TYPE }} | |
| subnet-id: ${{ vars.AWS_DEFAULT_SUBNET_ID }} | |
| security-group-id: ${{ vars.AWS_SECURITY_GROUP_ID }} | |
| integration-tests: | |
| name: Run integration tests | |
| needs: [start-runner] | |
| runs-on: ${{ needs.start-runner.outputs.label }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup environment | |
| run: | | |
| sudo -E bash ./tests/integration/setup-environment.sh | |
| env: | |
| MICROK8S_CHANNEL: 1.28/stable | |
| - name: Setup microk8s | |
| timeout-minutes: 30 | |
| run: sudo -E /bin/bash ./tests/integration/config-microk8s-gpu.sh | |
| env: | |
| MICROK8S_ADDONS: "hostpath-storage dns rbac gpu minio" | |
| - name: Configure microk8s | |
| timeout-minutes: 5 | |
| run: | | |
| sudo microk8s.status | |
| sudo snap alias microk8s.kubectl kubectl | |
| mkdir -p /home/ubuntu/.kube | |
| sudo chown -f -R ubuntu /home/ubuntu/.kube | |
| sudo microk8s config | tee /home/ubuntu/.kube/config | |
| - name: Install required packages/snaps | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install build-essential -yqq | |
| sudo snap install yq | |
| - name: Get Artifact Name | |
| id: artifact | |
| run: | | |
| GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ') | |
| echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: charmed-spark-gpu | |
| path: charmed-spark | |
| - name: Load image | |
| run: | | |
| # Unpack Artifact | |
| mv charmed-spark/${{ steps.artifact.outputs.gpu_artifact_name }} . | |
| touch .make_cache/k8s.tag | |
| # Import artifact into microk8s to be used in integration tests | |
| sudo make microk8s-import FLAVOUR=spark-gpu PREFIX=test- REPOSITORY=ghcr.io/canonical/ \ | |
| -o ${{ steps.artifact.outputs.gpu_artifact_name }} | |
| - name: Run integration tests on GPU | |
| timeout-minutes: 45 | |
| run: | | |
| echo "Start test..." | |
| /bin/bash ./tests/integration/setup-aws-cli.sh | |
| /bin/bash ./tests/integration/integration-tests-gpu.sh | |
| echo "End of the test :)" | |
| env: | |
| KUBECONFIG: "/home/ubuntu/.kube/config" | |
| USER: "ubuntu" | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: | |
| - start-runner | |
| - integration-tests | |
| runs-on: ubuntu-22.04 | |
| if: ${{ always() }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2.3.6 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.WELPAOLO_GITHUB_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |