testing train.yml #47
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Train model, evaluate and test champion model, build containerized model, and deploy model | |
| name: Data Prep | |
| on: | |
| push: | |
| branches: ["workflows"] | |
| workflow_dispatch: | |
| # schedule: | |
| # - cron: '00 4 * * *' # Runs at 4:00 AM UTC every day | |
| jobs: | |
| deploy-azure-resources: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checks-out repository under $GITHUB_WORKSPACE, so the job can access it | |
| # Use cache action to cache the virtual environment (https://stackoverflow.com/a/62639424) | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Login to Azure | |
| uses: azure/#@v2 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # # Deploys ARM template to create Azure resources for training | |
| # - name: Deploy ARM Template | |
| # id: deploy-arm | |
| # run: | | |
| # az deployment group create --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --template-file ./infrastructure/azure_train_resource.json --name TrainDeployment | |
| # KEY_VAULT_NAME=$(az deployment group show --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --name TrainDeployment --query 'properties.outputs.keyVaultName.value' -o tsv) | |
| # echo "KEY_VAULT_NAME=$KEY_VAULT_NAME" >> $GITHUB_ENV | |
| # echo "key_vault_name=$KEY_VAULT_NAME" >> $GITHUB_ENV | |
| # env: | |
| # GITHUB_OUTPUT: ${{ github.output }} | |
| # - name: Check and Update AML Environment if Necessary | |
| # run: | | |
| # echo "Setting up Azure credentials..." | |
| # AZURE_CREDENTIALS=$(echo '${{ secrets.AZURE_CREDENTIALS }}') | |
| # CLIENT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.clientId') | |
| # CLIENT_SECRET=$(echo $AZURE_CREDENTIALS | jq -r '.clientSecret') | |
| # TENANT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.tenantId') | |
| # echo "Upgrading Azure CLI..." | |
| # az upgrade --yes --allow-preview false | |
| # echo "Adding/upgrading Azure Machine Learning CLI extension..." | |
| # az extension add --name ml --upgrade --yes --allow-preview false | |
| # echo "Logging in to Azure..." | |
| # az login --service-principal -u $CLIENT_ID -p $CLIENT_SECRET --tenant $TENANT_ID | |
| # echo "Creating/updating Azure Machine Learning environment from YAML file..." | |
| # az ml environment create --file aml-train-env.yml --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --workspace-name ${{ vars.AML_WORKSPACE_NAME }} | |
| # - name: Check if AML environment is updated | |
| # id: diff | |
| # run: | | |
| # echo "Setting up Azure credentials..." | |
| # AZURE_CREDENTIALS=$(echo '${{ secrets.AZURE_CREDENTIALS }}') | |
| # CLIENT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.clientId') | |
| # CLIENT_SECRET=$(echo $AZURE_CREDENTIALS | jq -r '.clientSecret') | |
| # TENANT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.tenantId') | |
| # echo "Upgrading Azure CLI..." | |
| # az upgrade --yes --allow-preview false | |
| # echo "Adding/upgrading Azure Machine Learning CLI extension..." | |
| # az extension add --name ml --upgrade --yes --allow-preview false | |
| # echo "Logging in to Azure..." | |
| # az login --service-principal -u $CLIENT_ID -p $CLIENT_SECRET --tenant $TENANT_ID | |
| # LATEST_VERSION=$(az ml environment list --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --workspace-name ${{ vars.AML_WORKSPACE_NAME }} --query "[?name=='${{ vars.AML_TRAIN_ENV_NAME }}'].version" --output tsv | sort -V | tail -n1) | |
| # if [ -z "$LATEST_VERSION" ]; then | |
| # echo "Environment ${{ vars.AML_TRAIN_ENV_NAME }} not found. Setting update_required to true." | |
| # echo "update_required=true" >> $GITHUB_ENV | |
| # else | |
| # echo "Latest version of ${{ vars.AML_TRAIN_ENV_NAME }} is $LATEST_VERSION" | |
| # az ml environment show --name ${{ vars.AML_TRAIN_ENV_NAME }} --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --workspace-name ${{ vars.AML_WORKSPACE_NAME }} --version $LATEST_VERSION --debug > current-env.yml | |
| # if diff current-env.yml train-conda.yml; then | |
| # echo "No changes in the environment. Skipping update." | |
| # echo "update_required=false" >> $GITHUB_ENV | |
| # else | |
| # echo "Changes detected. Updating environment." | |
| # echo "update_required=true" >> $GITHUB_ENV | |
| # fi | |
| # fi | |
| # - name: Create or update Azure ML Environment | |
| # if: steps.diff.outputs.update_required == 'true' | |
| # run: | | |
| # echo "Setting up Azure credentials..." | |
| # AZURE_CREDENTIALS=$(echo '${{ secrets.AZURE_CREDENTIALS }}') | |
| # CLIENT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.clientId') | |
| # CLIENT_SECRET=$(echo $AZURE_CREDENTIALS | jq -r '.clientSecret') | |
| # TENANT_ID=$(echo $AZURE_CREDENTIALS | jq -r '.tenantId') | |
| # echo "Upgrading Azure CLI..." | |
| # az upgrade --yes --allow-preview false | |
| # echo "Adding/upgrading Azure Machine Learning CLI extension..." | |
| # az extension add --name ml --upgrade --yes --allow-preview false | |
| # echo "Logging in to Azure..." | |
| # az login --service-principal -u $CLIENT_ID -p $CLIENT_SECRET --tenant $TENANT_ID | |
| # echo "Creating/updating Azure Machine Learning environment from YAML file..." | |
| # az ml environment create --file aml-train-env.yml --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --workspace-name ${{ vars.AML_WORKSPACE_NAME }} | |
| # data-prep: | |
| # runs-on: ubuntu-latest | |
| # needs: deploy-azure-resources | |
| # steps: | |
| # - name: Checkout code | |
| # uses: actions/checkout@v4 | |
| # - name: Set up Python 3.10.* | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: 3.10.* | |
| # - name: Get pip cache dir | |
| # id: pip-cache | |
| # run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV | |
| # # Use cache action to cache the virtual environment | |
| # - name: Cache pip dependencies | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ${{ env.PIP_CACHE_DIR }} | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: | | |
| # make install | |
| # - name: Add project path to sys.path | |
| # run: | | |
| # echo "PYTHONPATH=${{github.workspace}}" >> $GITHUB_ENV | |
| # - name: Prepare data | |
| # run: | | |
| # make prep_data | |
| # - name: Setup Feast | |
| # run: | | |
| # make setup_feast | |
| # - name: Split data | |
| # run: | | |
| # make split_data | |
| train: | |
| runs-on: ubuntu-latest | |
| # needs: data-prep | |
| steps: | |
| # Checks-out repository under $GITHUB_WORKSPACE, so the job can access it | |
| # Use cache action to cache the virtual environment (https://stackoverflow.com/a/62639424) | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.10.* | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.10.* | |
| - name: Get pip cache dir | |
| id: pip-cache | |
| run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV | |
| # Use cache action to cache the virtual environment | |
| - name: Cache pip dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| make install | |
| - name: Add project path to sys.path | |
| run: | | |
| echo "PYTHONPATH=${{ github.workspace }}" >> $GITHUB_ENV | |
| - name: Login to Azure | |
| uses: azure/#@v2 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Submit train.py to Azure Compute Cluster | |
| run: | | |
| echo "Upgrading Azure CLI..." | |
| az upgrade --yes --allow-preview false | |
| echo "Adding/upgrading Azure Machine Learning CLI extension..." | |
| az extension add --name ml --upgrade --yes --allow-preview false | |
| echo "Submitting training script to Azure ML Compute..." | |
| az ml job create --file ./src/training/train.yml --resource-group ${{ vars.RESOURCE_GROUP_NAME }} --workspace-name ${{ vars.AML_WORKSPACE_NAME }} |