Regenerate mlops_stacks snapshot from MLOps Stacks v0.5

mlops_stacks/.github/workflows/README.md

@@ -1,7 +1,8 @@
 # CI/CD Workflow Definitions
 This directory contains CI/CD workflow definitions using [GitHub Actions](https://docs.github.com/en/actions),
-under ``workflows``. These workflows cover testing and deployment of both ML code (for model training, batch inference, etc) and the 
-Databricks ML resource definitions under ``mlops_stacks/resources``. 
+under ``workflows``. These workflows cover testing and deployment of both ML code (for model training, batch inference, etc) and 
+Databricks ML resource definitions. 
 
 To set up CI/CD for a new project,
-please refer to [ML resource config - set up CI CD](../../mlops_stacks/resources/README.md#set-up-ci-and-cd).
+please refer to [Setting up CI/CD](<../../README.md#Setting up CI/CD>) and following the [MLOps Setup Guide](../../docs/mlops-setup.md)
+to make sure your `WORKFLOW_TOKEN` secret has been properly set up with `Workflow` permissions.

mlops_stacks/.github/workflows/deploy-cicd.yml

@@ -0,0 +1,75 @@
+# This GitHub workflow sets up CI/CD workflows for a previously instantiated MLOps Stacks Project.
+name: Deploy CICD for Project Workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      project_name:
+        description: 'Project Name'
+        required: true
+        default: mlops_stacks
+
+env:
+  DATABRICKS_HOST: https://adb-xxxx.xx.azuredatabricks.net
+  ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }}
+  ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}
+  ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}
+
+
+jobs:
+  cicd:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get current timestamp
+        id: timestamp
+        run: |
+          echo "timestamp=$(date +'%s')" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          token: ${{ secrets.WORKFLOW_TOKEN }}
+      - uses: databricks/setup-cli@v0.236.0
+      - name: Convert Project Name to Only Alphanumeric Characters
+        run: |
+          PROJECT_NAME_ALPHA=$(echo "${{ github.event.inputs.project_name }}" | tr ' -' '_')
+          echo "PROJECT_NAME_ALPHA=$PROJECT_NAME_ALPHA" >> "$GITHUB_ENV"
+      - name: Install jq
+        run: sudo apt-get install jq
+      - name: Unzip Bundle and Append Parameters to Input JSON
+        id: unzip
+        run: |
+          tar -xzvf cicd.tar.gz
+          INPUT_CLOUD_1=$(jq -r '.input_cloud' cicd_params.json)
+          INPUT_CLOUD_2=$(jq -r '.input_cloud' "$PROJECT_NAME_ALPHA/project_params.json")
+          if [ "$INPUT_CLOUD_1" != "$INPUT_CLOUD_2" ]; then
+            printf "Error: CICD cloud '%s' does not match project cloud '%s'\n" "$INPUT_CLOUD_1" "$INPUT_CLOUD_2"
+            exit 1
+          fi
+          printf '%s' "$(jq '. += {"input_project_name":"${{ github.event.inputs.project_name }}"}' cicd_params.json)" > cicd_params.json
+          printf '%s' "$(jq -s '.[0] + .[1]' cicd_params.json "$PROJECT_NAME_ALPHA/project_params.json")" > cicd_params.json
+      - name: Update databricks.yml
+        id: update
+        run: |
+          echo -e "  staging:\n    variables:\n      catalog_name: staging\n    workspace:\n      host: https://adb-xxxx.xx.azuredatabricks.net\n\n  prod:\n    variables:\n      catalog_name: prod\n    workspace:\n      host: https://adb-xxxx.xx.azuredatabricks.net\n\n  test:\n    variables:\n      catalog_name: test\n    workspace:\n      host: https://adb-xxxx.xx.azuredatabricks.net" >> "$(PROJECT_NAME_ALPHA)\databricks.yml"
+      - name: Initialize Bundle
+        id: initialize
+        run: |
+          databricks bundle init ./cicd --config-file "cicd_params.json"
+      - name: Commit Changes
+        id: commit
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          git config --global user.name "Deploy CICD Bot"
+          git config --global user.email "noreply-cicd-bot@databricks.com"
+          git checkout -b add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }}
+          git add .github "$PROJECT_NAME_ALPHA/databricks.yml"
+          git commit -m "Add CICD for ${{ github.event.inputs.project_name }}"
+          git push origin add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }}
+
+      - name: Create Pull Request
+        id: pr
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          gh pr create --base main --head add-cicd-for-${{ github.event.inputs.project_name }}-${{ env.timestamp }} --title "Deploy CICD for ${{ github.event.inputs.project_name }}" --body "This PR was generated by the Deploy CICD workflow." --reviewer ${{ github.actor }}

mlops_stacks/.github/workflows/mlops_stacks-lint-cicd-workflow-files.yml → mlops_stacks/.github/workflows/lint-cicd-workflow-files.yml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download actionlint
         id: get_actionlint
         run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)

mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-prod.yml

@@ -18,14 +18,15 @@ env:
   ARM_TENANT_ID: ${{ secrets.PROD_AZURE_SP_TENANT_ID }}
   ARM_CLIENT_ID: ${{ secrets.PROD_AZURE_SP_APPLICATION_ID }}
   ARM_CLIENT_SECRET: ${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }}
+
 
 jobs:
   prod:
     concurrency: mlops_stacks-prod-bundle-job
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: databricks/setup-cli@main
+      - uses: actions/checkout@v4
+      - uses: databricks/setup-cli@v0.236.0
       - name: Validate Bundle For Prod
         id: validate
         run: |

mlops_stacks/.github/workflows/mlops_stacks-bundle-cd-staging.yml

@@ -18,14 +18,15 @@ env:
   ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }}
   ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}
   ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}
+
 
 jobs:
   staging:
     concurrency: mlops_stacks-staging-bundle-job
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: databricks/setup-cli@main
+      - uses: actions/checkout@v4
+      - uses: databricks/setup-cli@v0.236.0
       - name: Validate Bundle For Staging
         id: validate
         run: |

mlops_stacks/.github/workflows/mlops_stacks-bundle-ci.yml

@@ -5,7 +5,9 @@ name: Bundle validation for mlops_stacks
 
 on:
   workflow_dispatch:
-  pull_request_target:
+  pull_request:
+    paths:
+      - 'mlops_stacks/**'
 
 defaults:
   run:
@@ -18,26 +20,28 @@ env:
   PROD_ARM_TENANT_ID: ${{ secrets.PROD_AZURE_SP_TENANT_ID }}
   PROD_ARM_CLIENT_ID: ${{ secrets.PROD_AZURE_SP_APPLICATION_ID }}
   PROD_ARM_CLIENT_SECRET: ${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }}
+
 
 jobs:
   staging:
     concurrency: mlops_stacks-staging-bundle-job
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      - uses: databricks/setup-cli@main
+      - uses: databricks/setup-cli@v0.236.0
       - name: Validate Bundle For Staging
         id: validate
         env:
           ARM_TENANT_ID: ${{ env.STAGING_ARM_TENANT_ID }}
           ARM_CLIENT_ID: ${{ env.STAGING_ARM_CLIENT_ID }}
           ARM_CLIENT_SECRET: ${{ env.STAGING_ARM_CLIENT_SECRET }}
+
         run: |
           databricks bundle validate -t staging > ../validate_output.txt
       - name: Create Comment with Bundle Configuration
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         id: comment
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -62,22 +66,23 @@ jobs:
 
   prod:
     concurrency: mlops_stacks-prod-bundle-job
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      - uses: databricks/setup-cli@main
+      - uses: databricks/setup-cli@v0.236.0
       - name: Validate Bundle For Prod
         id: validate
         env:
           ARM_TENANT_ID: ${{ env.PROD_ARM_TENANT_ID }}
           ARM_CLIENT_ID: ${{ env.PROD_ARM_CLIENT_ID }}
           ARM_CLIENT_SECRET: ${{ env.PROD_ARM_CLIENT_SECRET }}
+
         run: |
           databricks bundle validate -t prod > ../validate_output.txt
       - name: Create Comment with Bundle Configuration
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         id: comment
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

mlops_stacks/.github/workflows/mlops_stacks-run-tests.yml

@@ -1,7 +1,10 @@
-name: ML Code Tests for mlops_stacks
+name: Training Unit and Integration Tests for mlops_stacks
 on:
   workflow_dispatch:
   pull_request:
+    paths:
+      - 'mlops_stacks/**'
+      - '.github/workflows/mlops_stacks-run-tests.yml'
 
 defaults:
   run:
@@ -11,17 +14,18 @@ env:
   ARM_TENANT_ID: ${{ secrets.STAGING_AZURE_SP_TENANT_ID }}
   ARM_CLIENT_ID: ${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}
   ARM_CLIENT_SECRET: ${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}
+
 
-concurrency: mlops_stacks-feature-training-integration-test-staging
+concurrency: mlops_stacks-training-integration-test-staging
 
 jobs:
   unit_tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install dependencies
         run: |
             python -m pip install --upgrade pip
@@ -33,11 +37,11 @@ jobs:
 
   integration_test:
     needs: unit_tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
-      - uses: databricks/setup-cli@main
+        uses: actions/checkout@v4
+      - uses: databricks/setup-cli@v0.236.0
       - name: Validate Bundle For Test Deployment Target in Staging Workspace
         id: validate
         run: |