diff --git a/knowledge_base/ci_cd_azure_devops/README.md b/knowledge_base/ci_cd_azure_devops/README.md new file mode 100644 index 0000000..43e0bcf --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/README.md @@ -0,0 +1,30 @@ +# Azure DevOps CI/CD + +This Declarative Automation Bundles example validates and deploys a serverless job with Azure DevOps OIDC. +See [Azure DevOps authentication](https://docs.databricks.com/aws/en/dev-tools/auth/provider-azure-devops). + +## Prerequisites + +- A Databricks workspace that supports serverless jobs. +- Permission to manage Databricks service principals and Azure DevOps environments. +- Databricks CLI v1.20.0 or above. + +## Usage + +1. Create a Databricks service principal for the pipeline. +2. Grant it workspace access and the required Unity Catalog privileges. +3. Create a federation policy for the service principal. +4. Set the issuer to `https://vstoken.dev.azure.com/`. +5. Set the subject to `p:////`. +6. Set the audience to [`api://AzureADTokenExchange`](https://docs.databricks.com/aws/en/dev-tools/auth/provider-azure-devops). +7. Set pipeline variables `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID`. +8. Create `staging` and `prod` Azure DevOps environments. +9. Add an approval check to the `prod` environment. +10. Copy `azure-pipelines.yml` to your repository root. Set `bundleDirectory` to your bundle path. +11. Open a pull request to run validation. + +| Trigger | Command and target | +|---|---| +| Pull request | `databricks bundle validate -t dev` | +| Push to `main` | `databricks bundle deploy -t staging` | +| Production approval | `databricks bundle deploy -t prod` | diff --git a/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml new file mode 100644 index 0000000..ab0ac7a --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/azure-pipelines.yml @@ -0,0 +1,71 @@ +trigger: + branches: + include: [main] + +pr: + branches: + include: [main] + +variables: + bundleDirectory: knowledge_base/ci_cd_azure_devops + +stages: + - stage: Validate + condition: eq(variables['Build.Reason'], 'PullRequest') + jobs: + - job: Validate + steps: + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh + displayName: Install Databricks CLI + - bash: databricks bundle validate -t dev + displayName: Validate bundle + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - stage: Staging + dependsOn: [] + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + jobs: + - deployment: DeployStaging + environment: staging + strategy: + runOnce: + deploy: + steps: + - checkout: self + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh + displayName: Install Databricks CLI + - bash: databricks bundle deploy -t staging + displayName: Deploy staging + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - stage: Prod + dependsOn: Staging + condition: succeeded() + jobs: + - deployment: DeployProd + environment: prod + strategy: + runOnce: + deploy: + steps: + - checkout: self + - bash: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/9b64c94a6d2b2e830592d80e9f14c26d32b42ad7/install.sh | sh + displayName: Install Databricks CLI + - bash: databricks bundle deploy -t prod + displayName: Deploy production + workingDirectory: $(bundleDirectory) + env: + DATABRICKS_AUTH_TYPE: azure-devops-oidc + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_CLIENT_ID: $(DATABRICKS_CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/knowledge_base/ci_cd_azure_devops/databricks.yml b/knowledge_base/ci_cd_azure_devops/databricks.yml new file mode 100644 index 0000000..d0a928f --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: ci_cd_azure_devops + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development + workspace: + # Give each developer a separate dev copy. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + staging: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml new file mode 100644 index 0000000..d153991 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/resources/serverless.job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + serverless_job: + name: ci-cd-azure-devops-${bundle.target} + tasks: + - task_key: hello + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/ci_cd_azure_devops/src/main.py b/knowledge_base/ci_cd_azure_devops/src/main.py new file mode 100644 index 0000000..db7b029 --- /dev/null +++ b/knowledge_base/ci_cd_azure_devops/src/main.py @@ -0,0 +1 @@ +print("Hello from Azure DevOps") diff --git a/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml new file mode 100644 index 0000000..b3ca0a2 --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/.github/workflows/bundle.yml @@ -0,0 +1,64 @@ +name: Databricks bundle + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + id-token: write + +defaults: + run: + working-directory: knowledge_base/ci_cd_github_actions + +jobs: + validate: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + environment: dev + concurrency: + group: databricks-bundle-dev + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 + - run: databricks bundle validate -t dev + + staging: + if: github.event_name == 'push' + runs-on: ubuntu-latest + environment: staging + concurrency: + group: databricks-bundle-staging + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 + - run: databricks bundle deploy -t staging + + prod: + if: github.event_name == 'push' + needs: staging + runs-on: ubuntu-latest + environment: prod + concurrency: + group: databricks-bundle-prod + cancel-in-progress: false + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + - uses: databricks/setup-cli@9b64c94a6d2b2e830592d80e9f14c26d32b42ad7 # v1.20.0 + - run: databricks bundle deploy -t prod diff --git a/knowledge_base/ci_cd_github_actions/README.md b/knowledge_base/ci_cd_github_actions/README.md new file mode 100644 index 0000000..e7e94c6 --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/README.md @@ -0,0 +1,31 @@ +# GitHub Actions CI/CD + +This Declarative Automation Bundles example validates and deploys a serverless job with GitHub Actions and OIDC. +See [GitHub Actions authentication](https://docs.databricks.com/aws/en/dev-tools/auth/provider-github). + +## Prerequisites + +- A Databricks workspace that supports serverless jobs. +- Permission to manage Databricks service principals and GitHub environments. +- Databricks CLI v1.20.0 or above. + +## Usage + +1. Create a Databricks service principal for the pipeline. +2. Grant it workspace access and the required Unity Catalog privileges. +3. Create federation policies with issuer `https://token.actions.githubusercontent.com`. +4. Use subject `repo:/:environment:dev` for dev. +5. Use subject `repo:/:environment:staging` for staging. +6. Use subject `repo:/:environment:prod` for prod. +7. Set the audience to your [Databricks account ID](https://docs.databricks.com/aws/en/dev-tools/auth/provider-github). +8. Create `dev`, `staging`, and `prod` GitHub environments. +9. Set `DATABRICKS_HOST` and `DATABRICKS_CLIENT_ID` in each environment. +10. Add required reviewers to the `prod` environment. +11. Copy `.github/workflows/bundle.yml` to your repository root. Set `working-directory` to your bundle path. +12. Open a pull request to run validation. + +| Trigger | Command and target | +|---|---| +| Pull request | `databricks bundle validate -t dev` | +| Push to `main` | `databricks bundle deploy -t staging` | +| Production approval | `databricks bundle deploy -t prod` | diff --git a/knowledge_base/ci_cd_github_actions/databricks.yml b/knowledge_base/ci_cd_github_actions/databricks.yml new file mode 100644 index 0000000..30c07ed --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: ci_cd_github_actions + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development + workspace: + # Give each developer a separate dev copy. + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + staging: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml new file mode 100644 index 0000000..bff0ad7 --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/resources/serverless.job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + serverless_job: + name: ci-cd-github-actions-${bundle.target} + tasks: + - task_key: hello + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/ci_cd_github_actions/src/main.py b/knowledge_base/ci_cd_github_actions/src/main.py new file mode 100644 index 0000000..a32fe8a --- /dev/null +++ b/knowledge_base/ci_cd_github_actions/src/main.py @@ -0,0 +1 @@ +print("Hello from GitHub Actions")